Merge remote-tracking branch 'upstream/dev' into feat/disk-skills

# Conflicts:
#	apps/x/apps/main/src/main.ts
#	apps/x/packages/core/src/application/assistant/instructions.ts
This commit is contained in:
Prakhar Pandey 2026-07-06 19:11:36 +05:30
commit 43590ca6d9
159 changed files with 18109 additions and 2589 deletions

View file

@ -199,6 +199,7 @@ module.exports = {
],
extendInfo: {
NSAudioCaptureUsageDescription: 'Rowboat needs access to system audio to transcribe meetings from other apps (Zoom, Meet, etc.)',
NSCameraUsageDescription: 'Rowboat uses your camera in video chat mode so the assistant can see you and give feedback (e.g. pitch practice).',
},
osxSign: {
batchCodesignCalls: true,

View file

@ -13,8 +13,8 @@
"make": "electron-forge make"
},
"dependencies": {
"@agentclientprotocol/claude-agent-acp": "^0.39.0",
"@agentclientprotocol/codex-acp": "^0.0.44",
"@agentclientprotocol/claude-agent-acp": "^0.55.0",
"@agentclientprotocol/codex-acp": "^1.1.0",
"@x/core": "workspace:*",
"@x/shared": "workspace:*",
"agent-slack": "0.9.3",

View file

@ -1,6 +1,6 @@
import { BrowserWindow } from 'electron';
import { ipc } from '@x/shared';
import { browserViewManager, type BrowserState } from './view.js';
import { browserViewManager, type BrowserState, type HttpAuthRequest } from './view.js';
type IPCChannels = ipc.IPCChannels;
@ -20,6 +20,7 @@ type BrowserHandlers = {
'browser:forward': InvokeHandler<'browser:forward'>;
'browser:reload': InvokeHandler<'browser:reload'>;
'browser:getState': InvokeHandler<'browser:getState'>;
'browser:httpAuthResponse': InvokeHandler<'browser:httpAuthResponse'>;
};
/**
@ -62,6 +63,9 @@ export const browserIpcHandlers: BrowserHandlers = {
'browser:getState': async () => {
return browserViewManager.getState();
},
'browser:httpAuthResponse': async (_event, args) => {
return browserViewManager.respondToHttpAuth(args);
},
};
/**
@ -70,12 +74,26 @@ export const browserIpcHandlers: BrowserHandlers = {
* window is created so the manager has a window to attach to.
*/
export function setupBrowserEventForwarding(): void {
browserViewManager.on('state-updated', (state: BrowserState) => {
const windows = BrowserWindow.getAllWindows();
for (const win of windows) {
if (!win.isDestroyed() && win.webContents) {
win.webContents.send('browser:didUpdateState', state);
}
// Only send to app windows, never to OAuth/SSO popup windows created by
// page window.open() — those render untrusted web content, and browsing
// state / auth-challenge metadata must not cross into them.
const broadcast = (channel: string, payload: unknown) => {
for (const win of BrowserWindow.getAllWindows()) {
if (win.isDestroyed() || !win.webContents) continue;
if (browserViewManager.isPopupWindow(win)) continue;
win.webContents.send(channel, payload);
}
};
browserViewManager.on('state-updated', (state: BrowserState) => {
broadcast('browser:didUpdateState', state);
});
browserViewManager.on('http-auth-request', (request: HttpAuthRequest) => {
broadcast('browser:httpAuthRequest', request);
});
browserViewManager.on('http-auth-resolved', (requestId: string) => {
broadcast('browser:httpAuthResolved', { requestId });
});
}

View file

@ -1,11 +1,12 @@
import { randomUUID } from 'node:crypto';
import { EventEmitter } from 'node:events';
import { BrowserWindow, WebContentsView, session, shell, type Session } from 'electron';
import { BrowserWindow, WebContentsView, session, shell, type Session, type WebContents } from 'electron';
import type {
BrowserPageElement,
BrowserPageSnapshot,
BrowserState,
BrowserTabState,
HttpAuthRequest,
} from '@x/shared/dist/browser-control.js';
import { normalizeNavigationTarget } from './navigation.js';
import {
@ -20,7 +21,7 @@ import {
type RawBrowserPageSnapshot,
} from './page-scripts.js';
export type { BrowserPageSnapshot, BrowserState, BrowserTabState };
export type { BrowserPageSnapshot, BrowserState, BrowserTabState, HttpAuthRequest };
/**
* Embedded browser pane implementation.
@ -36,13 +37,33 @@ export type { BrowserPageSnapshot, BrowserState, BrowserTabState };
export const BROWSER_PARTITION = 'persist:rowboat-browser';
// Claims Chrome 130 on macOS — close enough to recent stable for OAuth servers
// that sniff the UA looking for "real browser" shapes.
const SPOOF_UA =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36';
// Spoof a real Chrome UA so OAuth servers don't reject the embedded browser.
// The Chrome major version is derived from the running Chromium at startup:
// pinning a fixed version goes stale as Electron upgrades, and Chromium keeps
// emitting Sec-CH-UA client hints with the *real* version — a UA/client-hint
// version mismatch is a classic bot-detection signal (Google sign-in,
// Cloudflare). Minor version is frozen at 0.0.0, exactly like real Chrome's
// reduced UA. The platform token matches the actual OS for the same reason.
function getChromeMajorVersion(): number {
const major = Number.parseInt(process.versions.chrome ?? '', 10);
return Number.isFinite(major) && major > 0 ? major : 130;
}
function buildChromeUserAgent(): string {
const platformToken =
process.platform === 'darwin'
? 'Macintosh; Intel Mac OS X 10_15_7'
: process.platform === 'win32'
? 'Windows NT 10.0; Win64; x64'
: 'X11; Linux x86_64';
return `Mozilla/5.0 (${platformToken}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${getChromeMajorVersion()}.0.0.0 Safari/537.36`;
}
const SPOOF_UA = buildChromeUserAgent();
const HOME_URL = 'https://www.google.com';
const NAVIGATION_TIMEOUT_MS = 10000;
const HTTP_AUTH_TIMEOUT_MS = 120000;
const POST_ACTION_IDLE_MS = 400;
const POST_ACTION_MAX_ELEMENTS = 25;
const POST_ACTION_MAX_TEXT_LENGTH = 4000;
@ -68,6 +89,13 @@ type CachedSnapshot = {
elements: Array<{ index: number; selector: string }>;
};
type PendingHttpAuth = {
callback: (username?: string, password?: string) => void;
timer: NodeJS.Timeout;
// The webContents that raised the challenge, so its teardown can cancel it.
webContents: WebContents;
};
const EMPTY_STATE: BrowserState = {
activeTabId: null,
tabs: [],
@ -109,6 +137,12 @@ export class BrowserViewManager extends EventEmitter {
private visible = false;
private bounds: BrowserBounds = { x: 0, y: 0, width: 0, height: 0 };
private snapshotCache = new Map<string, CachedSnapshot>();
private pendingHttpAuth = new Map<string, PendingHttpAuth>();
// Child windows created by page window.open() (OAuth/SSO popups). Tracked so
// they can be closed when the host window goes away — otherwise an orphaned
// popup keeps BrowserWindow.getAllWindows() non-empty and, on macOS, blocks
// the app from reopening via the Dock (see main.ts 'activate' handler).
private popupWindows = new Set<BrowserWindow>();
private cleanupWindowListeners: (() => void) | null = null;
attach(window: BrowserWindow): void {
@ -137,6 +171,7 @@ export class BrowserViewManager extends EventEmitter {
if (this.window !== window) return;
const tabs = [...this.tabs.values()];
const popups = [...this.popupWindows];
this.cleanupWindowListeners = null;
this.window = null;
this.browserSession = null;
@ -150,6 +185,14 @@ export class BrowserViewManager extends EventEmitter {
this.attachedTabId = null;
this.visible = false;
this.snapshotCache.clear();
for (const requestId of [...this.pendingHttpAuth.keys()]) {
this.finishHttpAuth(requestId);
}
// Close any OAuth/SSO popups so they don't outlive the app window.
for (const popup of popups) {
if (!popup.isDestroyed()) popup.close();
}
this.popupWindows.clear();
};
hostWebContents.on('did-start-loading', handleDidStartLoading);
@ -171,6 +214,36 @@ export class BrowserViewManager extends EventEmitter {
if (this.browserSession) return this.browserSession;
const browserSession = session.fromPartition(BROWSER_PARTITION);
browserSession.setUserAgent(SPOOF_UA);
// Electron's Sec-CH-UA client hints only carry the "Chromium" brand;
// real Chrome also sends "Google Chrome". Some sign-in flows (notably
// Google's) distinguish the two, so rewrite the brand list to match what
// Chrome sends. Both the low-entropy header (`sec-ch-ua`, major versions)
// and the high-entropy one (`sec-ch-ua-full-version-list`, requested via
// Accept-CH and carrying full versions) must be rewritten together — a
// header that claims "Google Chrome" alongside one that doesn't is a
// stronger bot signal than the original. Only headers Chromium already
// attached are rewritten — none are added. (navigator.userAgentData JS
// brands still report only Chromium; there is no reliable hook to spoof
// that under sandbox+contextIsolation, and header-based detection is the
// common case.)
const chromeMajor = getChromeMajorVersion();
const chromeFull = process.versions.chrome ?? `${chromeMajor}.0.0.0`;
const brandLists: Record<string, string> = {
'sec-ch-ua': `"Chromium";v="${chromeMajor}", "Google Chrome";v="${chromeMajor}", "Not-A.Brand";v="99"`,
'sec-ch-ua-full-version-list': `"Chromium";v="${chromeFull}", "Google Chrome";v="${chromeFull}", "Not-A.Brand";v="99.0.0.0"`,
};
browserSession.webRequest.onBeforeSendHeaders((details, callback) => {
const requestHeaders = details.requestHeaders;
for (const name of Object.keys(requestHeaders)) {
const replacement = brandLists[name.toLowerCase()];
if (replacement !== undefined) {
requestHeaders[name] = replacement;
}
}
callback({ requestHeaders });
});
this.browserSession = browserSession;
return browserSession;
}
@ -196,17 +269,34 @@ export class BrowserViewManager extends EventEmitter {
return /^https?:\/\//i.test(url) || url === 'about:blank';
}
/**
* webPreferences shared by browser tabs and OAuth popups. Kept in one place
* so the security-sensitive popup surface can never drift from tabs.
*/
private browserWebPreferences(): Electron.WebPreferences {
return {
session: this.getSession(),
contextIsolation: true,
sandbox: true,
nodeIntegration: false,
// Chromium's built-in PDFium viewer, so PDFs render inline instead
// of showing a blank page.
plugins: true,
// Remove the WebAuthn API from the embedded browser only. Electron ships
// the API but not Chrome's authenticator UI (Touch ID sheet, QR/phone
// hybrid), so passkey challenges hang forever on "Verifying it's you...".
// With the API absent, sites feature-detect it and fall back to
// password/other verification. Scoped here (not app-wide) so the app's
// own renderer keeps WebAuthn.
disableBlinkFeatures: 'WebAuth',
};
}
private createView(): WebContentsView {
const view = new WebContentsView({
webPreferences: {
session: this.getSession(),
contextIsolation: true,
sandbox: true,
nodeIntegration: false,
},
webPreferences: this.browserWebPreferences(),
});
view.webContents.setUserAgent(SPOOF_UA);
return view;
}
@ -266,16 +356,145 @@ export class BrowserViewManager extends EventEmitter {
});
wc.on('page-title-updated', this.emitState.bind(this));
wc.setWindowOpenHandler(({ url }) => {
if (this.isEmbeddedTabUrl(url)) {
void this.newTab(url);
} else {
void shell.openExternal(url);
this.wireWindowPolicy(wc);
}
/**
* Window-open, popup, and HTTP-auth wiring shared by tabs and popups.
*/
private wireWindowPolicy(wc: WebContents): void {
wc.setWindowOpenHandler((details) => this.handleWindowOpen(details));
wc.on('did-create-window', (child) => this.wirePopupWindow(child));
this.wireHttpAuth(wc);
}
/**
* Shared window.open / target=_blank policy for tabs and popups.
*
* An open that hands a handle back to the opener must become a real child
* window so window.opener / postMessage survive this is how OAuth/SSO
* popups (Google, Microsoft, Plaid, ...) return their result; denying them
* also makes sites report "popup blocked". Those are: a sized popup
* (disposition 'new-window'), a *named* window.open(url, 'name') (non-empty
* frameName), or a scripted blank window the opener will populate
* (about:blank). A nameless target=_blank link (foreground-tab, empty
* frameName) has no opener contract and opens as a tab, matching browser
* behavior. Non-web schemes go to the system handler.
*
* Residual gap: a nameless, featureless window.open(url) is indistinguishable
* from a _blank link (both foreground-tab + empty frameName) and opens as a
* tab, losing its opener rare for OAuth, which virtually always names or
* sizes its popup.
*/
private handleWindowOpen(details: Electron.HandlerDetails): Electron.WindowOpenHandlerResponse {
const { url, disposition, frameName } = details;
if (this.isEmbeddedTabUrl(url)) {
const needsOpener =
disposition === 'new-window' || frameName !== '' || url === 'about:blank';
if (needsOpener) {
return {
action: 'allow',
overrideBrowserWindowOptions: {
autoHideMenuBar: true,
webPreferences: this.browserWebPreferences(),
},
};
}
void this.newTab(url);
return { action: 'deny' };
}
void shell.openExternal(url);
return { action: 'deny' };
}
private wirePopupWindow(child: BrowserWindow): void {
this.popupWindows.add(child);
child.once('closed', () => this.popupWindows.delete(child));
this.wireWindowPolicy(child.webContents);
}
/** True if `win` is an OAuth/SSO popup created by page window.open(). */
isPopupWindow(win: BrowserWindow): boolean {
return this.popupWindows.has(win);
}
/**
* HTTP basic/proxy auth. Chromium's default is to cancel the challenge, so
* 401-protected sites and authenticating proxies dead-end. When the browser
* pane is on screen to answer, forward the challenge to it as a credential
* prompt (cancelled after a timeout if unanswered). When the pane is closed
* e.g. agent-driven navigation don't preventDefault, so Chromium cancels
* immediately and the 401 page is readable rather than hanging.
*/
private wireHttpAuth(wc: WebContents): void {
wc.on('login', (event, _details, authInfo, callback) => {
if (!this.visible || !this.window) return;
event.preventDefault();
const requestId = randomUUID();
const timer = setTimeout(() => {
this.finishHttpAuth(requestId);
}, HTTP_AUTH_TIMEOUT_MS);
this.pendingHttpAuth.set(requestId, { callback, timer, webContents: wc });
// If the challenging contents dies before an answer, resolve now so the
// native callback and timer don't leak (backstop for paths other than
// destroyTab, which cancels explicitly before removeAllListeners()).
wc.once('destroyed', () => this.finishHttpAuth(requestId));
const request: HttpAuthRequest = {
requestId,
host: authInfo.host,
isProxy: authInfo.isProxy,
...(authInfo.realm ? { realm: authInfo.realm } : {}),
};
this.emit('http-auth-request', request);
});
}
/**
* Resolve a pending auth challenge. `username === undefined` cancels it; an
* empty-string username is a valid submission (token-style Basic auth).
* Always notifies the renderer so a dialog it may still be showing (e.g.
* after a timeout or tab close) is pruned.
*/
private finishHttpAuth(requestId: string, username?: string, password?: string): boolean {
const pending = this.pendingHttpAuth.get(requestId);
if (!pending) return false;
this.pendingHttpAuth.delete(requestId);
clearTimeout(pending.timer);
try {
if (username == null) {
pending.callback();
} else {
pending.callback(username, password ?? '');
}
} catch {
// The challenged webContents may already be destroyed.
}
this.emit('http-auth-resolved', requestId);
return true;
}
private cancelHttpAuthForWebContents(wc: WebContents): void {
const ids: string[] = [];
for (const [requestId, pending] of this.pendingHttpAuth) {
if (pending.webContents === wc) ids.push(requestId);
}
for (const requestId of ids) {
this.finishHttpAuth(requestId);
}
}
respondToHttpAuth(input: {
requestId: string;
username?: string;
password?: string;
}): { ok: boolean } {
return { ok: this.finishHttpAuth(input.requestId, input.username, input.password) };
}
private snapshotTabState(tab: BrowserTab): BrowserTabState {
const wc = tab.view.webContents;
return {
@ -364,6 +583,10 @@ export class BrowserViewManager extends EventEmitter {
private destroyTab(tab: BrowserTab): void {
this.invalidateSnapshot(tab.id);
// Cancel any auth challenge this tab raised before we drop its listeners,
// so the native callback + timer don't leak and the renderer prunes its
// dialog (removeAllListeners() below would kill the 'destroyed' backstop).
this.cancelHttpAuthForWebContents(tab.view.webContents);
tab.view.webContents.removeAllListeners();
if (!tab.view.webContents.isDestroyed()) {
tab.view.webContents.close();

View file

@ -315,3 +315,50 @@ export async function listToolkits() {
totalItems: filtered.length,
};
}
/**
* Execute a Composio tool by slug on behalf of a Mini App. The toolkit must be
* connected (ACTIVE). Mirrors the agent's composio-execute-tool builtin.
*/
export async function executeTool(
toolkitSlug: string,
toolSlug: string,
args?: Record<string, unknown>,
): Promise<{ successful: boolean; data?: unknown; error?: string }> {
const account = composioAccountsRepo.getAccount(toolkitSlug);
if (!account || account.status !== 'ACTIVE') {
return { successful: false, error: `Toolkit "${toolkitSlug}" is not connected.` };
}
try {
const result = await composioClient.executeAction(toolSlug, {
connected_account_id: account.id,
user_id: 'rowboat-user',
version: 'latest',
arguments: args ?? {},
});
return { successful: result.successful, data: result.data, error: result.error ?? undefined };
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.error(`[Composio] Mini App tool execution failed for ${toolSlug}:`, message);
return { successful: false, error: `Failed to execute ${toolSlug}: ${message}` };
}
}
/**
* Search Composio tools within a toolkit so a Mini App can discover the right
* tool slug + input schema at runtime (how generated apps will wire actions).
*/
export async function searchToolsInToolkit(
toolkitSlug: string,
query: string,
): Promise<{ tools: Array<{ slug: string; name: string; description?: string }>; error?: string }> {
try {
const { items } = await composioClient.searchTools(query, [toolkitSlug]);
return {
tools: items.map((t) => ({ slug: t.slug, name: t.name, description: t.description })),
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return { tools: [], error: message };
}
}

View file

@ -1,7 +1,8 @@
import { ipcMain, BrowserWindow, shell, dialog, systemPreferences, desktopCapturer, app } from 'electron';
import { ipcMain, BrowserWindow, shell, dialog, systemPreferences, desktopCapturer, app, screen } from 'electron';
import { ipc } from '@x/shared';
import path from 'node:path';
import os from 'node:os';
import { fileURLToPath } from 'node:url';
import {
connectProvider,
disconnectProvider,
@ -37,6 +38,7 @@ import type { IOAuthRepo } from '@x/core/dist/auth/repo.js';
import { IGranolaConfigRepo } from '@x/core/dist/knowledge/granola/repo.js';
import { ICodeModeConfigRepo } from '@x/core/dist/code-mode/repo.js';
import { CodePermissionRegistry } from '@x/core/dist/code-mode/acp/permission-registry.js';
import type { CodeRunFeed } from '@x/core/dist/code-mode/feed.js';
import { checkCodeModeAgentStatus } from '@x/core/dist/code-mode/status.js';
import { ensureEngine } from '@x/core/dist/code-mode/acp/engine-provisioner.js';
import type { ICodeProjectsRepo } from '@x/core/dist/code-mode/projects/repo.js';
@ -51,6 +53,8 @@ import type { CodeSession } from '@x/shared/dist/code-sessions.js';
import { invalidateCopilotInstructionsCache } from '@x/core/dist/application/assistant/instructions.js';
import { triggerSync as triggerGranolaSync } from '@x/core/dist/knowledge/granola/sync.js';
import { ISlackConfigRepo } from '@x/core/dist/slack/repo.js';
import { IChannelsConfigRepo } from '@x/core/dist/channels/repo.js';
import { applyChannelsConfig, getChannelsStatus, logoutWhatsApp, subscribeChannelsStatus } from '@x/core/dist/channels/service.js';
import { runAgentSlack, getAgentSlackCliStatus, AgentSlackRunError } from '@x/core/dist/slack/agent-slack-exec.js';
import { knowledgeSourcesRepo } from '@x/core/dist/knowledge/sources/repo.js';
import { rankSlackHomeMessages } from '@x/core/dist/knowledge/sources/rank_slack_home.js';
@ -58,6 +62,10 @@ import { syncSlackKnowledgeSources, triggerSync as triggerSlackKnowledgeSync, ge
import { isOnboardingComplete, markOnboardingComplete } from '@x/core/dist/config/note_creation_config.js';
import { loadNotificationSettings, saveNotificationSettings } from '@x/core/dist/config/notification_config.js';
import * as composioHandler from './composio-handler.js';
import * as appsIndexer from '@x/core/dist/apps/indexer.js';
import * as appsServer from '@x/core/dist/apps/server.js';
import * as appsAgents from '@x/core/dist/apps/agents.js';
import { capture } from '@x/core/dist/analytics/posthog.js';
import { consumePendingDeepLink } from './deeplink.js';
import { qualifyAndDisconnectComposioGoogle } from '@x/core/dist/migrations/composio-google-migration.js';
import { IAgentScheduleRepo } from '@x/core/dist/agent-schedule/repo.js';
@ -74,7 +82,7 @@ import { summarizeMeeting } from '@x/core/dist/knowledge/summarize_meeting.js';
import { getAccessToken } from '@x/core/dist/auth/tokens.js';
import { getRowboatConfig } from '@x/core/dist/config/rowboat.js';
import { runLiveNoteAgent } from '@x/core/dist/knowledge/live-note/runner.js';
import { listImportantThreads, listEverythingElseThreads, saveMessageBodyHeight, triggerSync as triggerGmailSync, sendThreadReply, archiveThread, trashThread, markThreadRead, getAccountEmail, getAccountName, getConnectionStatus as getGmailConnectionStatus } from '@x/core/dist/knowledge/sync_gmail.js';
import { listImportantThreads, listEverythingElseThreads, saveMessageBodyHeight, triggerSync as triggerGmailSync, sendThreadReply, saveThreadDraft, deleteThreadDraft, listDraftThreads, searchThreads, archiveThread, trashThread, markThreadRead, downloadAttachment, getAccountEmail, getAccountName, getConnectionStatus as getGmailConnectionStatus, setThreadImportance } from '@x/core/dist/knowledge/sync_gmail.js';
import { searchContacts as searchGmailContacts, warmContactIndex } from '@x/core/dist/knowledge/gmail_contacts.js';
import { searchSentContacts, warmSentContacts } from '@x/core/dist/knowledge/gmail_sent_contacts.js';
import { getGoogleDocsConnectionStatus, importGoogleDoc, syncGoogleDocDown, syncGoogleDocUp, getGoogleDocLink } from '@x/core/dist/knowledge/google_docs.js';
@ -381,9 +389,37 @@ type InvokeHandlers = {
[K in InvokeChannels]: InvokeHandler<K>;
};
// In-flight streaming TTS requests, keyed by renderer-chosen requestId.
const activeTtsStreams = new Map<string, AbortController>();
// Video-mode popout window (shown for the whole duration of a screen share,
// floating over every app including Rowboat itself) and the last call state
// pushed by the main window — replayed to the popout when it finishes loading.
let videoPopoutWin: BrowserWindow | null = null;
let lastVideoPopoutState: {
ttsState: 'idle' | 'synthesizing' | 'speaking';
status: 'listening' | 'thinking' | 'speaking' | null;
cameraOn: boolean;
micMuted: boolean;
screenSharing: boolean;
interimText: string | null;
} | null = null;
// Match only real app windows — getAllWindows() can also contain the popout
// itself and hidden utility windows (e.g. PDF-export renderers), which must
// not be shown, focused, or sent app events.
function findMainAppWindow(): BrowserWindow | undefined {
return BrowserWindow.getAllWindows().find((w) => {
if (w === videoPopoutWin || w.isDestroyed()) return false;
const url = w.webContents.getURL();
const isAppWindow = url.startsWith('app://') || url.startsWith('http://localhost');
return isAppWindow && !url.includes('#video-popout');
});
}
/**
* Register all IPC handlers with type safety and runtime validation
*
*
* This function ensures:
* 1. All invoke channels have handlers (exhaustiveness checking)
* 2. Handler signatures match channel definitions
@ -592,6 +628,9 @@ function emitServiceEvent(event: z.infer<typeof ServiceEvent>): void {
}
export function emitOAuthEvent(event: { provider: string; success: boolean; error?: string; userId?: string }): void {
// Native connection status (e.g. Google) is baked into the Copilot system
// prompt, so any OAuth state change must rebuild it.
invalidateCopilotInstructionsCache();
const windows = BrowserWindow.getAllWindows();
for (const win of windows) {
if (!win.isDestroyed() && win.webContents) {
@ -646,6 +685,20 @@ function emitSessionEvent(event: SessionBusEvent): void {
}
}
// Mobile channels: status changes (QR pairing, connect/disconnect) → renderer.
let channelsWatcher: (() => void) | null = null;
export function startChannelsWatcher(): void {
if (channelsWatcher) return;
channelsWatcher = subscribeChannelsStatus((status) => {
const windows = BrowserWindow.getAllWindows();
for (const win of windows) {
if (!win.isDestroyed() && win.webContents) {
win.webContents.send('channels:status', status);
}
}
});
}
let sessionsWatcher: (() => void) | null = null;
export function startSessionsWatcher(): void {
if (sessionsWatcher) {
@ -655,6 +708,38 @@ export function startSessionsWatcher(): void {
sessionsWatcher = sessionBus.subscribe((event) => emitSessionEvent(event));
}
// Ephemeral code-run stream: CodeRunFeed → all renderer windows. A direct
// tool→renderer side-channel that bypasses the turn runtime; the durable
// record is the settle-time code-run-events-batch tool progress.
let codeRunFeedWatcher: (() => void) | null = null;
export function startCodeRunFeedWatcher(): void {
if (codeRunFeedWatcher) {
return;
}
const feed = container.resolve<CodeRunFeed>('codeRunFeed');
codeRunFeedWatcher = feed.subscribe((event) => {
const windows = BrowserWindow.getAllWindows();
for (const win of windows) {
if (!win.isDestroyed() && win.webContents) {
win.webContents.send('codeRun:events', event);
}
}
});
}
// The renderer window is created before the session-index startup scan
// finishes, so an early sessions:list could observe a partially built index
// (the scan runs oldest-first — exactly the newest chats would be missing).
// sessions:list awaits this deferred; main.ts resolves it when the scan
// settles (success or failure, so the list never hangs).
let resolveSessionsIndexReady: () => void;
const sessionsIndexReady = new Promise<void>((resolve) => {
resolveSessionsIndexReady = resolve;
});
export function markSessionsIndexReady(): void {
resolveSessionsIndexReady();
}
let servicesWatcher: (() => void) | null = null;
export async function startServicesWatcher(): Promise<void> {
if (servicesWatcher) {
@ -782,6 +867,18 @@ export function setupIpcHandlers() {
'gmail:sendReply': async (_event, args) => {
return sendThreadReply(args);
},
'gmail:saveDraft': async (_event, args) => {
return saveThreadDraft(args);
},
'gmail:deleteDraft': async (_event, args) => {
return deleteThreadDraft(args.draftId);
},
'gmail:getDrafts': async () => {
return listDraftThreads();
},
'gmail:search': async (_event, args) => {
return searchThreads(args.query, { limit: args.limit });
},
'gmail:getConnectionStatus': async () => {
return getGmailConnectionStatus();
},
@ -791,6 +888,10 @@ export function setupIpcHandlers() {
'gmail:getAccountName': async () => {
return { name: await getAccountName() };
},
'gmail:setImportance': async (_event, args) => {
const result = setThreadImportance(args.threadId, args.importance);
return { ok: result.success, previous: result.previous, error: result.error };
},
'gmail:archiveThread': async (_event, args) => {
return archiveThread(args.threadId);
},
@ -798,7 +899,10 @@ export function setupIpcHandlers() {
return trashThread(args.threadId);
},
'gmail:markThreadRead': async (_event, args) => {
return markThreadRead(args.threadId);
return markThreadRead(args.threadId, args.read);
},
'gmail:downloadAttachment': async (_event, args) => {
return downloadAttachment(args);
},
'gmail:saveMessageHeight': async (_event, args) => {
saveMessageBodyHeight(args.threadId, args.messageId, args.height);
@ -872,6 +976,7 @@ export function setupIpcHandlers() {
return { sessionId };
},
'sessions:list': async () => {
await sessionsIndexReady;
return { sessions: container.resolve<ISessions>('sessions').listSessions() };
},
'sessions:get': async (_event, args) => {
@ -999,6 +1104,11 @@ export function setupIpcHandlers() {
await repo.setConfig(args);
return { success: true };
},
'models:updateConfig': async (_event, args) => {
const repo = container.resolve<IModelConfigRepo>('modelConfigRepo');
await repo.updateConfig(args);
return { success: true };
},
'oauth:connect': async (_event, args) => {
const credentials = args.clientId && args.clientSecret
? { clientId: args.clientId.trim(), clientSecret: args.clientSecret.trim() }
@ -1193,6 +1303,22 @@ export function setupIpcHandlers() {
return { success: true };
},
// ── Mobile channels (WhatsApp / Telegram bridge) ─────────────
'channels:getConfig': async () => {
return container.resolve<IChannelsConfigRepo>('channelsConfigRepo').getConfig();
},
'channels:setConfig': async (_event, args) => {
await container.resolve<IChannelsConfigRepo>('channelsConfigRepo').setConfig(args);
await applyChannelsConfig(args);
return { success: true };
},
'channels:getStatus': async () => {
return getChannelsStatus();
},
'channels:whatsappLogout': async () => {
await logoutWhatsApp();
return { success: true };
},
'slack:getConfig': async () => {
const repo = container.resolve<ISlackConfigRepo>('slackConfigRepo');
const config = await repo.getConfig();
@ -1422,9 +1548,56 @@ export function setupIpcHandlers() {
'composio:list-toolkits': async () => {
return composioHandler.listToolkits();
},
'composio:execute-tool': async (_event, args) => {
return composioHandler.executeTool(args.toolkitSlug, args.toolSlug, args.arguments);
},
'composio:search-tools': async (_event, args) => {
return composioHandler.searchToolsInToolkit(args.toolkitSlug, args.query);
},
'migration:check-composio-google': async () => {
return qualifyAndDisconnectComposioGoogle();
},
// Rowboat Apps handlers (spec §13)
'apps:list': async () => {
const status = appsServer.getServerStatus();
const apps = await appsIndexer.listApps();
// Keep bundled agents materialized (idempotent; disabled by default).
for (const app of apps) {
if (app.agentSlugs.length) await appsAgents.syncAppAgents(app);
}
return {
serverRunning: status.running,
...(status.error ? { serverError: status.error } : {}),
apps,
};
},
'apps:get': async (_event, args) => {
const app = await appsIndexer.getApp(args.folder);
if (!app) throw new Error(`no such app: ${args.folder}`);
const readme = await appsIndexer.readAppReadme(args.folder);
return {
app,
...(readme ? { readme } : {}),
rollbackAvailable: await appsIndexer.rollbackAvailable(args.folder),
};
},
'apps:create': async (_event, args) => {
const app = await appsIndexer.createApp(args);
capture('app_created', { folder: app.folder });
return { app };
},
'apps:delete': async (_event, args) => {
await appsIndexer.deleteApp(args.folder);
// Remove app-owned bg-tasks too — orphaned app--<folder>-- tasks firing
// against a deleted app was a painful prototype failure mode.
await appsAgents.deleteAppAgents(args.folder);
capture('app_deleted', { folder: args.folder });
return { ok: true as const };
},
'apps:setTheme': async (_event, args) => {
appsServer.setAppsTheme(args.theme);
return { ok: true as const };
},
// Agent schedule handlers
'agent-schedule:getConfig': async () => {
const repo = container.resolve<IAgentScheduleRepo>('agentScheduleRepo');
@ -1680,6 +1853,51 @@ export function setupIpcHandlers() {
'voice:synthesize': async (_event, args) => {
return voice.synthesizeSpeech(args.text);
},
'voice:synthesizeStreamStart': async (event, args) => {
const { requestId, text } = args;
const sender = event.sender;
const controller = new AbortController();
activeTtsStreams.set(requestId, controller);
// Fire-and-forget: chunks are pushed to the renderer as they arrive so
// playback can begin immediately; the invoke returns once started.
void voice
.synthesizeSpeechStream(
text,
(chunk) => {
if (!sender.isDestroyed()) {
sender.send('voice:tts-chunk', {
requestId,
chunkBase64: chunk.toString('base64'),
done: false,
});
}
},
controller.signal,
)
.then(() => {
if (!sender.isDestroyed()) {
sender.send('voice:tts-chunk', { requestId, done: true });
}
})
.catch((err: unknown) => {
if (!sender.isDestroyed() && !controller.signal.aborted) {
sender.send('voice:tts-chunk', {
requestId,
done: true,
error: err instanceof Error ? err.message : String(err),
});
}
})
.finally(() => {
activeTtsStreams.delete(requestId);
});
return { ok: true };
},
'voice:synthesizeStreamCancel': async (_event, args) => {
activeTtsStreams.get(args.requestId)?.abort();
activeTtsStreams.delete(args.requestId);
return {};
},
'voice:ensureMicAccess': async () => {
if (process.platform !== 'darwin') return { granted: true };
const status = systemPreferences.getMediaAccessStatus('microphone');
@ -1698,6 +1916,113 @@ export function setupIpcHandlers() {
return { granted: false };
}
},
'voice:ensureCameraAccess': async () => {
if (process.platform !== 'darwin') return { granted: true };
const status = systemPreferences.getMediaAccessStatus('camera');
console.log('[video] Camera permission status:', status);
if (status === 'granted') return { granted: true };
// Same flow as the microphone: settle the native TCC prompt before the
// renderer's getUserMedia so the first video click doesn't silently fail.
try {
const granted = await systemPreferences.askForMediaAccess('camera');
console.log('[video] Camera permission after prompt:', granted);
return { granted };
} catch {
return { granted: false };
}
},
'video:setPopout': async (_event, args) => {
if (!args.show) {
if (videoPopoutWin && !videoPopoutWin.isDestroyed()) videoPopoutWin.destroy();
videoPopoutWin = null;
return {};
}
if (videoPopoutWin && !videoPopoutWin.isDestroyed()) return {};
const workArea = screen.getPrimaryDisplay().workArea;
const width = 340;
const height = 184;
const ipcDir = path.dirname(fileURLToPath(import.meta.url));
const preloadPath = app.isPackaged
? path.join(ipcDir, '../preload/dist/preload.js')
: path.join(ipcDir, '../../../preload/dist/preload.js');
const win = new BrowserWindow({
width,
height,
x: workArea.x + workArea.width - width - 24,
y: workArea.y + 24,
frame: false,
resizable: false,
alwaysOnTop: true,
skipTaskbar: true,
show: false,
hasShadow: true,
backgroundColor: '#171717',
webPreferences: {
nodeIntegration: false,
contextIsolation: true,
sandbox: true,
preload: preloadPath,
},
});
// Float above other apps on every workspace. Deliberately NOT
// `visibleOnFullScreen: true`: on macOS that flag hides the app's Dock
// icon for as long as such a window exists (the app becomes an
// "agent" app), which reads as Rowboat having vanished. The trade-off
// is the popout won't hover over other apps' fullscreen Spaces.
win.setAlwaysOnTop(true, 'floating');
win.setVisibleOnAllWorkspaces(true);
win.webContents.once('did-finish-load', () => {
if (lastVideoPopoutState) {
win.webContents.send('video:popout-state', lastVideoPopoutState);
}
// showInactive: appearing must not steal focus from the app the user
// switched to — that would immediately re-hide the popout.
if (!win.isDestroyed()) win.showInactive();
});
win.on('closed', () => {
if (videoPopoutWin === win) videoPopoutWin = null;
});
videoPopoutWin = win;
if (app.isPackaged) {
win.loadURL('app://-/index.html#video-popout');
} else {
win.loadURL('http://localhost:5173/#video-popout');
}
return {};
},
'video:popoutState': async (_event, args) => {
lastVideoPopoutState = args;
if (videoPopoutWin && !videoPopoutWin.isDestroyed()) {
videoPopoutWin.webContents.send('video:popout-state', args);
}
return {};
},
'app:focusMainWindow': async () => {
const main = findMainAppWindow();
if (main) {
if (main.isMinimized()) main.restore();
main.show();
main.focus();
}
return {};
},
'video:getPopoutState': async () => {
return { state: lastVideoPopoutState };
},
'video:popoutAction': async (_event, args) => {
// Relay a popout control-bar action to the app window, which owns the
// call (mic, camera, screen capture) and executes it there. 'expand'
// additionally brings the app window back to the foreground.
const main = findMainAppWindow();
if (args.action === 'expand' && main) {
if (main.isMinimized()) main.restore();
main.show();
main.focus();
}
main?.webContents.send('video:popout-action', args);
return {};
},
// Live-note handlers
'live-note:run': async (_event, args) => {
const result = await runLiveNoteAgent(args.filePath, 'manual', args.context);

View file

@ -2,7 +2,9 @@ import { app, BrowserWindow, desktopCapturer, protocol, net, shell, session, typ
import path from "node:path";
import {
setupIpcHandlers,
startRunsWatcher, startSessionsWatcher,
startRunsWatcher, startSessionsWatcher, markSessionsIndexReady,
startCodeRunFeedWatcher,
startChannelsWatcher,
startCodeSessionStatusWatcher,
startServicesWatcher,
startLiveNoteAgentWatcher,
@ -25,6 +27,7 @@ import { init as initEmailLabeling } from "@x/core/dist/knowledge/label_emails.j
import { init as initNoteTagging } from "@x/core/dist/knowledge/tag_notes.js";
import { init as initInlineTasks } from "@x/core/dist/knowledge/inline_tasks.js";
import { init as initAgentRunner } from "@x/core/dist/agent-schedule/runner.js";
import { init as initChannels } from "@x/core/dist/channels/service.js";
import { init as initAgentNotes } from "@x/core/dist/knowledge/agent_notes.js";
import { init as initCalendarNotifications } from "@x/core/dist/knowledge/notify_calendar_meetings.js";
import { init as initMeetingPrep } from "@x/core/dist/knowledge/meeting_prep_scheduler.js";
@ -33,10 +36,12 @@ import { init as initEventProcessor, registerConsumer } from "@x/core/dist/event
import { liveNoteEventConsumer } from "@x/core/dist/knowledge/live-note/event-consumer.js";
import { init as initBackgroundTaskScheduler } from "@x/core/dist/background-tasks/scheduler.js";
import { backgroundTaskEventConsumer } from "@x/core/dist/background-tasks/event-consumer.js";
import { init as initLocalSites, shutdown as shutdownLocalSites } from "@x/core/dist/local-sites/server.js";
import { startSkillsWatcher, stopSkillsWatcher } from "@x/core/dist/application/assistant/skills/watcher.js";
import { init as initAppsServer, shutdown as shutdownAppsServer } from "@x/core/dist/apps/server.js";
import { registerAppsHostApi } from "@x/core/dist/apps/host-api.js";
import { shutdown as shutdownAnalytics } from "@x/core/dist/analytics/posthog.js";
import { identifyIfSignedIn } from "@x/core/dist/analytics/identify.js";
import { migrateRuns } from "@x/core/dist/migrations/runs/migrate.js";
import { initConfigs } from "@x/core/dist/config/initConfigs.js";
import { getAgentSlackCliStatus } from "@x/core/dist/slack/agent-slack-exec.js";
@ -390,10 +395,44 @@ app.whenReady().then(async () => {
// start runs watcher
startRunsWatcher();
// New runtime: build the in-memory session index (startup scan) before the
// renderer can list sessions, then forward the session bus to windows.
await container.resolve<ISessions>('sessions').initialize();
// One-time: port legacy runs/*.jsonl into the new turn/session runtime.
// Must run BEFORE the session index is built so migrated sessions are picked
// up by the startup scan. Fully defensive — never blocks boot.
try {
const migration = migrateRuns();
if (migration.scanned > 0) {
console.log(
`[runs-migration] migrated ${migration.migratedTurns} turn(s) across ` +
`${migration.migratedSessions} session(s) from ${migration.scanned} run(s) ` +
`(${migration.skipped} skipped, ${migration.failed.length} failed)`,
);
for (const failure of migration.failed) {
console.warn(`[runs-migration] left in place (failed): ${failure.file}${failure.error}`);
}
}
} catch (error) {
console.error('[runs-migration] pass failed:', error);
}
// New runtime: build the in-memory session index (startup scan), then
// forward the session bus to windows. The renderer window is already up and
// may have called sessions:list — that handler blocks on
// markSessionsIndexReady, which must fire even if the scan throws so the
// list never hangs.
try {
await container.resolve<ISessions>('sessions').initialize();
} finally {
markSessionsIndexReady();
}
startSessionsWatcher();
startCodeRunFeedWatcher();
// Mobile channels (WhatsApp/Telegram bridge): needs the session index, so
// start after initialize(). Failures must never block boot.
startChannelsWatcher();
initChannels().catch((error) => {
console.error('[Channels] Failed to start mobile channels:', error);
});
// start code-session status tracker (derives working/needs-you/idle + notifications)
startCodeSessionStatusWatcher();
@ -468,9 +507,11 @@ app.whenReady().then(async () => {
// start chrome extension sync server
initChromeSync();
// start local sites server for iframe dashboards and other mini apps
initLocalSites().catch((error) => {
console.error('[LocalSites] Failed to start:', error);
// start the Rowboat Apps server (per-app origins on 127.0.0.1:3210) with the
// full Host API (tools/fetch/llm/copilot behind the capability gate)
registerAppsHostApi();
initAppsServer().catch((error) => {
console.error('[Apps] Failed to start:', error);
});
app.on("activate", () => {
@ -500,8 +541,8 @@ stopSkillsWatcher();
}
// Kill embedded terminal shells.
disposeAllTerminals();
shutdownLocalSites().catch((error) => {
console.error('[LocalSites] Failed to shut down cleanly:', error);
shutdownAppsServer().catch((error) => {
console.error('[Apps] Failed to shut down cleanly:', error);
});
shutdownAnalytics().catch((error) => {
console.error('[Analytics] Failed to flush on quit:', error);

View file

@ -0,0 +1,62 @@
/**
* Regenerates the bundled product-tour narration clips.
*
* Parses the TOUR_STEPS texts straight out of product-tour.tsx (so the code
* stays the single source of truth), synthesizes each one via @x/core's
* synthesizeSpeech (Rowboat proxy when signed in, direct ElevenLabs otherwise,
* using the voice id configured there / in ~/.rowboat/config/elevenlabs.json),
* and writes MP3s to src/assets/tour/<step-id>.mp3.
*
* Run whenever a step's narration text or the tour voice changes:
* cd apps/x && npm run deps # script imports core's built output
* node apps/renderer/scripts/generate-tour-audio.mjs
*
* Pass step ids to regenerate only those clips (e.g. to re-roll one whose
* synthesis came out glitchy):
* node apps/renderer/scripts/generate-tour-audio.mjs welcome done
*/
import { mkdir, readFile, writeFile } from 'node:fs/promises'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
const here = path.dirname(fileURLToPath(import.meta.url))
const tourSource = path.join(here, '../src/components/product-tour.tsx')
const outDir = path.join(here, '../src/assets/tour')
const corePath = path.join(here, '../../../packages/core/dist/voice/voice.js')
const { synthesizeSpeech } = await import(corePath)
const src = await readFile(tourSource, 'utf8')
const start = src.indexOf('const TOUR_STEPS')
const end = src.indexOf('\n]', start)
if (start === -1 || end === -1) throw new Error('Could not locate TOUR_STEPS in product-tour.tsx')
const block = src.slice(start, end)
const steps = []
// voiceText, when present, is the spoken variant of the bubble text.
const re = /id: '([^']+)'[\s\S]*?text:\s*("[^"]*"|'[^']*')(?:,\s*voiceText:\s*("[^"]*"|'[^']*'))?/g
for (let m; (m = re.exec(block)); ) {
// The captures are JS string literals from our own source; evaluate them
// to resolve the quoting.
steps.push({ id: m[1], text: new Function(`return ${m[3] ?? m[2]}`)() })
}
if (steps.length === 0) throw new Error('Parsed zero tour steps — regex out of sync with product-tour.tsx?')
console.log(`Parsed ${steps.length} tour steps`)
const only = process.argv.slice(2)
if (only.length > 0) {
const unknown = only.filter((id) => !steps.some((s) => s.id === id))
if (unknown.length > 0) throw new Error(`Unknown step ids: ${unknown.join(', ')}`)
steps.splice(0, steps.length, ...steps.filter((s) => only.includes(s.id)))
console.log(`Regenerating only: ${only.join(', ')}`)
}
await mkdir(outDir, { recursive: true })
for (const step of steps) {
process.stdout.write(`synthesizing ${step.id}... `)
const { audioBase64 } = await synthesizeSpeech(step.text)
const file = path.join(outDir, `${step.id}.mp3`)
await writeFile(file, Buffer.from(audioBase64, 'base64'))
console.log(`${(audioBase64.length * 0.75 / 1024).toFixed(0)} KB`)
}
console.log(`Done — ${steps.length} clips in ${path.relative(process.cwd(), outDir)}`)

View file

@ -169,6 +169,25 @@
color: var(--gm-placeholder);
}
.gmail-search-clear {
display: inline-flex;
align-items: center;
justify-content: center;
width: 20px;
height: 20px;
border: none;
border-radius: 4px;
background: transparent;
color: var(--gm-text-muted);
cursor: pointer;
transition: background 120ms ease, color 120ms ease;
}
.gmail-search-clear:hover {
background: var(--gm-icon-hover-bg);
color: var(--gm-text);
}
.gmail-icon-button {
display: inline-flex;
align-items: center;
@ -205,6 +224,46 @@
flex-direction: column;
}
/* Native list virtualization: offscreen rows skip layout and paint entirely.
Applied only to rows without a mounted ThreadDetail (those hold iframes and
composers, which must keep rendering while offscreen). */
.gmail-row-group-cv {
content-visibility: auto;
contain-intrinsic-size: auto 40px;
}
/* While the list is scrolling, rows ignore the pointer so hover restyles and
prefetch timers don't compete with frame rendering. */
.gmail-shell[data-scrolling] .gmail-row-shell {
pointer-events: none;
}
/* Archived/trashed rows slide out and collapse before removal. Removing the
class (failed action) snaps the row back. */
.gmail-row-group-leaving {
overflow: hidden;
pointer-events: none;
animation: gmail-row-leave 160ms ease-in forwards;
}
@keyframes gmail-row-leave {
0% {
opacity: 1;
transform: translateX(0);
max-height: 48px;
}
60% {
opacity: 0;
transform: translateX(32px);
max-height: 48px;
}
100% {
opacity: 0;
transform: translateX(32px);
max-height: 0;
}
}
.gmail-list-header {
position: sticky;
top: 0;
@ -271,6 +330,11 @@
opacity: 0;
pointer-events: none;
transition: opacity 120ms ease;
/* The overlay may extend over the subject text; a backdrop matching the
row's current background (see --gm-row-actions-bg below) plus a soft left
fade keeps it readable instead of colliding. */
padding-left: 18px;
background: linear-gradient(to right, transparent, var(--gm-row-actions-bg, var(--gm-bg-row-hover)) 18px);
}
.gmail-row-shell:hover .gmail-row-actions {
@ -278,6 +342,19 @@
pointer-events: auto;
}
/* Backdrop color tracks the row state underneath the overlay. */
.gmail-row-shell:hover {
--gm-row-actions-bg: var(--gm-bg-row-hover);
}
.gmail-row-shell:hover:has(.gmail-row-selected) {
--gm-row-actions-bg: var(--gm-bg-row-selected-hover);
}
.gmail-row-shell:hover:has(.gmail-row-selected.gmail-row-focused) {
--gm-row-actions-bg: var(--gm-bg-row-selected);
}
.gmail-row-shell:hover .gmail-row-date {
visibility: hidden;
}
@ -305,7 +382,7 @@
color: var(--destructive);
}
.gmail-row:hover {
.gmail-row-shell:hover .gmail-row {
background: var(--gm-bg-row-hover);
box-shadow: none;
}
@ -315,10 +392,24 @@
box-shadow: inset 2px 0 0 var(--gm-accent);
}
.gmail-row-selected:hover {
.gmail-row-shell:hover .gmail-row-selected {
background: var(--gm-bg-row-selected-hover);
}
/* The j/k keyboard cursor. Declared after the hover rules (same specificity)
so the focus ring survives hovering the focused row. */
.gmail-row-focused,
.gmail-row-shell:hover .gmail-row-focused {
background: var(--gm-bg-row-hover);
box-shadow: inset 0 0 0 1px var(--gm-accent);
}
.gmail-row-selected.gmail-row-focused,
.gmail-row-shell:hover .gmail-row-selected.gmail-row-focused {
background: var(--gm-bg-row-selected);
box-shadow: inset 2px 0 0 var(--gm-accent), inset 0 0 0 1px var(--gm-accent);
}
.gmail-row-unread {
color: var(--gm-text);
}
@ -394,12 +485,50 @@
border-top: 1px solid var(--gm-border);
border-bottom: 1px solid var(--gm-border);
box-shadow: inset 2px 0 0 var(--gm-accent);
/* Replays whenever the detail is shown hidden details are display: none,
so un-hiding restarts the animation. */
animation: gmail-detail-open 140ms ease-out;
}
.gmail-detail-hidden {
display: none;
}
@keyframes gmail-detail-open {
from {
opacity: 0;
transform: translateY(-6px);
}
to {
opacity: 1;
transform: none;
}
}
/* The inline reply/forward composer pops in from below the thread. */
.gmail-compose-inline {
animation: gmail-compose-open 140ms ease-out;
}
@keyframes gmail-compose-open {
from {
opacity: 0;
transform: translateY(8px);
}
to {
opacity: 1;
transform: none;
}
}
@media (prefers-reduced-motion: reduce) {
.gmail-detail-inline,
.gmail-compose-inline,
.gmail-row-group-leaving {
animation: none;
}
}
.gmail-detail-toolbar {
display: flex;
align-items: center;

File diff suppressed because it is too large Load diff

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -22,7 +22,7 @@ import {
import { type ComponentProps, type ReactNode, isValidElement, useState } from "react";
import { AnimatePresence, motion } from "motion/react";
import type { ToolCall, ToolGroup as ToolGroupType } from "@/lib/chat-conversation";
import { getToolActionsSummary, getToolDisplayName, getToolGroupSummary, toToolState } from "@/lib/chat-conversation";
import { getToolActionsSummary, getToolDisplayName, getToolErrorText, getToolGroupSummary, toToolState } from "@/lib/chat-conversation";
const formatToolValue = (value: unknown) => {
if (typeof value === "string") return value;
@ -329,7 +329,7 @@ export const ToolGroupComponent = ({ group, isToolOpen, onToolOpenChange }: Tool
<ToolTabbedContent
input={tool.input as ToolUIPart["input"]}
output={tool.result as ToolUIPart["output"]}
errorText={tool.status === 'error' ? 'Tool error' : undefined}
errorText={getToolErrorText(tool)}
/>
</ToolContent>
</Tool>

View file

@ -0,0 +1,186 @@
import { useEffect, useState } from 'react'
import { X, RotateCcw, Play } from 'lucide-react'
import type { rowboatApp } from '@x/shared'
// App detail panel (spec §14): manifest info, provenance/publish state,
// bundled agents with enable toggles, rollback when available. Update/publish
// actions land with M3.
type AgentRow = {
slug: string
name: string
active: boolean
lastRunAt?: string
lastRunError?: string
}
export function AppDetail({ folder, onClose }: { folder: string; onClose: () => void }) {
const [app, setApp] = useState<rowboatApp.AppSummary | null>(null)
const [readme, setReadme] = useState<string | undefined>(undefined)
const [rollback, setRollback] = useState(false)
const [agents, setAgents] = useState<AgentRow[]>([])
const [error, setError] = useState<string | null>(null)
const [reloadNonce, setReloadNonce] = useState(0)
useEffect(() => {
let cancelled = false
;(async () => {
try {
const r = await window.ipc.invoke('apps:get', { folder })
if (cancelled) return
setApp(r.app)
setReadme(r.readme)
setRollback(r.rollbackAvailable)
const rows: AgentRow[] = []
for (const slug of r.app.agentSlugs) {
try {
const t = await window.ipc.invoke('bg-task:get', { slug })
if (t.task) {
rows.push({
slug,
name: t.task.name,
active: t.task.active,
lastRunAt: t.task.lastRunAt,
lastRunError: t.task.lastRunError,
})
}
} catch { /* not materialized yet */ }
}
if (!cancelled) setAgents(rows)
} catch (e) {
if (!cancelled) setError(e instanceof Error ? e.message : String(e))
}
})()
return () => { cancelled = true }
}, [folder, reloadNonce])
const toggleAgent = async (slug: string, active: boolean) => {
setAgents((prev) => prev.map((a) => (a.slug === slug ? { ...a, active } : a)))
try {
await window.ipc.invoke('bg-task:patch', { slug, partial: { active } })
} catch {
setReloadNonce((n) => n + 1) // revert to truth on failure
}
}
const runAgent = async (slug: string) => {
try {
await window.ipc.invoke('bg-task:run', { slug })
} catch { /* surfaced via bg-task UI */ }
}
const manifest = app?.manifest
return (
<div className="flex h-full flex-col overflow-hidden">
<div className="flex items-center gap-2 border-b border-border px-4 py-2.5">
<span className="flex-1 truncate text-sm font-semibold">{manifest?.name ?? folder}</span>
<button type="button" onClick={onClose} className="rounded-md p-1.5 text-muted-foreground hover:bg-accent hover:text-foreground">
<X className="size-4" />
</button>
</div>
<div className="min-h-0 flex-1 overflow-y-auto px-4 py-4 text-sm">
{error && <div className="mb-3 rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-destructive">{error}</div>}
{!app ? (
<div className="text-muted-foreground">Loading</div>
) : (
<div className="space-y-5">
<section className="space-y-1.5">
<div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">App</div>
{app.status === 'invalid' && (
<div className="rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-xs text-destructive">
Invalid manifest: {app.manifestError}
</div>
)}
<InfoRow k="Version" v={manifest ? `v${manifest.version}` : '—'} />
<InfoRow k="Folder" v={app.folder} />
<InfoRow k="Origin" v={app.origin} mono />
{manifest?.description ? <p className="pt-1 text-muted-foreground">{manifest.description}</p> : null}
</section>
<section className="space-y-1.5">
<div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">Capabilities</div>
{manifest && manifest.capabilities.length > 0 ? (
<div className="flex flex-wrap gap-1.5">
{manifest.capabilities.map((c) => (
<span key={c} className="rounded-full bg-muted px-2.5 py-0.5 text-xs font-medium text-muted-foreground">{c}</span>
))}
</div>
) : (
<p className="text-muted-foreground">None declared this app cant use tools, LLM, or the copilot.</p>
)}
</section>
<section className="space-y-1.5">
<div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">Source</div>
{app.kind === 'installed' && app.install ? (
<>
<InfoRow k="Installed from" v={app.install.repo ?? app.install.sourceUrl ?? 'unknown'} mono />
<InfoRow k="Installed" v={new Date(app.install.installedAt).toLocaleString()} />
{app.install.updatedAt && <InfoRow k="Updated" v={new Date(app.install.updatedAt).toLocaleString()} />}
</>
) : (
<p className="text-muted-foreground">Local app created on this machine{app.publish ? `, published as ${app.publish.repo}` : ''}.</p>
)}
{rollback && (
<button type="button" className="mt-1 flex items-center gap-1.5 rounded-md border border-border px-2.5 py-1 text-xs font-medium hover:bg-accent">
<RotateCcw className="size-3.5" /> Roll back to previous version
</button>
)}
</section>
<section className="space-y-2">
<div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">Background agents</div>
{agents.length === 0 ? (
<p className="text-muted-foreground">No bundled agents.</p>
) : agents.map((a) => (
<div key={a.slug} className="flex items-center gap-2 rounded-lg border border-border px-3 py-2">
<div className="min-w-0 flex-1">
<div className="truncate text-sm font-medium">{a.name}</div>
<div className="truncate text-xs text-muted-foreground">
{a.lastRunError ? `Failed: ${a.lastRunError}` : a.lastRunAt ? `Last run ${new Date(a.lastRunAt).toLocaleString()}` : 'Never run'}
</div>
</div>
<button
type="button"
title="Run now"
onClick={() => void runAgent(a.slug)}
className="rounded-md p-1.5 text-muted-foreground hover:bg-accent hover:text-foreground"
>
<Play className="size-3.5" />
</button>
<button
type="button"
role="switch"
aria-checked={a.active}
onClick={() => void toggleAgent(a.slug, !a.active)}
className={`relative h-5 w-9 rounded-full transition ${a.active ? 'bg-primary' : 'bg-muted'}`}
>
<span className={`absolute top-0.5 size-4 rounded-full bg-background shadow transition-all ${a.active ? 'left-[18px]' : 'left-0.5'}`} />
</button>
</div>
))}
</section>
{readme && (
<section className="space-y-1.5">
<div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">README</div>
<pre className="whitespace-pre-wrap rounded-lg border border-border bg-muted/40 p-3 text-xs leading-relaxed">{readme}</pre>
</section>
)}
</div>
)}
</div>
</div>
)
}
function InfoRow({ k, v, mono }: { k: string; v: string; mono?: boolean }) {
return (
<div className="flex items-baseline gap-2">
<span className="w-28 shrink-0 text-xs text-muted-foreground">{k}</span>
<span className={`min-w-0 truncate ${mono ? 'font-mono text-xs' : ''}`}>{v}</span>
</div>
)
}

View file

@ -0,0 +1,105 @@
import { useEffect, useState } from 'react'
import { ArrowLeft, ExternalLink, Info, RotateCw } from 'lucide-react'
import type { rowboatApp } from '@x/shared'
import { appOpened } from '@/lib/analytics'
import { AppDetail } from '@/components/apps/app-detail'
// Full-height iframe on the app's own origin (spec §6.6). No sandbox attr —
// per-app browser origins are the isolation boundary. Toolbar: back, reload,
// open-in-browser, detail panel.
export function AppFrame({ app, onBack }: { app: rowboatApp.AppSummary; onBack: () => void }) {
const [reloadNonce, setReloadNonce] = useState(0)
const [showDetail, setShowDetail] = useState(false)
// Load watchdog: if the iframe hasn't fired `load` within the deadline,
// surface a visible retry state instead of a silent blank pane.
const [loadState, setLoadState] = useState<'loading' | 'ok' | 'stuck'>('loading')
const title = app.manifest?.name ?? app.folder
useEffect(() => {
appOpened(app.folder)
}, [app.folder])
// Reset the watchdog when the target changes (adjust-during-render pattern).
const [watchKey, setWatchKey] = useState(`${app.folder}:${reloadNonce}`)
if (watchKey !== `${app.folder}:${reloadNonce}`) {
setWatchKey(`${app.folder}:${reloadNonce}`)
setLoadState('loading')
}
useEffect(() => {
const timer = window.setTimeout(() => {
setLoadState((s) => (s === 'loading' ? 'stuck' : s))
}, 6000)
return () => window.clearTimeout(timer)
}, [watchKey])
return (
<div className="flex h-full flex-col">
<div className="flex items-center gap-2 border-b border-border px-3 py-2">
<button
type="button"
onClick={onBack}
className="flex items-center gap-1.5 rounded-md px-2 py-1 text-sm text-muted-foreground hover:bg-accent hover:text-foreground"
>
<ArrowLeft className="size-4" />
Apps
</button>
<span className="flex-1 truncate text-sm font-medium">{title}</span>
<button
type="button"
title="Reload"
onClick={() => setReloadNonce((n) => n + 1)}
className="rounded-md p-1.5 text-muted-foreground hover:bg-accent hover:text-foreground"
>
<RotateCw className="size-4" />
</button>
<button
type="button"
title="Open in browser"
onClick={() => window.open(app.origin, '_blank')}
className="rounded-md p-1.5 text-muted-foreground hover:bg-accent hover:text-foreground"
>
<ExternalLink className="size-4" />
</button>
<button
type="button"
title="App details"
onClick={() => setShowDetail((v) => !v)}
className={`rounded-md p-1.5 hover:bg-accent hover:text-foreground ${showDetail ? 'text-foreground' : 'text-muted-foreground'}`}
>
<Info className="size-4" />
</button>
</div>
<div className="flex min-h-0 flex-1">
<div className="relative min-w-0 flex-1">
<iframe
key={reloadNonce}
title={title}
src={`${app.origin}/`}
onLoad={() => setLoadState('ok')}
className="h-full w-full border-0 bg-background"
/>
{loadState === 'stuck' && (
<div className="absolute inset-0 flex flex-col items-center justify-center gap-3 bg-background/95 text-sm">
<div className="text-muted-foreground">This app is taking too long to load.</div>
<button
type="button"
onClick={() => setReloadNonce((n) => n + 1)}
className="rounded-md border border-border px-3 py-1.5 font-medium hover:bg-accent"
>
Retry
</button>
<div className="font-mono text-xs text-muted-foreground">{app.origin}</div>
</div>
)}
</div>
{showDetail && (
<div className="w-80 shrink-0 border-l border-border">
<AppDetail folder={app.folder} onClose={() => setShowDetail(false)} />
</div>
)}
</div>
</div>
)
}

View file

@ -0,0 +1,238 @@
import { useEffect, useState } from 'react'
import { Plus, RefreshCw } from 'lucide-react'
import type { rowboatApp } from '@x/shared'
import { AppFrame } from '@/components/apps/app-frame'
// Apps home (spec §14): "My apps" grid + Catalog placeholder (M3). Cards are
// AppSummary-driven; click opens the app full-height on its own origin.
type Theme = { accent: string; glow: string }
const THEMES: Theme[] = [
{ accent: '#FF4D8D', glow: 'rgba(255,77,141,0.45)' }, // Pink
{ accent: '#EF4444', glow: 'rgba(239,68,68,0.45)' }, // Red
{ accent: '#22C55E', glow: 'rgba(34,197,94,0.40)' }, // Emerald
{ accent: '#F59E0B', glow: 'rgba(245,158,11,0.42)' }, // Amber
{ accent: '#14B8A6', glow: 'rgba(20,184,166,0.40)' }, // Teal
{ accent: '#EC4899', glow: 'rgba(236,72,153,0.42)' }, // Rose
]
const PATTERNS = ['dots', 'grid', 'diagonal', 'radial', 'waves', 'mesh', 'cross', 'rings', 'zigzag', 'plus', 'checker', 'beams']
function hash(s: string): number {
let h = 0
for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) | 0
return Math.abs(h)
}
const themeForIndex = (i: number): Theme => THEMES[i % THEMES.length]
const patternFor = (id: string): string => PATTERNS[hash(id + '·pat') % PATTERNS.length]
const CARD_CSS = `
.ma-page {
container-type: inline-size;
--ma-bg:#f8f8f9;
--ma-card-from:#ffffff; --ma-card-mid:#f2f3f6; --ma-card-to:#e6e8ee;
--ma-card-hover-from:#ffffff; --ma-card-hover-mid:#f5f6f9; --ma-card-hover-to:#eaecf1;
--ma-sheen:rgba(255,255,255,0.55); --ma-top-highlight:rgba(255,255,255,0.9);
--ma-border:rgba(0,0,0,0.09); --ma-border-hover:rgba(0,0,0,0.15);
--ma-shadow:0 1px 2px rgba(0,0,0,0.08);
--ma-title:#0d0e11; --ma-desc:rgba(0,0,0,0.6);
--ma-h1:#0d0e11; --ma-sub:rgba(0,0,0,0.5); --ma-lastrun:rgba(0,0,0,0.42);
--ma-off-bg:rgba(0,0,0,0.05); --ma-off-fg:rgba(0,0,0,0.5);
--ma-new-border:rgba(0,0,0,0.14); --ma-new-title:rgba(0,0,0,0.6); --ma-new-hint:rgba(0,0,0,0.4);
--ma-pat-opacity:0.10; --ma-glow-opacity:0.16; --ma-glow-hover-opacity:0.24;
--ma-badge-mix:20%; --ma-pill-mix:16%; --ma-tint:16%; --ma-tint-hover:22%;
height:100%; overflow:auto; background:var(--ma-bg);
}
.dark .ma-page {
--ma-bg:#0b0b0d;
--ma-card-from:#262930; --ma-card-mid:#191b21; --ma-card-to:#101116;
--ma-card-hover-from:#2b2e36; --ma-card-hover-mid:#1c1e25; --ma-card-hover-to:#131419;
--ma-sheen:rgba(255,255,255,0.07); --ma-top-highlight:rgba(255,255,255,0.09);
--ma-border:rgba(255,255,255,0.07); --ma-border-hover:rgba(255,255,255,0.12);
--ma-shadow:0 1px 2px rgba(0,0,0,0.35);
--ma-title:#f4f5f7; --ma-desc:rgba(255,255,255,0.66);
--ma-h1:#f4f5f7; --ma-sub:rgba(255,255,255,0.52); --ma-lastrun:rgba(255,255,255,0.38);
--ma-off-bg:rgba(255,255,255,0.06); --ma-off-fg:rgba(255,255,255,0.5);
--ma-new-border:rgba(255,255,255,0.12); --ma-new-title:rgba(255,255,255,0.6); --ma-new-hint:rgba(255,255,255,0.38);
--ma-pat-opacity:0.05; --ma-glow-opacity:0.10; --ma-glow-hover-opacity:0.16;
--ma-badge-mix:15%; --ma-pill-mix:13%; --ma-tint:20%; --ma-tint-hover:26%;
}
.ma-inner { max-width:1120px; margin:0 auto; padding:clamp(20px,3.5cqw,34px) clamp(16px,3cqw,30px) 48px; }
.ma-h1 { font-size:clamp(19px,2.6cqw,24px); font-weight:650; letter-spacing:-0.02em; color:var(--ma-h1); margin:0 0 4px; }
.ma-sub { font-size:clamp(13px,1.5cqw,14px); color:var(--ma-sub); margin:0 0 clamp(14px,2cqw,20px); }
.ma-tabs { display:flex; gap:6px; margin-bottom:clamp(14px,2cqw,22px); }
.ma-tab { border:1px solid var(--ma-border); background:transparent; color:var(--ma-sub); border-radius:999px; padding:5px 14px; font-size:13px; font-weight:600; cursor:pointer; }
.ma-tab.on { color:var(--ma-title); border-color:var(--ma-border-hover); background:color-mix(in srgb, var(--ma-title) 6%, transparent); }
.ma-banner { border:1px solid rgba(239,68,68,.4); background:rgba(239,68,68,.1); color:var(--ma-title); border-radius:12px; padding:10px 14px; font-size:13px; margin-bottom:16px; }
.ma-grid { display:grid; grid-template-columns:repeat(auto-fill, minmax(min(100%,248px),1fr)); gap:clamp(14px,2cqw,24px); }
.ma-card {
position:relative; min-height:clamp(190px,24cqw,244px); border-radius:18px;
border:1px solid var(--ma-border);
background:
linear-gradient(135deg, var(--ma-sheen) 0%, transparent 34%),
linear-gradient(158deg, color-mix(in srgb, var(--accent) var(--ma-tint), transparent) 0%, transparent 62%),
linear-gradient(158deg, var(--ma-card-from) 0%, var(--ma-card-mid) 52%, var(--ma-card-to) 100%);
padding:clamp(15px,2cqw,22px); text-align:left; cursor:pointer; overflow:hidden;
display:flex; flex-direction:column; isolation:isolate;
box-shadow: var(--ma-shadow), inset 0 1px 0 var(--ma-top-highlight), 0 8px 22px -20px var(--glow);
transition: box-shadow .22s ease, border-color .22s ease, background .22s ease;
}
.ma-card:hover {
border-color: var(--ma-border-hover);
background:
linear-gradient(135deg, var(--ma-sheen) 0%, transparent 36%),
linear-gradient(158deg, color-mix(in srgb, var(--accent) var(--ma-tint-hover), transparent) 0%, transparent 64%),
linear-gradient(158deg, var(--ma-card-hover-from) 0%, var(--ma-card-hover-mid) 52%, var(--ma-card-hover-to) 100%);
}
.ma-card::before { content:''; position:absolute; inset:0; z-index:-1; opacity:var(--ma-pat-opacity); pointer-events:none; }
.ma-card::after {
content:''; position:absolute; top:-45%; right:-25%; width:75%; height:75%; z-index:-1;
background: radial-gradient(circle, var(--accent) 0%, transparent 70%);
opacity:var(--ma-glow-opacity); filter: blur(18px); pointer-events:none; transition: opacity .22s ease;
}
.ma-card:hover::after { opacity:var(--ma-glow-hover-opacity); }
.ma-pat-dots::before { background-image: radial-gradient(var(--accent) 1px, transparent 1.4px); background-size:16px 16px; }
.ma-pat-grid::before { background-image: linear-gradient(var(--accent) 1px, transparent 1px), linear-gradient(90deg, var(--accent) 1px, transparent 1px); background-size:26px 26px; }
.ma-pat-diagonal::before { background-image: repeating-linear-gradient(45deg, var(--accent) 0 1px, transparent 1px 14px); }
.ma-pat-radial::before { background-image: radial-gradient(circle at 78% 18%, var(--accent) 0%, transparent 55%); opacity:calc(var(--ma-pat-opacity) + 0.05); }
.ma-pat-waves::before { background-image: repeating-radial-gradient(circle at 50% -30%, transparent 0 20px, var(--accent) 20px 21px); }
.ma-pat-mesh::before { background-image: radial-gradient(circle at 12% 18%, var(--accent) 0%, transparent 42%), radial-gradient(circle at 88% 82%, var(--accent) 0%, transparent 42%); opacity:calc(var(--ma-pat-opacity) + 0.03); }
.ma-pat-cross::before { background-image: repeating-linear-gradient(45deg, var(--accent) 0 1px, transparent 1px 18px), repeating-linear-gradient(-45deg, var(--accent) 0 1px, transparent 1px 18px); }
.ma-pat-rings::before { background-image: repeating-radial-gradient(circle at 82% 20%, transparent 0 14px, var(--accent) 14px 15px); }
.ma-pat-zigzag::before { background-image: linear-gradient(135deg, var(--accent) 25%, transparent 25%), linear-gradient(225deg, var(--accent) 25%, transparent 25%); background-size: 22px 12px; background-position: 0 0, 11px 0; opacity:calc(var(--ma-pat-opacity) - 0.03); }
.ma-pat-plus::before { background-image: radial-gradient(var(--accent) 0.8px, transparent 1px), linear-gradient(var(--accent) 1px, transparent 1px), linear-gradient(90deg, var(--accent) 1px, transparent 1px); background-size: 24px 24px, 24px 24px, 24px 24px; background-position: 12px 12px, 0 11.5px, 11.5px 0; opacity:calc(var(--ma-pat-opacity) - 0.02); }
.ma-pat-checker::before { background-image: repeating-conic-gradient(var(--accent) 0% 25%, transparent 0% 50%); background-size: 26px 26px; opacity:calc(var(--ma-pat-opacity) - 0.04); }
.ma-pat-beams::before { background-image: repeating-linear-gradient(100deg, var(--accent) 0 2px, transparent 2px 34px); }
.ma-top { display:flex; justify-content:flex-end; gap:6px; }
.ma-badge {
display:inline-flex; align-items:center; height:22px; padding:0 10px; border-radius:999px;
font-size:9.5px; font-weight:600; letter-spacing:0.07em;
color: var(--accent); background: color-mix(in srgb, var(--accent) var(--ma-badge-mix), transparent);
}
.ma-badge.off { color: var(--ma-off-fg); background: var(--ma-off-bg); }
.ma-badge.err { color:#ef4444; background:rgba(239,68,68,.14); }
.ma-title { font-size:clamp(17px,2.3cqw,21px); font-weight:600; letter-spacing:-0.02em; color:var(--ma-title); margin:clamp(12px,2cqw,18px) 0 8px; }
.ma-desc {
font-size:clamp(13px,1.5cqw,14.5px); font-weight:400; line-height:1.45; color:var(--ma-desc); margin:0;
display:-webkit-box; -webkit-line-clamp:2; -webkit-box-orient:vertical; overflow:hidden;
}
.ma-footer { margin-top:auto; padding-top:clamp(14px,2cqw,22px); display:flex; align-items:center; justify-content:space-between; gap:10px; }
.ma-source { font-size:11.5px; font-weight:600; color:var(--accent); background: color-mix(in srgb, var(--accent) var(--ma-pill-mix), transparent); padding:5px 10px; border-radius:999px; white-space:nowrap; }
.ma-lastrun { font-size:11.5px; color:var(--ma-lastrun); white-space:nowrap; overflow:hidden; text-overflow:ellipsis; }
.ma-new {
width:100%; font:inherit; min-height:clamp(190px,24cqw,244px); border-radius:18px; border:1px dashed var(--ma-new-border);
background:transparent; display:flex; flex-direction:column; align-items:center; justify-content:center;
gap:8px; color:var(--ma-new-title); cursor:pointer; transition: border-color .2s ease, background .2s ease;
}
.ma-new:hover { border-color:var(--ma-border-hover); background:color-mix(in srgb, var(--accent, #888) 6%, transparent); }
.ma-new-title { font-size:14.5px; font-weight:600; color:var(--ma-new-title); }
.ma-new-hint { font-size:12px; color:var(--ma-new-hint); text-align:center; padding:0 12px; }
.ma-empty { padding:36px 8px; text-align:center; color:var(--ma-sub); font-size:14px; grid-column:1/-1; }
@container (max-width: 380px) {
.ma-footer { flex-direction:column; align-items:flex-start; gap:6px; }
}
`
function Card({ app, index, onOpen }: { app: rowboatApp.AppSummary; index: number; onOpen: () => void }) {
const theme = themeForIndex(index)
const pattern = patternFor(app.folder)
const invalid = app.status === 'invalid'
return (
<button
type="button"
onClick={onOpen}
title={invalid ? app.manifestError : undefined}
className={`ma-card ma-pat-${pattern}`}
style={{ '--accent': theme.accent, '--glow': theme.glow } as React.CSSProperties}
>
<div className="ma-top">
{invalid && <span className="ma-badge err">INVALID</span>}
<span className={`ma-badge${app.kind === 'installed' ? '' : ' off'}`}>
{app.kind === 'installed' ? 'INSTALLED' : 'LOCAL'}
</span>
</div>
<div className="ma-title">{app.manifest?.name ?? app.folder}</div>
<div className="ma-desc">{invalid ? (app.manifestError ?? 'Invalid manifest') : (app.manifest?.description || 'No description yet.')}</div>
<div className="ma-footer">
<span className="ma-source">v{app.manifest?.version ?? '?'}</span>
<span className="ma-lastrun">{app.folder}</span>
</div>
</button>
)
}
export function AppsView({ initialAppFolder, initialVersion, onNewApp }: {
initialAppFolder?: string | null
initialVersion?: number
onNewApp?: () => void
} = {}) {
const [tab, setTab] = useState<'mine' | 'catalog'>('mine')
const [selectedFolder, setSelectedFolder] = useState<string | null>(initialAppFolder ?? null)
const [apps, setApps] = useState<rowboatApp.AppSummary[]>([])
const [serverError, setServerError] = useState<string | null>(null)
// Open a specific app when asked from outside (app-navigation open-app).
const [appliedVersion, setAppliedVersion] = useState(initialVersion)
if (initialVersion !== appliedVersion) {
setAppliedVersion(initialVersion)
if (initialAppFolder) setSelectedFolder(initialAppFolder)
}
useEffect(() => {
let cancelled = false
const load = async () => {
try {
const r = await window.ipc.invoke('apps:list', {})
if (cancelled) return
setApps(r.apps)
setServerError(r.serverRunning ? null : (r.serverError ?? 'Apps server is not running.'))
} catch (e) {
if (!cancelled) setServerError(e instanceof Error ? e.message : String(e))
}
}
void load()
const interval = setInterval(load, 4000) // keep the grid fresh (copilot installs)
return () => { cancelled = true; clearInterval(interval) }
}, [initialVersion])
const selected = selectedFolder ? apps.find((a) => a.folder === selectedFolder) : undefined
if (selected) {
return <AppFrame app={selected} onBack={() => setSelectedFolder(null)} />
}
return (
<div className="ma-page">
<style>{CARD_CSS}</style>
<div className="ma-inner">
<h1 className="ma-h1">Apps</h1>
<p className="ma-sub">Apps that live inside Rowboat, powered by your agents and integrations.</p>
<div className="ma-tabs">
<button type="button" className={`ma-tab${tab === 'mine' ? ' on' : ''}`} onClick={() => setTab('mine')}>My apps</button>
<button type="button" className={`ma-tab${tab === 'catalog' ? ' on' : ''}`} onClick={() => setTab('catalog')}>Catalog</button>
</div>
{serverError && (
<div className="ma-banner">
<RefreshCw className="mr-1.5 inline size-3.5" /> Apps server unavailable: {serverError}
</div>
)}
{tab === 'catalog' ? (
<div className="ma-empty">The community catalog is coming soon.</div>
) : (
<div className="ma-grid">
{apps.map((app, i) => (
<Card key={app.folder} app={app} index={i} onOpen={() => setSelectedFolder(app.folder)} />
))}
<button type="button" className="ma-new" onClick={onNewApp}>
<Plus className="size-5" />
<div className="ma-new-title">New app</div>
<div className="ma-new-hint">Describe one to the copilot</div>
</button>
</div>
)}
</div>
</div>
)
}

View file

@ -1,7 +1,19 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { ArrowLeft, ArrowRight, Loader2, Plus, RotateCw, X } from 'lucide-react'
import type { HttpAuthRequest } from '@x/shared/dist/browser-control.js'
import { TabBar } from '@/components/tab-bar'
import { Button } from '@/components/ui/button'
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from '@/components/ui/dialog'
import { Input } from '@/components/ui/input'
import { cn } from '@/lib/utils'
/**
@ -86,9 +98,80 @@ const getBrowserTabTitle = (tab: BrowserTabState) => {
}
}
/**
* Credential prompt for HTTP basic/proxy auth challenges raised by pages in
* the embedded browser. Rendered as a regular app dialog: BrowserPane already
* hides the native WebContentsView whenever a dialog overlay is open, so the
* prompt is never obscured by the page that triggered it.
*/
function BrowserHttpAuthDialog({
request,
onSubmit,
onCancel,
}: {
request: HttpAuthRequest
onSubmit: (username: string, password: string) => void
onCancel: () => void
}) {
const [username, setUsername] = useState('')
const [password, setPassword] = useState('')
// Basic auth allows an empty username (token-style `curl -u :TOKEN`), so the
// only invalid submission is fully empty. The server decides the rest.
const canSubmit = username.length > 0 || password.length > 0
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault()
if (!canSubmit) return
onSubmit(username, password)
}
return (
<Dialog open onOpenChange={(open) => { if (!open) onCancel() }}>
<DialogContent className="w-[min(24rem,calc(100%-2rem))] max-w-sm">
<DialogHeader>
<DialogTitle>Sign in</DialogTitle>
<DialogDescription>
{request.isProxy
? `The proxy ${request.host} requires a username and password.`
: `${request.host} requires a username and password.`}
{request.realm ? ` (${request.realm})` : ''}
</DialogDescription>
</DialogHeader>
<form onSubmit={handleSubmit} className="flex flex-col gap-3">
<Input
autoFocus
value={username}
onChange={(e) => setUsername(e.target.value)}
placeholder="Username"
autoCapitalize="off"
autoCorrect="off"
spellCheck={false}
/>
<Input
type="password"
value={password}
onChange={(e) => setPassword(e.target.value)}
placeholder="Password"
/>
<DialogFooter>
<Button type="button" variant="outline" onClick={onCancel}>
Cancel
</Button>
<Button type="submit" disabled={!canSubmit}>
Sign in
</Button>
</DialogFooter>
</form>
</DialogContent>
</Dialog>
)
}
export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps) {
const [state, setState] = useState<BrowserState>(EMPTY_STATE)
const [addressValue, setAddressValue] = useState('')
const [authQueue, setAuthQueue] = useState<HttpAuthRequest[]>([])
const activeTabIdRef = useRef<string | null>(null)
const addressFocusedRef = useRef(false)
@ -121,6 +204,51 @@ export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps)
return cleanup
}, [applyState])
// Mirror of authQueue for the unmount handler, which must read the latest
// queue without re-subscribing on every change.
const authQueueRef = useRef<HttpAuthRequest[]>([])
useEffect(() => {
authQueueRef.current = authQueue
}, [authQueue])
useEffect(() => {
const offRequest = window.ipc.on('browser:httpAuthRequest', (incoming) => {
setAuthQueue((queue) => [...queue, incoming as HttpAuthRequest])
})
// Main resolved a challenge on its own (timeout, or its tab/window was
// destroyed) — drop the corresponding dialog so it can't linger over an
// unrelated page with a submit that would no-op.
const offResolved = window.ipc.on('browser:httpAuthResolved', (incoming) => {
const { requestId } = incoming as { requestId: string }
setAuthQueue((queue) => queue.filter((request) => request.requestId !== requestId))
})
return () => {
offRequest()
offResolved()
// Cancel anything still pending so the main-process login callbacks and
// timers are freed immediately instead of waiting out the timeout.
for (const request of authQueueRef.current) {
void window.ipc.invoke('browser:httpAuthResponse', { requestId: request.requestId })
}
}
}, [])
const respondToAuth = useCallback(
(requestId: string, credentials: { username: string; password: string } | null) => {
setAuthQueue((queue) => queue.filter((request) => request.requestId !== requestId))
// Omit username to cancel; include it (even empty) to submit.
void window.ipc.invoke(
'browser:httpAuthResponse',
credentials
? { requestId, username: credentials.username, password: credentials.password }
: { requestId },
)
},
[],
)
const activeAuthRequest = authQueue[0] ?? null
const setViewVisible = useCallback((visible: boolean) => {
if (viewVisibleRef.current === visible) return
viewVisibleRef.current = visible
@ -420,6 +548,17 @@ export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps)
className="relative min-h-0 min-w-0 flex-1"
data-browser-viewport
/>
{activeAuthRequest && (
<BrowserHttpAuthDialog
key={activeAuthRequest.requestId}
request={activeAuthRequest}
onSubmit={(username, password) =>
respondToAuth(activeAuthRequest.requestId, { username, password })
}
onCancel={() => respondToAuth(activeAuthRequest.requestId, null)}
/>
)}
</div>
)
}

View file

@ -1,4 +1,4 @@
import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'
import {
ArrowUp,
@ -15,16 +15,19 @@ import {
FolderCog,
FolderOpen,
Globe,
Headphones,
ImagePlus,
LoaderIcon,
Lock,
Mic,
MoreHorizontal,
Phone,
PhoneOff,
Plus,
Presentation,
ShieldCheck,
Square,
Terminal,
Video,
X,
} from 'lucide-react'
@ -210,6 +213,18 @@ function compactWorkDirPath(path: string) {
return path.replace(/^\/Users\/[^/]+/, '~')
}
// Call presets: front doors into the same call engine, differing only in
// starting devices. 'share' is the call button's main click — the "work
// together" default (screen shared, camera off, floating pill). The chevron
// menu holds the deviations.
export type CallPreset = 'voice' | 'video' | 'share' | 'practice'
const CALL_PRESET_MENU: Array<{ preset: CallPreset; label: string; description: string; Icon: typeof Phone }> = [
{ preset: 'voice', label: 'Voice call', description: 'Just talk — nothing is shared, the mascot hovers while you work', Icon: AudioLines },
{ preset: 'video', label: 'Video call', description: 'Camera on, face to face — it sees your expressions', Icon: Video },
{ preset: 'practice', label: 'Practice session', description: 'Rehearse a pitch or interview with live coaching', Icon: Presentation },
]
interface ChatInputInnerProps {
onSubmit: (message: PromptInputMessage, mentions?: FileMention[], attachments?: StagedAttachment[], searchEnabled?: boolean, codeMode?: 'claude' | 'codex', permissionMode?: PermissionMode) => void
onStop?: () => void
@ -230,11 +245,13 @@ interface ChatInputInnerProps {
onSubmitRecording?: () => void | Promise<void>
onCancelRecording?: () => void
voiceAvailable?: boolean
ttsAvailable?: boolean
ttsEnabled?: boolean
ttsMode?: 'summary' | 'full'
onToggleTts?: () => void
onTtsModeChange?: (mode: 'summary' | 'full') => void
/** A call is live (hands-free voice loop + spoken responses). */
inCall?: boolean
/** Start a call with the given preset's device defaults. */
onStartCall?: (preset: CallPreset) => void
onEndCall?: () => void
/** Calls need both voice input (STT) and voice output (TTS) configured. */
callAvailable?: boolean
/** Fired when the user picks a different model in the dropdown (only when no run exists yet). */
onSelectedModelChange?: (model: SelectedModel | null) => void
/** Work directory for this chat (per-chat). Null when none is set. */
@ -268,11 +285,10 @@ function ChatInputInner({
onSubmitRecording,
onCancelRecording,
voiceAvailable,
ttsAvailable,
ttsEnabled,
ttsMode,
onToggleTts,
onTtsModeChange,
inCall,
onStartCall,
onEndCall,
callAvailable,
onSelectedModelChange,
workDir = null,
onWorkDirChange,
@ -287,6 +303,11 @@ function ChatInputInner({
const [configuredModels, setConfiguredModels] = useState<ConfiguredModel[]>([])
const [activeModelKey, setActiveModelKey] = useState('')
// The effective runtime default (what a run actually uses when the user
// hasn't picked a model) — shown in the picker instead of guessing from
// list order, which can disagree with the real default.
const [defaultModel, setDefaultModel] = useState<ConfiguredModel | null>(null)
const loadModelConfigEpoch = useRef(0)
const [lockedModel, setLockedModel] = useState<SelectedModel | null>(null)
const [searchEnabled, setSearchEnabled] = useState(false)
const [searchAvailable, setSearchAvailable] = useState(false)
@ -379,47 +400,54 @@ function ChatInputInner({
return cleanup
}, [])
// Load the list of models the user can choose from.
// Signed-in: gateway model list. Signed-out: providers configured in models.json.
// Load the list of models the user can choose from. Hybrid mode: signed-in
// users get the gateway list AND every BYOK provider configured in
// models.json (selecting a BYOK model routes that message through the
// user's own key / local server). Signed-out users get BYOK only.
const loadModelConfig = useCallback(async () => {
// Concurrent runs race (mount fires one before the sign-in state resolves,
// which fires another) — only the newest run may write state, else a slow
// stale run can clobber the fresh list with an empty one.
const epoch = ++loadModelConfigEpoch.current
try {
if (isRowboatConnected) {
const def = await window.ipc.invoke('llm:getDefaultModel', null)
if (loadModelConfigEpoch.current !== epoch) return
setDefaultModel({ provider: def.provider as ProviderName, model: def.model })
} catch {
if (loadModelConfigEpoch.current === epoch) setDefaultModel(null)
}
try {
const models: ConfiguredModel[] = []
const seen = new Set<string>()
const push = (provider: string, model: string) => {
if (!model) return
const key = `${provider}/${model}`
if (seen.has(key)) return
seen.add(key)
models.push({ provider: provider as ProviderName, model })
}
// Full catalog per provider (gateway + cloud). Providers with no
// catalog (Ollama, OpenAI-compatible) fall back to the models saved in
// config below.
const catalog: Record<string, string[]> = {}
try {
const listResult = await window.ipc.invoke('models:list', null)
const rowboatProvider = listResult.providers?.find(
(p: { id: string }) => p.id === 'rowboat'
)
const models: ConfiguredModel[] = (rowboatProvider?.models || []).map(
(m: { id: string }) => ({ provider: 'rowboat', model: m.id })
)
setConfiguredModels(models)
} else {
for (const p of listResult.providers || []) {
catalog[p.id] = (p.models || []).map((m: { id: string }) => m.id)
}
} catch { /* offline / no catalog — fall back to saved config below */ }
if (isRowboatConnected) {
for (const m of catalog['rowboat'] || []) push('rowboat', m)
}
try {
const result = await window.ipc.invoke('workspace:readFile', { path: 'config/models.json' })
const parsed = JSON.parse(result.data)
// Offer every model the configured key supports — the same full catalog
// Settings uses for its dropdowns — so BYOK chat matches the signed-in
// gateway picker. The picker is no longer limited to a hand-curated
// config.models list. Providers with no catalog (Ollama, OpenAI-compatible)
// fall back to the model saved in config.
const catalog: Record<string, string[]> = {}
try {
const listResult = await window.ipc.invoke('models:list', null)
for (const p of listResult.providers || []) {
catalog[p.id] = (p.models || []).map((m: { id: string }) => m.id)
}
} catch { /* offline / no catalog — fall back to saved config below */ }
const models: ConfiguredModel[] = []
const seen = new Set<string>()
const push = (provider: string, model: string) => {
if (!model) return
const key = `${provider}/${model}`
if (seen.has(key)) return
seen.add(key)
models.push({ provider: provider as ProviderName, model })
}
// List the default provider first so its default model leads the picker.
// List the default provider first so its default model leads the
// BYOK section of the picker.
const defaultFlavor = typeof parsed?.provider?.flavor === 'string' ? parsed.provider.flavor : ''
const flavors = Object.keys(parsed?.providers || {})
.sort((a, b) => (a === defaultFlavor ? -1 : b === defaultFlavor ? 1 : 0))
@ -441,10 +469,24 @@ function ChatInputInner({
for (const m of saved) push(flavor, m)
}
}
setConfiguredModels(models)
}
} catch {
// No config yet
// The user's explicit default selection leads the whole picker.
const sel = parsed?.defaultSelection
if (sel && typeof sel.provider === 'string' && typeof sel.model === 'string') {
const selKey = `${sel.provider}/${sel.model}`
const index = models.findIndex((m) => `${m.provider}/${m.model}` === selKey)
if (index > 0) {
const [entry] = models.splice(index, 1)
models.unshift(entry)
}
}
} catch { /* no BYOK config yet */ }
if (loadModelConfigEpoch.current !== epoch) return
setConfiguredModels(models)
} catch (err) {
// No config yet — but surface unexpected failures for diagnosis.
console.error('[chat-input] failed to load model list', err)
}
}, [isRowboatConnected])
@ -631,15 +673,25 @@ function ChatInputInner({
checkSearch()
}, [isActive, isRowboatConnected])
// The dropdown's items: always include the effective default so the picker
// is never empty (and never missing the model that actually runs) even
// while the full list is still loading.
const pickerModels = useMemo<ConfiguredModel[]>(() => {
if (!defaultModel) return configuredModels
const defaultKey = `${defaultModel.provider}/${defaultModel.model}`
if (configuredModels.some((m) => `${m.provider}/${m.model}` === defaultKey)) return configuredModels
return [defaultModel, ...configuredModels]
}, [configuredModels, defaultModel])
// Selecting a model affects only the *next* run created from this tab.
// Once a run exists, model is frozen on the run and the dropdown is read-only.
const handleModelChange = useCallback((key: string) => {
if (lockedModel) return
const entry = configuredModels.find((m) => `${m.provider}/${m.model}` === key)
const entry = pickerModels.find((m) => `${m.provider}/${m.model}` === key)
if (!entry) return
setActiveModelKey(key)
onSelectedModelChange?.({ provider: entry.provider, model: entry.model })
}, [configuredModels, lockedModel, onSelectedModelChange])
}, [pickerModels, lockedModel, onSelectedModelChange])
// Restore the tab draft when this input mounts.
useEffect(() => {
@ -752,7 +804,7 @@ function ChatInputInner({
const currentWorkDirPath = effectiveWorkDir ? compactWorkDirPath(effectiveWorkDir) : ''
return (
<div className="rowboat-chat-input rounded-lg border border-border bg-background shadow-none">
<div data-tour-id="chat-composer" className="rowboat-chat-input rounded-lg border border-border bg-background shadow-none">
{attachments.length > 0 && (
<div className="flex flex-wrap gap-2 px-4 pb-1 pt-3">
{attachments.map((attachment) => {
@ -1243,7 +1295,7 @@ function ChatInputInner({
{providerDisplayNames[lockedModel.provider] || lockedModel.provider} fixed for this chat
</TooltipContent>
</Tooltip>
) : configuredModels.length > 0 ? (
) : pickerModels.length > 0 ? (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<button
@ -1251,14 +1303,22 @@ function ChatInputInner({
className="flex h-7 min-w-0 items-center gap-1 rounded-full px-2 text-xs text-muted-foreground transition-colors hover:bg-muted hover:text-foreground"
>
<span className="min-w-0 truncate">
{getSelectedModelDisplayName(configuredModels.find((m) => `${m.provider}/${m.model}` === activeModelKey)?.model || configuredModels[0]?.model || 'Model')}
{getSelectedModelDisplayName(
pickerModels.find((m) => `${m.provider}/${m.model}` === activeModelKey)?.model
|| defaultModel?.model
|| pickerModels[0]?.model
|| 'Model'
)}
</span>
<ChevronDown className="h-3 w-3 shrink-0" />
</button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuRadioGroup value={activeModelKey} onValueChange={handleModelChange}>
{configuredModels.map((m) => {
<DropdownMenuRadioGroup
value={activeModelKey || (defaultModel ? `${defaultModel.provider}/${defaultModel.model}` : '')}
onValueChange={handleModelChange}
>
{pickerModels.map((m) => {
const key = `${m.provider}/${m.model}`
return (
<DropdownMenuRadioItem key={key} value={key}>
@ -1271,48 +1331,66 @@ function ChatInputInner({
</DropdownMenuContent>
</DropdownMenu>
) : null}
{onToggleTts && ttsAvailable && (
{onStartCall && (
<div className="flex shrink-0 items-center">
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
<TooltipTrigger asChild>
<button
type="button"
onClick={onToggleTts}
onClick={() => {
if (inCall) {
onEndCall?.()
} else if (callAvailable) {
onStartCall('share')
}
}}
className={cn(
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
ttsEnabled
? 'text-foreground hover:bg-muted'
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
'flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
inCall
? 'bg-red-600 text-white hover:bg-red-500'
: callAvailable
? 'text-muted-foreground hover:bg-muted hover:text-foreground'
: 'cursor-default text-muted-foreground/40'
)}
aria-label={ttsEnabled ? 'Disable voice output' : 'Enable voice output'}
aria-label={inCall ? 'End call' : 'Start a call'}
>
<Headphones className="h-4 w-4" />
{!ttsEnabled && (
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
</span>
)}
{inCall ? <PhoneOff className="h-4 w-4" /> : <Phone className="h-4 w-4" />}
</button>
</TooltipTrigger>
<TooltipContent side="top">
{ttsEnabled ? 'Voice output on' : 'Voice output off'}
{inCall
? 'End call'
: callAvailable
? 'Start a call — it sees your screen while you talk it through'
: 'Calls need voice input and output configured'}
</TooltipContent>
</Tooltip>
{ttsEnabled && onTtsModeChange && (
{!inCall && (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<button
type="button"
className="flex h-7 w-4 shrink-0 items-center justify-center text-muted-foreground transition-colors hover:text-foreground"
aria-label="Call options"
>
<ChevronDown className="h-3 w-3" />
</button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuRadioGroup value={ttsMode ?? 'summary'} onValueChange={(v) => onTtsModeChange(v as 'summary' | 'full')}>
<DropdownMenuRadioItem value="summary">Speak summary</DropdownMenuRadioItem>
<DropdownMenuRadioItem value="full">Speak full response</DropdownMenuRadioItem>
</DropdownMenuRadioGroup>
<DropdownMenuContent align="end" className="w-72">
{CALL_PRESET_MENU.map(({ preset, label, description, Icon }) => (
<DropdownMenuItem
key={preset}
disabled={!callAvailable}
onSelect={() => onStartCall(preset)}
className="items-start gap-3 py-2"
>
<Icon className="mt-0.5 h-4 w-4 shrink-0 text-muted-foreground" />
<span className="min-w-0">
<span className="block text-sm font-medium leading-tight">{label}</span>
<span className="block pt-0.5 text-xs leading-tight text-muted-foreground">{description}</span>
</span>
</DropdownMenuItem>
))}
</DropdownMenuContent>
</DropdownMenu>
)}
@ -1487,11 +1565,10 @@ export interface ChatInputWithMentionsProps {
onSubmitRecording?: () => void | Promise<void>
onCancelRecording?: () => void
voiceAvailable?: boolean
ttsAvailable?: boolean
ttsEnabled?: boolean
ttsMode?: 'summary' | 'full'
onToggleTts?: () => void
onTtsModeChange?: (mode: 'summary' | 'full') => void
inCall?: boolean
onStartCall?: (preset: CallPreset) => void
onEndCall?: () => void
callAvailable?: boolean
onSelectedModelChange?: (model: SelectedModel | null) => void
workDir?: string | null
onWorkDirChange?: (value: string | null) => void
@ -1521,11 +1598,10 @@ export function ChatInputWithMentions({
onSubmitRecording,
onCancelRecording,
voiceAvailable,
ttsAvailable,
ttsEnabled,
ttsMode,
onToggleTts,
onTtsModeChange,
inCall,
onStartCall,
onEndCall,
callAvailable,
onSelectedModelChange,
workDir,
onWorkDirChange,
@ -1552,11 +1628,10 @@ export function ChatInputWithMentions({
onSubmitRecording={onSubmitRecording}
onCancelRecording={onCancelRecording}
voiceAvailable={voiceAvailable}
ttsAvailable={ttsAvailable}
ttsEnabled={ttsEnabled}
ttsMode={ttsMode}
onToggleTts={onToggleTts}
onTtsModeChange={onTtsModeChange}
inCall={inCall}
onStartCall={onStartCall}
onEndCall={onEndCall}
callAvailable={callAvailable}
onSelectedModelChange={onSelectedModelChange}
workDir={workDir}
onWorkDirChange={onWorkDirChange}

View file

@ -91,11 +91,28 @@ interface ChatMessageAttachmentsProps {
export function ChatMessageAttachments({ attachments, className }: ChatMessageAttachmentsProps) {
if (attachments.length === 0) return null
const imageAttachments = attachments.filter((attachment) => isImageMime(attachment.mimeType))
const fileAttachments = attachments.filter((attachment) => !isImageMime(attachment.mimeType))
const videoFrames = attachments.filter((attachment) => attachment.isVideoFrame)
const imageAttachments = attachments.filter(
(attachment) => !attachment.isVideoFrame && isImageMime(attachment.mimeType)
)
const fileAttachments = attachments.filter(
(attachment) => !attachment.isVideoFrame && !isImageMime(attachment.mimeType)
)
return (
<div className={cn('flex flex-col items-end gap-2', className)}>
{videoFrames.length > 0 && (
<div className="flex max-w-[340px] flex-wrap justify-end gap-1">
{videoFrames.map((frame, index) => (
<img
key={`frame-${index}`}
src={frame.thumbnailUrl}
alt={`Camera frame ${index + 1}`}
className="h-12 w-auto rounded-md border border-border/60 bg-muted object-cover"
/>
))}
</div>
)}
{imageAttachments.length > 0 && (
<div className="flex flex-wrap justify-end gap-2">
{imageAttachments.map((attachment, index) => (

View file

@ -37,7 +37,7 @@ import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-over
import { defaultRemarkPlugins } from 'streamdown'
import remarkBreaks from 'remark-breaks'
import { type ChatTab } from '@/components/tab-bar'
import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions'
import { ChatInputWithMentions, type CallPreset, type PermissionMode, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions'
import { ChatMessageAttachments } from '@/components/chat-message-attachments'
import { useSidebar } from '@/components/ui/sidebar'
import { wikiLabel } from '@/lib/wiki-links'
@ -51,6 +51,7 @@ import {
getWebSearchCardData,
getComposioConnectCardData,
getToolDisplayName,
getToolErrorText,
groupConversationItems,
isChatMessage,
isErrorMessage,
@ -187,11 +188,10 @@ interface ChatSidebarProps {
onSubmitRecording?: () => void | Promise<void>
onCancelRecording?: () => void
voiceAvailable?: boolean
ttsAvailable?: boolean
ttsEnabled?: boolean
ttsMode?: 'summary' | 'full'
onToggleTts?: () => void
onTtsModeChange?: (mode: 'summary' | 'full') => void
inCall?: boolean
onStartCall?: (preset: CallPreset) => void
onEndCall?: () => void
callAvailable?: boolean
onComposioConnected?: (toolkitSlug: string) => void
}
@ -251,11 +251,10 @@ export function ChatSidebar({
onSubmitRecording,
onCancelRecording,
voiceAvailable,
ttsAvailable,
ttsEnabled,
ttsMode,
onToggleTts,
onTtsModeChange,
inCall,
onStartCall,
onEndCall,
callAvailable,
onComposioConnected,
}: ChatSidebarProps) {
const { state: sidebarState } = useSidebar()
@ -486,7 +485,7 @@ export function ChatSidebar({
)
}
const toolTitle = getToolDisplayName(item)
const errorText = item.status === 'error' ? 'Tool error' : ''
const errorText = getToolErrorText(item)
const output = normalizeToolOutput(item.result, item.status)
const input = normalizeToolInput(item.input)
return (
@ -825,11 +824,10 @@ export function ChatSidebar({
onSubmitRecording={isActive ? onSubmitRecording : undefined}
onCancelRecording={isActive ? onCancelRecording : undefined}
voiceAvailable={isActive && voiceAvailable}
ttsAvailable={isActive && ttsAvailable}
ttsEnabled={ttsEnabled}
ttsMode={ttsMode}
onToggleTts={isActive ? onToggleTts : undefined}
onTtsModeChange={isActive ? onTtsModeChange : undefined}
inCall={inCall}
onStartCall={isActive ? onStartCall : undefined}
onEndCall={isActive ? onEndCall : undefined}
callAvailable={callAvailable}
/>
</div>
)

View file

@ -10,7 +10,7 @@ import { Conversation, ConversationContent, ConversationScrollButton } from '@/c
import { MessageResponse } from '@/components/ai-elements/message'
import { Shimmer } from '@/components/ai-elements/shimmer'
import { Tool, ToolContent, ToolHeader } from '@/components/ai-elements/tool'
import { toToolState, getToolDisplayName, getWebSearchCardData, type ToolCall } from '@/lib/chat-conversation'
import { toToolState, getToolDisplayName, getToolErrorText, getWebSearchCardData, type ToolCall } from '@/lib/chat-conversation'
import { CodeRunPermissionRequest, CodingRunTimeline } from '@/components/coding-run'
import { PermissionRequest } from '@/components/ai-elements/permission-request'
import { AskHumanRequest } from '@/components/ai-elements/ask-human-request'
@ -84,7 +84,7 @@ function RowboatToolCall({ item, onOpenDiff }: { item: ToolCall; onOpenDiff: (pa
<Tool open={open || item.status === 'running'} onOpenChange={setOpen}>
<ToolHeader title={AGENT_LABEL[agent ?? ''] ?? 'Coding agent'} type="tool-code_agent_run" state={toToolState(item.status)} />
<ToolContent>
<CodingRunTimeline events={item.codeRunEvents ?? []} onOpenDiff={onOpenDiff} />
<CodingRunTimeline events={item.codeRunEvents ?? []} error={getToolErrorText(item)} onOpenDiff={onOpenDiff} />
</ToolContent>
</Tool>
)

View file

@ -1,5 +1,6 @@
import { useMemo, useState } from 'react'
import { useEffect, useMemo, useState } from 'react'
import {
AlertCircle,
CheckCircle2,
Circle,
CircleDot,
@ -17,7 +18,8 @@ import {
import type { CodeRunEvent, PermissionAsk, PermissionDecision } from '@x/shared/src/code-mode.js'
import { cn } from '@/lib/utils'
import { Tool, ToolContent, ToolHeader } from '@/components/ai-elements/tool'
import { toToolState, type ToolCall } from '@/lib/chat-conversation'
import { getToolErrorText, toToolState, type ToolCall } from '@/lib/chat-conversation'
import { clearCodeRunBuffer, useCodeRunFeed } from '@/lib/code-run-feed'
// ── Timeline reduction ──────────────────────────────────────────────
// The raw ACP stream is a flat list of events; collapse it into ordered rows,
@ -111,14 +113,26 @@ const basename = (p: string) => p.split(/[\\/]/).pop() || p
export function CodingRunTimeline({
events,
error,
onOpenDiff,
}: {
events: CodeRunEvent[]
error?: string
// When set, changed-file names become clickable (the Code section opens the diff).
onOpenDiff?: (path: string) => void
}) {
const rows = useMemo(() => reduceEvents(events), [events])
if (rows.length === 0) {
if (error) {
return (
<div className="px-4 py-3">
<div className="flex items-start gap-2 rounded-md border border-destructive/30 bg-destructive/5 px-3 py-2 text-sm text-destructive">
<AlertCircle className="mt-0.5 size-3.5 shrink-0" />
<span className="min-w-0 whitespace-pre-wrap break-words">{error}</span>
</div>
</div>
)
}
return <div className="px-4 py-3 text-xs text-muted-foreground">Starting the agent</div>
}
return (
@ -133,12 +147,17 @@ export function CodingRunTimeline({
}
if (row.kind === 'tool') {
const running = row.status !== 'completed' && row.status !== 'failed'
const failed = row.status === 'failed'
return (
<div key={row.id} className="flex flex-col gap-1">
<div className="flex items-center gap-2 text-sm">
{running
? <Loader className="size-3.5 shrink-0 animate-spin text-muted-foreground" />
: <CheckCircle2 className="size-3.5 shrink-0 text-green-600" />}
{running ? (
<Loader className="size-3.5 shrink-0 animate-spin text-muted-foreground" />
) : failed ? (
<AlertCircle className="size-3.5 shrink-0 text-destructive" />
) : (
<CheckCircle2 className="size-3.5 shrink-0 text-green-600" />
)}
{toolKindIcon(row.toolKind, row.title)}
<span className="truncate text-foreground/90">{row.title ?? row.toolKind ?? 'Tool call'}</span>
</div>
@ -189,6 +208,12 @@ export function CodingRunTimeline({
</div>
)
})}
{error && (
<div className="flex items-start gap-2 rounded-md border border-destructive/30 bg-destructive/5 px-3 py-2 text-sm text-destructive">
<AlertCircle className="mt-0.5 size-3.5 shrink-0" />
<span className="min-w-0 whitespace-pre-wrap break-words">{error}</span>
</div>
)}
</div>
)
}
@ -258,12 +283,23 @@ export function CodingRunBlock({
(item.result as { agent?: string } | undefined)?.agent ??
(item.input as { agent?: string } | undefined)?.agent
const title = AGENT_LABEL[agent ?? ''] ?? 'Coding agent'
const error = getToolErrorText(item)
// Timeline source: the durable record (item.codeRunEvents — the settle-time
// batch, or the legacy path's inline accumulation) wins; while it's absent
// the live CodeRunFeed buffer streams the run in real time.
const liveEvents = useCodeRunFeed(item.id)
const durableEvents = item.codeRunEvents
const events = durableEvents?.length ? durableEvents : liveEvents
// Once the durable batch has landed the buffer is redundant — drop it.
useEffect(() => {
if (durableEvents?.length) clearCodeRunBuffer(item.id)
}, [durableEvents?.length, item.id])
return (
<>
<Tool open={open} onOpenChange={onOpenChange}>
<ToolHeader title={title} type="tool-code_agent_run" state={toToolState(item.status)} />
<ToolContent>
<CodingRunTimeline events={item.codeRunEvents ?? []} />
<CodingRunTimeline events={events} error={error} />
</ToolContent>
</Tool>
{item.pendingCodePermission && (

View file

@ -7,6 +7,7 @@ import {
isErrorMessage,
isToolCall,
getToolDisplayName,
getToolErrorText,
toToolState,
normalizeToolOutput,
} from '@/lib/chat-conversation'
@ -62,7 +63,7 @@ function CompactToolRow({ tool }: { tool: ToolCall }) {
const [open, setOpen] = useState(false)
const title = getToolDisplayName(tool)
const state = toToolState(tool.status)
const errorText = tool.status === 'error' && typeof tool.result === 'string' ? tool.result : undefined
const errorText = getToolErrorText(tool)
return (
<Tool open={open} onOpenChange={setOpen} className="mb-0 text-xs">
<ToolHeader title={title} type={`tool-${tool.name}` as `tool-${string}`} state={state} />

File diff suppressed because it is too large Load diff

View file

@ -183,8 +183,8 @@ export function OnboardingModal({ open, onComplete }: OnboardingModalProps) {
// Preferred default models for each provider
const preferredDefaults: Partial<Record<LlmProviderFlavor, string>> = {
openai: "gpt-5.2",
anthropic: "claude-opus-4-6-20260202",
openai: "gpt-5.4",
anthropic: "claude-opus-4-8",
}
// Initialize default models from catalog

View file

@ -155,8 +155,8 @@ export function useOnboardingState(open: boolean, onComplete: () => void) {
// Preferred default models for each provider
const preferredDefaults: Partial<Record<LlmProviderFlavor, string>> = {
openai: "gpt-5.2",
anthropic: "claude-opus-4-6-20260202",
openai: "gpt-5.4",
anthropic: "claude-opus-4-8",
}
// Initialize default models from catalog
@ -434,10 +434,29 @@ export function useOnboardingState(open: boolean, onComplete: () => void) {
}
const catalog: string[] = result.models ?? []
const typed = activeConfig.model.trim()
// Hosted providers hide the model field (it holds an auto-seeded
// default), so only treat it as user intent where the field is shown —
// mirrors showModelInput in llm-setup-step.
const hostedProviders: LlmProviderFlavor[] = ["openai", "anthropic", "google"]
const modelInputShown = !hostedProviders.includes(llmProvider)
if (modelInputShown && typed && llmProvider === "ollama" && catalog.length > 0 && !catalog.includes(typed)) {
// Ollama's tag list is authoritative: an unlisted model isn't pulled,
// so saving it would break chat at runtime with no obvious cause.
const error = `Model '${typed}' is not available on this Ollama server. Pull it first (ollama pull ${typed}) or pick one of: ${catalog.slice(0, 5).join(", ")}${catalog.length > 5 ? ", …" : ""}`
setTestState({ status: "error", error })
toast.error(error)
return false
}
const preferred = preferredDefaults[llmProvider]
const model =
(preferred && catalog.includes(preferred) && preferred) ||
catalog[0] || activeConfig.model.trim() || ""
// A model the user explicitly entered always wins — this used to prefer
// catalog[0], which silently replaced the user's Ollama model with
// whatever model the local server happened to list first.
const model = modelInputShown
? (typed || catalog[0] || "")
: ((preferred && catalog.includes(preferred) && preferred) || catalog[0] || typed || "")
// `models` is the user's curated assistant-model list (shown in Settings),
// NOT the full provider catalog. Onboarding seeds it with just the selected

View file

@ -0,0 +1,878 @@
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
import { createPortal } from 'react-dom'
import { X } from 'lucide-react'
import { Button } from '@/components/ui/button'
import { TalkingHead, type MascotHat } from '@/components/talking-head'
import { AgentsFleet, MascotVignette, type TourVignetteKind } from '@/components/tour-vignettes'
import { TourSounds } from '@/lib/tour-sounds'
import type { TTSState } from '@/hooks/useVoiceTTS'
import { cn } from '@/lib/utils'
import tourClipWelcome from '@/assets/tour/welcome.mp3'
import tourClipHome from '@/assets/tour/home.mp3'
import tourClipEmail from '@/assets/tour/email.mp3'
import tourClipMeetings from '@/assets/tour/meetings.mp3'
import tourClipCode from '@/assets/tour/code.mp3'
import tourClipKnowledge from '@/assets/tour/knowledge.mp3'
import tourClipAgents from '@/assets/tour/agents.mp3'
import tourClipWorkspaces from '@/assets/tour/workspaces.mp3'
import tourClipChats from '@/assets/tour/chats.mp3'
import tourClipComposer from '@/assets/tour/composer.mp3'
import tourClipDone from '@/assets/tour/done.mp3'
export type TourNavTarget =
| 'home'
| 'email'
| 'meetings'
| 'code'
| 'knowledge'
| 'agents'
| 'workspaces'
type TourStep = {
id: string
/** Matches a [data-tour-id] element. Steps whose target is absent are skipped. */
targetId?: string
/** View to open when the step starts, via App's navigation handlers. */
navigate?: TourNavTarget
/** Costume the mascot wears at this stop. */
hat?: MascotHat
/** Looping animation staged around the mascot while it presents this stop. */
vignette?: TourVignetteKind
title: string
text: string
/** Spoken narration, when it should differ from the bubble text. */
voiceText?: string
}
const TOUR_STEPS: TourStep[] = [
{
id: 'welcome',
title: 'All aboard! ⚓',
text: "I'm your captain for the next minute. The lights are down and the water's in — let me row you across Rowboat, one stop at a time. Use Next or your arrow keys.",
voiceText: "I'm your captain for the next minute. The lights are down and the water's in — let me row you across Rowboat, one stop at a time.",
},
{
id: 'home',
targetId: 'nav-home',
navigate: 'home',
title: 'First stop: Home',
text: 'Home is your landing spot — a quick overview of what needs your attention to get you back into the flow.',
},
{
id: 'email',
targetId: 'nav-email',
navigate: 'email',
hat: 'mailcap',
vignette: 'email',
title: 'Email',
text: 'Read and triage your inbox right here. Rowboat can summarize threads, label messages, and help you draft replies.',
},
{
id: 'meetings',
targetId: 'nav-meetings',
navigate: 'meetings',
hat: 'headphones',
vignette: 'meetings',
title: 'Meetings',
text: 'Record or join meetings, and get transcripts and notes automatically — prep briefs show up before your calls, too.',
},
{
id: 'code',
targetId: 'nav-code',
navigate: 'code',
hat: 'hardhat',
title: 'Code',
text: 'The Code section runs coding agents on your projects — point one at a folder and drive it from a chat.',
},
{
id: 'knowledge',
targetId: 'nav-knowledge',
navigate: 'knowledge',
hat: 'gradcap',
vignette: 'brain',
title: 'Brain',
text: "Brain is your knowledge base — notes, files, and everything Rowboat learns for you, all connected and searchable.",
},
{
id: 'agents',
targetId: 'nav-agents',
navigate: 'agents',
hat: 'captain',
vignette: 'agents',
title: 'Background agents',
text: 'Background agents work on schedules — they keep your Brain fresh and take care of recurring tasks while you row elsewhere.',
},
{
id: 'workspaces',
targetId: 'nav-workspaces',
navigate: 'workspaces',
hat: 'explorer',
title: 'Workspaces',
text: 'Workspaces hold your project folders and files, so related work stays docked together.',
},
{
id: 'chats',
targetId: 'nav-chats',
title: 'Chats',
text: 'Your recent conversations live here — pick any of them back up right where you left off.',
},
{
id: 'composer',
targetId: 'chat-composer',
title: 'Talk to Rowboat',
text: 'And this is where we talk! Type, dictate with the mic, or turn on voice output — tap my face button and Ill read replies out loud myself.',
},
{
id: 'done',
hat: 'party',
title: "Land ho! 🎉",
text: "That's the whole bay — and there's my wake to prove it. Take this voyage again anytime from the bottom of the sidebar. Happy rowing!",
},
]
// Pre-recorded narration bundled with the app (no TTS API call, works
// offline/signed-out). Regenerate with scripts/generate-tour-audio.mjs after
// editing any step's text. Steps without a clip fall back to live TTS.
const TOUR_CLIPS: Record<string, string> = {
welcome: tourClipWelcome,
home: tourClipHome,
email: tourClipEmail,
meetings: tourClipMeetings,
code: tourClipCode,
knowledge: tourClipKnowledge,
agents: tourClipAgents,
workspaces: tourClipWorkspaces,
chats: tourClipChats,
composer: tourClipComposer,
done: tourClipDone,
}
const MASCOT_SIZE = 120
const VIEWPORT_MARGIN = 16
const BUBBLE_WIDTH = 288
const TARGET_RESOLVE_TIMEOUT_MS = 1500
const ZOOM_SCALE = 1.05
const GLIDE_EASING = 'cubic-bezier(0.45, 0, 0.2, 1)'
type Pt = { x: number; y: number }
type Rect = { left: number; top: number; width: number; height: number }
type Spot = Rect & { round: boolean }
function clamp(v: number, min: number, max: number): number {
return Math.max(min, Math.min(max, v))
}
function easeInOutCubic(t: number): number {
return t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2
}
function quadPoint(p0: Pt, c: Pt, p1: Pt, t: number): Pt {
const u = 1 - t
return {
x: u * u * p0.x + 2 * u * t * c.x + t * t * p1.x,
y: u * u * p0.y + 2 * u * t * c.y + t * t * p1.y,
}
}
function quadPathLength(d: string): number {
const p = document.createElementNS('http://www.w3.org/2000/svg', 'path')
p.setAttribute('d', d)
return p.getTotalLength()
}
// A data-tour-id can legitimately appear on several elements (e.g. the chat
// composer renders in both the full-screen chat and the side pane) — pick the
// one that is actually laid out.
function findTourTarget(targetId: string): HTMLElement | null {
const nodes = document.querySelectorAll<HTMLElement>(`[data-tour-id="${targetId}"]`)
for (const el of nodes) {
const rect = el.getBoundingClientRect()
if (rect.width > 0 && rect.height > 0) return el
}
return null
}
function mascotDestForCenter(): Pt {
return {
x: window.innerWidth / 2 - MASCOT_SIZE / 2 - BUBBLE_WIDTH / 2,
y: window.innerHeight / 2 - MASCOT_SIZE / 2,
}
}
function mascotDestForRect(rect: Rect): Pt {
let x: number
let y: number
const fitsRight = rect.left + rect.width + MASCOT_SIZE + BUBBLE_WIDTH + VIEWPORT_MARGIN * 3 < window.innerWidth
if (fitsRight) {
x = rect.left + rect.width + 20
y = rect.top + rect.height / 2 - MASCOT_SIZE / 2
} else if (rect.top > MASCOT_SIZE + VIEWPORT_MARGIN * 2) {
x = rect.left + rect.width / 2 - MASCOT_SIZE / 2
y = rect.top - MASCOT_SIZE - 20
} else {
x = rect.left - MASCOT_SIZE - 20
y = rect.top + rect.height / 2 - MASCOT_SIZE / 2
}
return {
x: clamp(x, VIEWPORT_MARGIN, window.innerWidth - MASCOT_SIZE - VIEWPORT_MARGIN),
y: clamp(y, VIEWPORT_MARGIN, window.innerHeight - MASCOT_SIZE - VIEWPORT_MARGIN),
}
}
function spotForRect(rect: Rect): Spot {
return {
left: rect.left - 6,
top: rect.top - 6,
width: rect.width + 12,
height: rect.height + 12,
round: false,
}
}
function spotForMascot(dest: Pt): Spot {
return {
left: dest.x - 26,
top: dest.y - 18,
width: MASCOT_SIZE + 52,
height: MASCOT_SIZE + 44,
round: true,
}
}
type ProductTourProps = {
onClose: () => void
onNavigate: (target: TourNavTarget) => void
ttsAvailable: boolean
ttsState: TTSState
speak: (text: string) => void
speakUrl: (url: string) => void
cancelSpeech: () => void
getLevel: () => number
}
/**
* The Grand Voyage: a mascot-guided walkthrough where the app dims to a
* night-time bay, the boat rows curved routes between [data-tour-id] anchors
* (leaving a dotted wake behind it), a spotlight and gentle camera zoom reveal
* each section, and a parchment mini-map charts progress. Narrated with TTS
* lip sync when available; ends in confetti.
*
* Rendered through a portal to <body> so the camera zoom applied to the app
* shell never transforms the tour's own fixed-position layers.
*/
export function ProductTour({
onClose,
onNavigate,
ttsAvailable,
ttsState,
speak,
speakUrl,
cancelSpeech,
getLevel,
}: ProductTourProps) {
const [stepIndex, setStepIndex] = useState(0)
const [arrived, setArrived] = useState(false)
const [rowing, setRowing] = useState(false)
const [flipped, setFlipped] = useState(false)
const [bubbleSide, setBubbleSide] = useState<'left' | 'right'>('right')
const [spot, setSpot] = useState<Spot | null>(null)
const [wakes, setWakes] = useState<{ id: number; d: string }[]>([])
const [activeWake, setActiveWake] = useState<{ d: string; len: number } | null>(null)
const [confettiOn, setConfettiOn] = useState(false)
const [resizeNonce, setResizeNonce] = useState(0)
const reducedMotion = useMemo(
() => window.matchMedia('(prefers-reduced-motion: reduce)').matches,
[]
)
// Mascot position is animated by mutating the container's transform directly
// (60fps travel without re-rendering React); posRef is the source of truth.
const mascotElRef = useRef<HTMLDivElement>(null)
const posRef = useRef<Pt>({
x: window.innerWidth / 2 - MASCOT_SIZE / 2,
y: window.innerHeight + 60,
})
const travelRafRef = useRef(0)
const wakePathElRef = useRef<SVGPathElement>(null)
const wakeIdRef = useRef(0)
const curveSideRef = useRef(1)
const lastSplashRef = useRef(0)
const directionRef = useRef(1)
const enteredStepRef = useRef(-1)
const stepIndexRef = useRef(stepIndex)
// Camera zoom state applied to the app shell (outside the portal)
const shellRef = useRef<HTMLElement | null>(null)
const zoomRef = useRef<{ ox: number; oy: number; s: number } | null>(null)
const soundsRef = useRef<TourSounds | null>(null)
const onCloseRef = useRef(onClose)
const onNavigateRef = useRef(onNavigate)
const speakRef = useRef(speak)
const speakUrlRef = useRef(speakUrl)
const cancelSpeechRef = useRef(cancelSpeech)
const ttsAvailableRef = useRef(ttsAvailable)
// Keep latest callbacks/state in refs so the step effect and key handlers
// stay stable. Runs before the step effect below (effect order = call order).
useEffect(() => {
stepIndexRef.current = stepIndex
onCloseRef.current = onClose
onNavigateRef.current = onNavigate
speakRef.current = speak
speakUrlRef.current = speakUrl
cancelSpeechRef.current = cancelSpeech
ttsAvailableRef.current = ttsAvailable
})
useLayoutEffect(() => {
if (mascotElRef.current) {
mascotElRef.current.style.transform = `translate(${posRef.current.x}px, ${posRef.current.y}px)`
}
soundsRef.current = new TourSounds()
return () => {
soundsRef.current?.dispose()
soundsRef.current = null
}
}, [])
// Grab the app shell for the camera zoom; restore it when the tour ends
useEffect(() => {
const shell = document.querySelector<HTMLElement>('.rowboat-shell')
shellRef.current = shell
return () => {
if (shell) {
shell.style.transform = ''
shell.style.transformOrigin = ''
shell.style.transition = ''
}
shellRef.current = null
zoomRef.current = null
}
}, [])
const applyZoom = useCallback((origin: { ox: number; oy: number } | null) => {
const shell = shellRef.current
if (!shell || reducedMotion) return
if (origin) {
// transform-origin transitions too, so moving between targets pans
// smoothly instead of jumping when the origin changes
shell.style.transition = `transform 0.9s ${GLIDE_EASING}, transform-origin 0.9s ${GLIDE_EASING}`
shell.style.transformOrigin = `${origin.ox}px ${origin.oy}px`
shell.style.transform = `scale(${ZOOM_SCALE})`
zoomRef.current = { ox: origin.ox, oy: origin.oy, s: ZOOM_SCALE }
} else {
shell.style.transition = `transform 0.9s ${GLIDE_EASING}, transform-origin 0.9s ${GLIDE_EASING}`
shell.style.transform = 'scale(1)'
zoomRef.current = null
}
}, [reducedMotion])
// Where the element will sit on screen once this step's zoom settles:
// undo the current zoom mathematically, then apply the upcoming one (whose
// origin is the target's own center, so the center never moves).
const displayedRect = useCallback((el: HTMLElement, willZoom: boolean): { rect: Rect; origin: { ox: number; oy: number } } => {
const m = el.getBoundingClientRect()
const z = zoomRef.current
let cx = m.left + m.width / 2
let cy = m.top + m.height / 2
let w = m.width
let h = m.height
if (z) {
cx = z.ox + (cx - z.ox) / z.s
cy = z.oy + (cy - z.oy) / z.s
w /= z.s
h /= z.s
}
const s = willZoom && !reducedMotion ? ZOOM_SCALE : 1
return {
rect: { left: cx - (w * s) / 2, top: cy - (h * s) / 2, width: w * s, height: h * s },
origin: { ox: cx, oy: cy },
}
}, [reducedMotion])
const cancelTravel = useCallback(() => {
cancelAnimationFrame(travelRafRef.current)
}, [])
const moveMascot = useCallback((p: Pt) => {
posRef.current = p
if (mascotElRef.current) {
mascotElRef.current.style.transform = `translate(${p.x}px, ${p.y}px)`
}
}, [])
// Row along a curved path from the current position, drawing the wake as we
// go and splashing the oar; commits the wake as a dotted trail on arrival.
const startTravel = useCallback((dest: Pt, onArrive: () => void) => {
cancelTravel()
const from = { ...posRef.current }
const dist = Math.hypot(dest.x - from.x, dest.y - from.y)
if (dist < 6 || reducedMotion) {
moveMascot(dest)
setRowing(false)
onArrive()
return
}
const dur = clamp(dist * 1.1, 550, 1500)
curveSideRef.current = -curveSideRef.current
const mx = (from.x + dest.x) / 2
const my = (from.y + dest.y) / 2
const nx = -(dest.y - from.y) / dist
const ny = (dest.x - from.x) / dist
const mag = Math.min(140, dist * 0.3) * curveSideRef.current
const c = { x: mx + nx * mag, y: my + ny * mag }
// Wake follows the stern (bottom-center of the mascot box)
const sternX = MASCOT_SIZE / 2
const sternY = MASCOT_SIZE * 0.82
const d = `M ${from.x + sternX} ${from.y + sternY} Q ${c.x + sternX} ${c.y + sternY} ${dest.x + sternX} ${dest.y + sternY}`
const len = quadPathLength(d)
setActiveWake({ d, len })
setFlipped(dest.x < from.x - 4)
setRowing(true)
lastSplashRef.current = 0
const t0 = performance.now()
const frame = (now: number) => {
const raw = Math.min(1, (now - t0) / dur)
const t = easeInOutCubic(raw)
moveMascot(quadPoint(from, c, dest, t))
if (wakePathElRef.current) {
wakePathElRef.current.style.strokeDashoffset = String(len * (1 - t))
}
if (now - lastSplashRef.current > 420) {
lastSplashRef.current = now
soundsRef.current?.splash()
}
if (raw < 1) {
travelRafRef.current = requestAnimationFrame(frame)
} else {
setRowing(false)
setActiveWake(null)
setWakes((ws) => [...ws, { id: wakeIdRef.current++, d }])
onArrive()
}
}
travelRafRef.current = requestAnimationFrame(frame)
}, [cancelTravel, moveMascot, reducedMotion])
const playDing = useCallback(() => soundsRef.current?.ding(), [])
const finish = useCallback(() => {
cancelSpeechRef.current()
onCloseRef.current()
}, [])
const goTo = useCallback((index: number, direction: 1 | -1) => {
directionRef.current = direction
if (index < 0) return
if (index >= TOUR_STEPS.length) {
finish()
return
}
// Silence the current step's narration right away — not on arrival —
// so it can't talk over (or into) the next step's speech.
cancelSpeechRef.current()
setStepIndex(index)
}, [finish])
// Stop any in-flight narration when the tour unmounts
useEffect(() => () => cancelSpeechRef.current(), [])
useEffect(() => {
const handleResize = () => setResizeNonce((n) => n + 1)
window.addEventListener('resize', handleResize)
return () => window.removeEventListener('resize', handleResize)
}, [])
// Enter the current step: navigate, wait for its anchor, aim the spotlight
// and camera, row over, then narrate. Re-runs on resize to re-anchor.
useEffect(() => {
const step = TOUR_STEPS[stepIndex]
const entering = enteredStepRef.current !== stepIndex
let cancelled = false
if (entering && step.navigate) {
onNavigateRef.current(step.navigate)
}
const settle = (dest: Pt, side: 'left' | 'right', spotlight: Spot, origin: { ox: number; oy: number } | null) => {
applyZoom(origin)
setSpot(spotlight)
setBubbleSide(side)
if (!entering) {
// Resize while already at this step: jump, keep the bubble up
cancelTravel()
moveMascot(dest)
return
}
enteredStepRef.current = stepIndex
setArrived(false)
startTravel(dest, () => {
if (cancelled) return
setArrived(true)
soundsRef.current?.bump()
cancelSpeechRef.current()
const clip = TOUR_CLIPS[step.id]
if (clip) {
speakUrlRef.current(clip)
} else if (ttsAvailableRef.current) {
speakRef.current(step.voiceText ?? step.text)
}
if (stepIndex === TOUR_STEPS.length - 1) {
setConfettiOn(true)
soundsRef.current?.fanfare()
}
})
}
if (!step.targetId) {
const dest = mascotDestForCenter()
applyZoom(null)
settle(dest, 'right', spotForMascot(dest), null)
return () => {
cancelled = true
cancelTravel()
}
}
const startedAt = performance.now()
let pollRaf = 0
const attempt = () => {
if (cancelled) return
const el = findTourTarget(step.targetId!)
if (el) {
const { rect, origin } = displayedRect(el, true)
const dest = mascotDestForRect(rect)
const side: 'left' | 'right' = dest.x + MASCOT_SIZE / 2 < window.innerWidth / 2 ? 'right' : 'left'
settle(dest, side, spotForRect(rect), origin)
return
}
if (performance.now() - startedAt < TARGET_RESOLVE_TIMEOUT_MS) {
pollRaf = requestAnimationFrame(attempt)
} else if (entering) {
// Anchor never appeared (feature disabled / pane closed) — skip past it
goTo(stepIndex + directionRef.current, directionRef.current as 1 | -1)
}
}
attempt()
return () => {
cancelled = true
cancelAnimationFrame(pollRaf)
cancelTravel()
}
}, [stepIndex, resizeNonce, goTo, applyZoom, displayedRect, startTravel, cancelTravel, moveMascot])
useEffect(() => {
const handleKeyDown = (e: KeyboardEvent) => {
if (e.key === 'Escape') {
e.preventDefault()
finish()
} else if (e.key === 'ArrowRight' || e.key === 'Enter') {
e.preventDefault()
goTo(stepIndexRef.current + 1, 1)
} else if (e.key === 'ArrowLeft') {
e.preventDefault()
goTo(stepIndexRef.current - 1, -1)
}
}
window.addEventListener('keydown', handleKeyDown)
return () => window.removeEventListener('keydown', handleKeyDown)
}, [finish, goTo])
const step = TOUR_STEPS[stepIndex]
const isFirst = stepIndex === 0
const isLast = stepIndex === TOUR_STEPS.length - 1
return createPortal(
<>
<style>{`
@keyframes tour-bubble-in {
0% { opacity: 0; transform: translateY(6px) scale(0.97); }
100% { opacity: 1; transform: translateY(0) scale(1); }
}
@keyframes tour-wave-drift {
from { transform: translateX(0); }
to { transform: translateX(-50%); }
}
@keyframes tour-stamp-in {
0% { opacity: 0; transform: scale(2); }
100% { opacity: 1; transform: scale(1); }
}
`}</style>
{/* night falls: dim everything except the spotlight cutout */}
{spot && (
<div
className="pointer-events-none fixed z-[64]"
style={{
left: spot.left,
top: spot.top,
width: spot.width,
height: spot.height,
borderRadius: spot.round ? 9999 : 14,
boxShadow: '0 0 0 200vmax rgba(7, 14, 26, 0.52)',
transition: `all 0.9s ${GLIDE_EASING}`,
}}
/>
)}
<TourWater />
{/* wake trails: the committed dotted route + the wake being drawn now */}
<svg className="pointer-events-none fixed inset-0 z-[66] h-full w-full">
{wakes.map((w) => (
<path
key={w.id}
d={w.d}
fill="none"
stroke="#9CCBEA"
strokeWidth={4}
strokeLinecap="round"
strokeDasharray="1 11"
opacity={0.75}
/>
))}
{activeWake && (
<path
ref={wakePathElRef}
d={activeWake.d}
fill="none"
stroke="#BFE0F5"
strokeWidth={5}
strokeLinecap="round"
strokeDasharray={activeWake.len}
strokeDashoffset={activeWake.len}
opacity={0.8}
/>
)}
</svg>
<TourMiniMap total={TOUR_STEPS.length} current={stepIndex} arrived={arrived} />
{/* the boat (position driven imperatively during travel) */}
<div
ref={mascotElRef}
className="fixed left-0 top-0 z-[70]"
style={{ width: MASCOT_SIZE, pointerEvents: 'none' }}
>
{arrived && !reducedMotion && step.vignette && step.vignette !== 'agents' && (
<MascotVignette kind={step.vignette} playDing={playDing} />
)}
<div style={{ transform: flipped ? 'scaleX(-1)' : undefined, transition: 'transform 0.35s ease-in-out' }}>
<TalkingHead ttsState={ttsState} getLevel={getLevel} size={MASCOT_SIZE} hat={step.hat} rowing={rowing} />
</div>
{arrived && (
<div
key={step.id}
className={cn(
'pointer-events-auto absolute top-0 z-10 rounded-xl border border-border bg-popover p-4 text-popover-foreground shadow-lg',
bubbleSide === 'right' ? 'left-full ml-3' : 'right-full mr-3'
)}
style={{
width: BUBBLE_WIDTH,
animation: 'tour-bubble-in 0.25s ease-out',
}}
>
<button
type="button"
onClick={finish}
className="absolute right-2 top-2 flex h-6 w-6 items-center justify-center rounded-md text-muted-foreground transition-colors hover:bg-muted hover:text-foreground"
aria-label="End tour"
>
<X className="h-3.5 w-3.5" />
</button>
<p className="pr-6 text-sm font-semibold">{step.title}</p>
<p className="mt-1.5 text-sm text-muted-foreground">{step.text}</p>
<div className="mt-3 flex items-center justify-between">
<span className="text-xs tabular-nums text-muted-foreground">
{stepIndex + 1} / {TOUR_STEPS.length}
</span>
<div className="flex items-center gap-1.5">
{!isFirst && (
<Button variant="ghost" size="sm" className="h-7 px-2.5" onClick={() => goTo(stepIndex - 1, -1)}>
Back
</Button>
)}
<Button
size="sm"
className="h-7 px-3"
onClick={() => (isLast ? finish() : goTo(stepIndex + 1, 1))}
>
{isLast ? 'Done' : 'Next'}
</Button>
</div>
</div>
</div>
)}
</div>
{arrived && !reducedMotion && step.vignette === 'agents' && <AgentsFleet />}
{confettiOn && !reducedMotion && <ConfettiBurst />}
</>,
document.body
)
}
/** Animated translucent waves lapping at the bottom of the screen. */
function TourWater() {
const back = useMemo(() => wavePath(2400, 96, 8, 22), [])
const front = useMemo(() => wavePath(2400, 96, 12, 30), [])
return (
<div className="pointer-events-none fixed inset-x-0 bottom-0 z-[65] h-24 overflow-hidden" aria-hidden="true">
<svg
className="absolute bottom-0 left-0 h-full"
style={{ width: '200%', animation: 'tour-wave-drift 11s linear infinite' }}
viewBox="0 0 2400 96"
preserveAspectRatio="none"
>
<path d={back} fill="#5F9BC9" opacity={0.3} />
</svg>
<svg
className="absolute bottom-0 left-0 h-full"
style={{ width: '200%', animation: 'tour-wave-drift 7s linear infinite reverse' }}
viewBox="0 0 2400 96"
preserveAspectRatio="none"
>
<path d={front} fill="#8FB6D9" opacity={0.35} />
</svg>
</div>
)
}
// Periodic wave: alternating up/down humps so a -50% translate loops seamlessly
// (both hump counts are even, so half the width is a whole number of periods).
function wavePath(width: number, height: number, humps: number, amp: number): string {
const yTop = 34
const seg = width / humps
let d = `M 0 ${yTop}`
for (let i = 0; i < humps; i++) {
d += ` q ${seg / 2} ${i % 2 === 0 ? -amp : amp} ${seg} 0`
}
d += ` L ${width} ${height} L 0 ${height} Z`
return d
}
/** Parchment chart in the corner: islands per stop, dotted route, boat marker. */
function TourMiniMap({ total, current, arrived }: { total: number; current: number; arrived: boolean }) {
const MW = 184
const MH = 96
const points = useMemo(
() =>
Array.from({ length: total }, (_, i) => {
const t = total === 1 ? 0 : i / (total - 1)
return {
x: 14 + (MW - 28) * t,
y: MH / 2 + 4 + Math.sin(t * Math.PI * 1.5 + 0.6) * (MH * 0.26),
}
}),
[total]
)
const route = points.map((p, i) => (i === 0 ? `M ${p.x} ${p.y}` : `L ${p.x} ${p.y}`)).join(' ')
const boat = points[clamp(current, 0, total - 1)]
return (
<div
className="fixed bottom-5 left-5 z-[71] rounded-lg border-2 border-[#8A6B3D]/60 bg-[#F4E9CE] px-2 pb-1.5 pt-2 shadow-xl"
style={{ transform: 'rotate(-1.2deg)' }}
aria-hidden="true"
>
<p className="mb-0.5 px-1 text-[9px] font-semibold uppercase tracking-[0.22em] text-[#6B5138]">
Voyage chart
</p>
<svg width={MW} height={MH} viewBox={`0 0 ${MW} ${MH}`}>
<path d={route} fill="none" stroke="#8A6B3D" strokeWidth={1.5} strokeDasharray="3 4" opacity={0.65} />
{points.map((p, i) => {
const visited = i < current || (i === current && arrived)
return (
<g key={i}>
<ellipse cx={p.x} cy={p.y} rx={7} ry={5} fill="#DFC896" stroke="#8A6B3D" strokeWidth={1.5} />
{visited && (
<g style={{ animation: 'tour-stamp-in 0.3s ease-out backwards' }}>
<line x1={p.x - 1} y1={p.y - 13} x2={p.x - 1} y2={p.y - 3} stroke="#7A4A21" strokeWidth={1.5} />
<path d={`M ${p.x - 1} ${p.y - 13} L ${p.x + 6} ${p.y - 10.5} L ${p.x - 1} ${p.y - 8} Z`} fill="#D9534F" />
</g>
)}
</g>
)
})}
{/* boat marker glides between islands in step with the real mascot */}
<g style={{ transform: `translate(${boat.x}px, ${boat.y - 7}px)`, transition: `transform 0.9s ${GLIDE_EASING}` }}>
<path d="M -7 0 Q 0 4 7 0 Q 4 6 0 6 Q -4 6 -7 0 Z" fill="#54402F" stroke="#3E2E24" strokeWidth={1} />
<circle cx={0} cy={-3} r={3} fill="#E8E9F5" stroke="#17171B" strokeWidth={1} />
</g>
</svg>
</div>
)
}
/** Two confetti cannons firing from the bottom corners, canvas-driven. */
function ConfettiBurst() {
const canvasRef = useRef<HTMLCanvasElement>(null)
useEffect(() => {
const canvas = canvasRef.current
const ctx = canvas?.getContext('2d')
if (!canvas || !ctx) return
const w = window.innerWidth
const h = window.innerHeight
const dpr = window.devicePixelRatio || 1
canvas.width = w * dpr
canvas.height = h * dpr
ctx.scale(dpr, dpr)
const colors = ['#5B8DEF', '#F2B8BE', '#FFD166', '#7AC74F', '#8FB6D9', '#F2699C']
const parts = Array.from({ length: 150 }, (_, i) => {
const fromLeft = i % 2 === 0
return {
x: fromLeft ? 24 : w - 24,
y: h - 40,
vx: (fromLeft ? 1 : -1) * (2.5 + Math.random() * 6.5),
vy: -(9 + Math.random() * 8),
size: 5 + Math.random() * 5,
color: colors[i % colors.length],
rot: Math.random() * Math.PI,
vr: (Math.random() - 0.5) * 0.3,
}
})
let raf = 0
const t0 = performance.now()
const frame = (now: number) => {
ctx.clearRect(0, 0, w, h)
for (const p of parts) {
p.vy += 0.18
p.vx *= 0.99
p.x += p.vx
p.y += p.vy
p.rot += p.vr
ctx.save()
ctx.translate(p.x, p.y)
ctx.rotate(p.rot)
ctx.fillStyle = p.color
ctx.fillRect(-p.size / 2, -p.size / 3, p.size, p.size * 0.66)
ctx.restore()
}
if (now - t0 < 3200) {
raf = requestAnimationFrame(frame)
} else {
ctx.clearRect(0, 0, w, h)
}
}
raf = requestAnimationFrame(frame)
return () => cancelAnimationFrame(raf)
}, [])
return (
<canvas
ref={canvasRef}
className="pointer-events-none fixed inset-0 z-[72]"
style={{ width: '100vw', height: '100vh' }}
/>
)
}

View file

@ -2,7 +2,7 @@
import * as React from "react"
import { useState, useEffect, useCallback, useMemo } from "react"
import { Server, Key, Shield, Palette, Monitor, Sun, Moon, Loader2, CheckCircle2, Plus, X, Wrench, Search, ChevronRight, Link2, Tags, Mail, BookOpen, User, Plug, HelpCircle, MessageCircle, Bug, Terminal, AlertTriangle, RefreshCw, PanelRight, Bell } from "lucide-react"
import { Server, Key, Shield, Palette, Monitor, Sun, Moon, Loader2, CheckCircle2, Plus, X, Wrench, Search, ChevronRight, Link2, Tags, Mail, BookOpen, User, Plug, HelpCircle, MessageCircle, Bug, Terminal, AlertTriangle, RefreshCw, PanelRight, Bell, Smartphone } from "lucide-react"
import {
Dialog,
@ -25,10 +25,11 @@ import { useTheme } from "@/contexts/theme-context"
import { toast } from "sonner"
import { AccountSettings } from "@/components/settings/account-settings"
import { ConnectedAccountsSettings } from "@/components/settings/connected-accounts-settings"
import { MobileChannelsSettings } from "@/components/settings/mobile-channels-settings"
import type { ApprovalPolicy } from "@x/shared/src/code-mode.js"
import { startProvisioning, useProvisioning, enabledOptimistic, type AgentStatus, type CodeModeAgentStatus } from "@/lib/code-mode-provisioning"
type ConfigTab = "account" | "connections" | "models" | "mcp" | "security" | "code-mode" | "appearance" | "notifications" | "note-tagging" | "help"
type ConfigTab = "account" | "connections" | "mobile" | "models" | "mcp" | "security" | "code-mode" | "appearance" | "notifications" | "note-tagging" | "help"
interface TabConfig {
id: ConfigTab
@ -51,6 +52,12 @@ const tabs: TabConfig[] = [
icon: Plug,
description: "Manage accounts and tools",
},
{
id: "mobile",
label: "Mobile",
icon: Smartphone,
description: "Chat with Rowboat from WhatsApp or Telegram",
},
{
id: "models",
label: "Models",
@ -320,8 +327,8 @@ const moreProviders: Array<{ id: LlmProviderFlavor; name: string; description: s
]
const preferredDefaults: Partial<Record<LlmProviderFlavor, string>> = {
openai: "gpt-5.2",
anthropic: "claude-opus-4-6-20260202",
openai: "gpt-5.4",
anthropic: "claude-opus-4-8",
}
const defaultBaseURLs: Partial<Record<LlmProviderFlavor, string>> = {
@ -339,7 +346,7 @@ type ProviderModelConfig = {
autoPermissionDecisionModel: string
}
function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
function ModelSettings({ dialogOpen, rowboatConnected = false }: { dialogOpen: boolean; rowboatConnected?: boolean }) {
const [provider, setProvider] = useState<LlmProviderFlavor>("openai")
const [defaultProvider, setDefaultProvider] = useState<LlmProviderFlavor | null>(null)
const [providerConfigs, setProviderConfigs] = useState<Record<LlmProviderFlavor, ProviderModelConfig>>({
@ -357,6 +364,11 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
const [testState, setTestState] = useState<{ status: "idle" | "testing" | "success" | "error"; error?: string }>({ status: "idle" })
const [configLoading, setConfigLoading] = useState(true)
const [showMoreProviders, setShowMoreProviders] = useState(false)
// "Defer background tasks while a chat is running" — a top-level
// models.json flag. deferExplicit tracks whether the user (or the Ollama
// auto-enable) has ever set it, so we only auto-enable once.
const [deferBackgroundTasks, setDeferBackgroundTasks] = useState(false)
const [deferExplicit, setDeferExplicit] = useState(false)
const activeConfig = providerConfigs[provider]
const showApiKey = provider === "openai" || provider === "anthropic" || provider === "google" || provider === "openrouter" || provider === "aigateway" || provider === "openai-compatible"
@ -390,6 +402,8 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
useEffect(() => {
if (!dialogOpen) return
const asString = (v: unknown): string => (typeof v === "string" ? v : "")
async function loadCurrentConfig() {
try {
setConfigLoading(true)
@ -397,6 +411,8 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
path: "config/models.json",
})
const parsed = JSON.parse(result.data)
setDeferBackgroundTasks(parsed?.deferBackgroundTasks === true)
setDeferExplicit(typeof parsed?.deferBackgroundTasks === "boolean")
if (parsed?.provider?.flavor && parsed?.model) {
const flavor = parsed.provider.flavor as LlmProviderFlavor
setProvider(flavor)
@ -415,10 +431,10 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
apiKey: e.apiKey || "",
baseURL: e.baseURL || (defaultBaseURLs[key as LlmProviderFlavor] || ""),
models: savedModels,
knowledgeGraphModel: e.knowledgeGraphModel || "",
meetingNotesModel: e.meetingNotesModel || "",
liveNoteAgentModel: e.liveNoteAgentModel || "",
autoPermissionDecisionModel: e.autoPermissionDecisionModel || "",
knowledgeGraphModel: asString(e.knowledgeGraphModel),
meetingNotesModel: asString(e.meetingNotesModel),
liveNoteAgentModel: asString(e.liveNoteAgentModel),
autoPermissionDecisionModel: asString(e.autoPermissionDecisionModel),
};
}
}
@ -434,10 +450,10 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
apiKey: parsed.provider.apiKey || "",
baseURL: parsed.provider.baseURL || (defaultBaseURLs[flavor] || ""),
models: activeModels.length > 0 ? activeModels : [""],
knowledgeGraphModel: parsed.knowledgeGraphModel || "",
meetingNotesModel: parsed.meetingNotesModel || "",
liveNoteAgentModel: parsed.liveNoteAgentModel || "",
autoPermissionDecisionModel: parsed.autoPermissionDecisionModel || "",
knowledgeGraphModel: asString(parsed.knowledgeGraphModel),
meetingNotesModel: asString(parsed.meetingNotesModel),
liveNoteAgentModel: asString(parsed.liveNoteAgentModel),
autoPermissionDecisionModel: asString(parsed.autoPermissionDecisionModel),
};
}
return next;
@ -453,6 +469,17 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
loadCurrentConfig()
}, [dialogOpen])
const handleDeferToggle = useCallback(async (value: boolean) => {
setDeferBackgroundTasks(value)
setDeferExplicit(true)
try {
await window.ipc.invoke("models:updateConfig", { deferBackgroundTasks: value })
window.dispatchEvent(new Event("models-config-changed"))
} catch {
toast.error("Failed to save setting")
}
}, [])
// Load models catalog
useEffect(() => {
if (!dialogOpen) return
@ -510,10 +537,12 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
},
model: allModels[0] || "",
models: allModels,
knowledgeGraphModel: activeConfig.knowledgeGraphModel.trim() || undefined,
meetingNotesModel: activeConfig.meetingNotesModel.trim() || undefined,
liveNoteAgentModel: activeConfig.liveNoteAgentModel.trim() || undefined,
autoPermissionDecisionModel: activeConfig.autoPermissionDecisionModel.trim() || undefined,
...(rowboatConnected ? {} : {
knowledgeGraphModel: activeConfig.knowledgeGraphModel.trim() || undefined,
meetingNotesModel: activeConfig.meetingNotesModel.trim() || undefined,
liveNoteAgentModel: activeConfig.liveNoteAgentModel.trim() || undefined,
autoPermissionDecisionModel: activeConfig.autoPermissionDecisionModel.trim() || undefined,
}),
}
const result = await window.ipc.invoke("models:test", providerConfig)
if (result.success) {
@ -521,7 +550,23 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
setDefaultProvider(provider)
setTestState({ status: "success" })
window.dispatchEvent(new Event('models-config-changed'))
toast.success("Model configuration saved")
// Local models compete with background agents for the same hardware:
// when the user connects Ollama and has never touched the defer
// flag, enable it for them (they can switch it off below).
if (provider === "ollama" && !deferExplicit && !deferBackgroundTasks) {
void handleDeferToggle(true)
}
// Capability probe caveats (local models): saved, but the user should
// know when the model can't do tools or has a too-small context.
const warnings: string[] = result.warnings ?? []
if (warnings.length > 0) {
for (const warning of warnings) {
toast.warning(warning, { duration: 12000 })
}
toast.success("Model configuration saved (with warnings)")
} else {
toast.success("Model configuration saved")
}
} else {
setTestState({ status: "error", error: result.error })
toast.error(result.error || "Connection test failed")
@ -530,7 +575,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
setTestState({ status: "error", error: "Connection test failed" })
toast.error("Connection test failed")
}
}, [canTest, provider, activeConfig])
}, [canTest, provider, activeConfig, rowboatConnected, deferExplicit, deferBackgroundTasks, handleDeferToggle])
const handleSetDefault = useCallback(async (prov: LlmProviderFlavor) => {
const config = providerConfigs[prov]
@ -545,10 +590,12 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
},
model: allModels[0],
models: allModels,
knowledgeGraphModel: config.knowledgeGraphModel.trim() || undefined,
meetingNotesModel: config.meetingNotesModel.trim() || undefined,
liveNoteAgentModel: config.liveNoteAgentModel.trim() || undefined,
autoPermissionDecisionModel: config.autoPermissionDecisionModel.trim() || undefined,
...(rowboatConnected ? {} : {
knowledgeGraphModel: config.knowledgeGraphModel.trim() || undefined,
meetingNotesModel: config.meetingNotesModel.trim() || undefined,
liveNoteAgentModel: config.liveNoteAgentModel.trim() || undefined,
autoPermissionDecisionModel: config.autoPermissionDecisionModel.trim() || undefined,
}),
})
setDefaultProvider(prov)
window.dispatchEvent(new Event('models-config-changed'))
@ -556,7 +603,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
} catch {
toast.error("Failed to set default provider")
}
}, [providerConfigs])
}, [providerConfigs, rowboatConnected])
const handleDeleteProvider = useCallback(async (prov: LlmProviderFlavor) => {
try {
@ -577,10 +624,12 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
}
parsed.model = defModels[0] || ""
parsed.models = defModels
parsed.knowledgeGraphModel = defConfig.knowledgeGraphModel.trim() || undefined
parsed.meetingNotesModel = defConfig.meetingNotesModel.trim() || undefined
parsed.liveNoteAgentModel = defConfig.liveNoteAgentModel.trim() || undefined
parsed.autoPermissionDecisionModel = defConfig.autoPermissionDecisionModel.trim() || undefined
if (!rowboatConnected) {
parsed.knowledgeGraphModel = defConfig.knowledgeGraphModel.trim() || undefined
parsed.meetingNotesModel = defConfig.meetingNotesModel.trim() || undefined
parsed.liveNoteAgentModel = defConfig.liveNoteAgentModel.trim() || undefined
parsed.autoPermissionDecisionModel = defConfig.autoPermissionDecisionModel.trim() || undefined
}
}
await window.ipc.invoke("workspace:writeFile", {
path: "config/models.json",
@ -596,7 +645,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
} catch {
toast.error("Failed to remove provider")
}
}, [defaultProvider, providerConfigs])
}, [defaultProvider, providerConfigs, rowboatConnected])
const renderProviderCard = (p: { id: LlmProviderFlavor; name: string; description: string }) => {
const isDefault = defaultProvider === p.id
@ -618,7 +667,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
>
<div className="flex items-center gap-1.5">
<span className="text-sm font-medium">{p.name}</span>
{isDefault && (
{isDefault && !rowboatConnected && (
<span className="rounded-full bg-primary/10 px-1.5 py-0.5 text-[10px] font-medium leading-none text-primary">
Default
</span>
@ -627,16 +676,18 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
<div className="text-xs text-muted-foreground mt-0.5">{p.description}</div>
{!isDefault && hasModel && isSelected && (
<div className="mt-1.5 flex items-center gap-3">
<span
role="button"
onClick={(e) => {
e.stopPropagation()
handleSetDefault(p.id)
}}
className="inline-flex text-[11px] text-muted-foreground hover:text-primary transition-colors cursor-pointer"
>
Set as default
</span>
{!rowboatConnected && (
<span
role="button"
onClick={(e) => {
e.stopPropagation()
handleSetDefault(p.id)
}}
className="inline-flex text-[11px] text-muted-foreground hover:text-primary transition-colors cursor-pointer"
>
Set as default
</span>
)}
<span
role="button"
onClick={(e) => {
@ -688,7 +739,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
<div className="grid grid-cols-2 gap-3">
{/* Assistant models (left column) */}
<div className="space-y-2">
<span className="text-xs font-medium text-muted-foreground uppercase tracking-wider">Assistant model</span>
<span className="text-xs font-medium text-muted-foreground uppercase tracking-wider">{rowboatConnected ? "Model" : "Assistant model"}</span>
{modelsLoading ? (
<div className="flex items-center gap-2 text-sm text-muted-foreground">
<Loader2 className="size-4 animate-spin" />
@ -726,6 +777,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
)}
</div>
{!rowboatConnected && (<>
{/* Knowledge graph model (right column) */}
<div className="space-y-2">
<span className="text-xs font-medium text-muted-foreground uppercase tracking-wider">Knowledge graph model</span>
@ -861,6 +913,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
</Select>
)}
</div>
</>)}
</div>
{/* API Key */}
@ -909,6 +962,17 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
</div>
)}
{/* Defer background tasks while chatting */}
<div className="flex items-center justify-between gap-4 rounded-md border px-3 py-2.5">
<div className="min-w-0">
<div className="text-sm font-medium">Defer background tasks while chatting</div>
<div className="text-xs text-muted-foreground mt-0.5">
Background agents (knowledge sync, live notes, tasks) wait until your chat finishes. Recommended for local models.
</div>
</div>
<Switch checked={deferBackgroundTasks} onCheckedChange={handleDeferToggle} />
</div>
{/* Test & Save button */}
<Button
onClick={handleTestAndSave}
@ -1245,11 +1309,45 @@ function ToolsLibrarySettings({ dialogOpen, rowboatConnected }: { dialogOpen: bo
}
// --- Rowboat Model Settings (when signed in via Rowboat) ---
//
// Hybrid mode: every dropdown lists the gateway catalog PLUS any models from
// BYOK providers configured below. Values are provider-qualified
// ("provider::model") and saved via models:updateConfig as {provider, model}
// refs, so a signed-in user can e.g. keep the gateway assistant while
// running background agents on a local Ollama model.
interface HybridModelOption {
provider: string
model: string
label: string
}
const providerDisplayNames: Record<string, string> = {
openai: 'OpenAI',
anthropic: 'Anthropic',
google: 'Gemini',
ollama: 'Ollama',
openrouter: 'OpenRouter',
aigateway: 'AI Gateway',
'openai-compatible': 'OpenAI-Compatible',
rowboat: 'Rowboat',
}
const HYBRID_SEP = "::"
const hybridKey = (provider: string, model: string) => `${provider}${HYBRID_SEP}${model}`
function parseHybridKey(key: string): { provider: string; model: string } | null {
const index = key.indexOf(HYBRID_SEP)
if (index <= 0) return null
return { provider: key.slice(0, index), model: key.slice(index + HYBRID_SEP.length) }
}
function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
const [gatewayModels, setGatewayModels] = useState<LlmModelOption[]>([])
const [selectedModel, setSelectedModel] = useState("")
const [selectedKgModel, setSelectedKgModel] = useState("")
const [options, setOptions] = useState<HybridModelOption[]>([])
const [selectedDefault, setSelectedDefault] = useState("")
const [selectedKg, setSelectedKg] = useState("")
const [selectedLiveNote, setSelectedLiveNote] = useState("")
const [selectedAutoPermission, setSelectedAutoPermission] = useState("")
const [loading, setLoading] = useState(true)
const [saving, setSaving] = useState(false)
@ -1259,22 +1357,63 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
async function load() {
setLoading(true)
try {
// Fetch gateway models
const listResult = await window.ipc.invoke("models:list", null)
const rowboatProvider = listResult.providers?.find((p: { id: string }) => p.id === "rowboat")
const models = rowboatProvider?.models || []
setGatewayModels(models)
const collected: HybridModelOption[] = []
const seen = new Set<string>()
const push = (provider: string, model: string, label?: string) => {
if (!model) return
const key = hybridKey(provider, model)
if (seen.has(key)) return
seen.add(key)
collected.push({ provider, model, label: label || model })
}
// Read current selection from config
const catalog: Record<string, LlmModelOption[]> = {}
try {
const listResult = await window.ipc.invoke("models:list", null)
for (const p of listResult.providers || []) {
catalog[p.id] = p.models || []
}
} catch { /* offline — BYOK entries below still load */ }
for (const m of catalog["rowboat"] || []) push("rowboat", m.id, m.name || m.id)
let parsed: Record<string, unknown> = {}
try {
const configResult = await window.ipc.invoke("workspace:readFile", { path: "config/models.json" })
const parsed = JSON.parse(configResult.data)
if (parsed?.model) setSelectedModel(parsed.model)
if (parsed?.knowledgeGraphModel) setSelectedKgModel(parsed.knowledgeGraphModel)
} catch {
// No config yet — pick first model as default
if (models.length > 0) setSelectedModel(models[0].id)
parsed = JSON.parse(configResult.data)
} catch { /* no BYOK config yet */ }
const providersMap = (parsed.providers ?? {}) as Record<string, Record<string, unknown>>
for (const [flavor, entry] of Object.entries(providersMap)) {
const hasKey = typeof entry.apiKey === "string" && (entry.apiKey as string).trim().length > 0
const hasBaseURL = typeof entry.baseURL === "string" && (entry.baseURL as string).trim().length > 0
if (!hasKey && !hasBaseURL) continue
push(flavor, typeof entry.model === "string" ? entry.model : "")
const catalogModels = catalog[flavor] || []
if (catalogModels.length > 0) {
for (const m of catalogModels) push(flavor, m.id, m.name || m.id)
} else {
for (const m of Array.isArray(entry.models) ? entry.models as string[] : []) push(flavor, m)
}
}
setOptions(collected)
// Current selections. Legacy string overrides pair with the BYOK
// top-level flavor (mirrors core/models/defaults.ts).
const legacyFlavor = (parsed.provider as Record<string, unknown> | undefined)?.flavor
const toKey = (value: unknown): string => {
if (!value) return ""
if (typeof value === "string") {
return typeof legacyFlavor === "string" ? hybridKey(legacyFlavor, value) : ""
}
const ref = value as { provider?: unknown; model?: unknown }
return typeof ref.provider === "string" && typeof ref.model === "string"
? hybridKey(ref.provider, ref.model)
: ""
}
setSelectedDefault(toKey(parsed.defaultSelection))
setSelectedKg(toKey(parsed.knowledgeGraphModel))
setSelectedLiveNote(toKey(parsed.liveNoteAgentModel))
setSelectedAutoPermission(toKey(parsed.autoPermissionDecisionModel))
} catch {
toast.error("Failed to load models")
} finally {
@ -1286,13 +1425,14 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
}, [dialogOpen])
const handleSave = useCallback(async () => {
if (!selectedModel) return
setSaving(true)
try {
await window.ipc.invoke("models:saveConfig", {
provider: { flavor: "openrouter" as const },
model: selectedModel,
knowledgeGraphModel: selectedKgModel || undefined,
const toRef = (key: string) => (key ? parseHybridKey(key) : null)
await window.ipc.invoke("models:updateConfig", {
defaultSelection: toRef(selectedDefault),
knowledgeGraphModel: toRef(selectedKg),
liveNoteAgentModel: toRef(selectedLiveNote),
autoPermissionDecisionModel: toRef(selectedAutoPermission),
})
window.dispatchEvent(new Event("models-config-changed"))
toast.success("Model configuration saved")
@ -1301,7 +1441,37 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
} finally {
setSaving(false)
}
}, [selectedModel, selectedKgModel])
}, [selectedDefault, selectedKg, selectedLiveNote, selectedAutoPermission])
const renderSelect = (
label: string,
value: string,
onChange: (v: string) => void,
defaultLabel: string,
) => (
<div className="space-y-2">
<label className="text-sm font-medium">{label}</label>
<Select value={value || "__default__"} onValueChange={(v) => onChange(v === "__default__" ? "" : v)}>
<SelectTrigger className="w-full">
<SelectValue placeholder={defaultLabel} />
</SelectTrigger>
<SelectContent>
<SelectItem value="__default__">{defaultLabel}</SelectItem>
{options.map((o) => {
const key = hybridKey(o.provider, o.model)
return (
<SelectItem key={key} value={key}>
{o.label}
<span className="ml-2 text-xs text-muted-foreground">
{providerDisplayNames[o.provider] || o.provider}
</span>
</SelectItem>
)
})}
</SelectContent>
</Select>
</div>
)
if (loading) {
return (
@ -1314,46 +1484,16 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
return (
<div className="space-y-6">
<p className="text-sm text-muted-foreground">
Select the models Rowboat uses. These are provided through your Rowboat account.
Select the models Rowboat uses. Rowboat models are provided through your account; models from your own providers route through your keys or local runtimes.
</p>
{/* Assistant model */}
<div className="space-y-2">
<label className="text-sm font-medium">Assistant model</label>
<Select value={selectedModel} onValueChange={setSelectedModel}>
<SelectTrigger className="w-full">
<SelectValue placeholder="Select a model" />
</SelectTrigger>
<SelectContent>
{gatewayModels.map((m) => (
<SelectItem key={m.id} value={m.id}>
{m.name || m.id}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
{/* Knowledge graph model */}
<div className="space-y-2">
<label className="text-sm font-medium">Knowledge graph model</label>
<Select value={selectedKgModel || "__same__"} onValueChange={(v) => setSelectedKgModel(v === "__same__" ? "" : v)}>
<SelectTrigger className="w-full">
<SelectValue placeholder="Same as assistant" />
</SelectTrigger>
<SelectContent>
<SelectItem value="__same__">Same as assistant</SelectItem>
{gatewayModels.map((m) => (
<SelectItem key={m.id} value={m.id}>
{m.name || m.id}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
{renderSelect("Assistant model", selectedDefault, setSelectedDefault, "Rowboat default")}
{renderSelect("Knowledge graph model", selectedKg, setSelectedKg, "Rowboat default")}
{renderSelect("Background agents model", selectedLiveNote, setSelectedLiveNote, "Rowboat default")}
{renderSelect("Permission checks model", selectedAutoPermission, setSelectedAutoPermission, "Rowboat default")}
{/* Save */}
<Button onClick={handleSave} disabled={!selectedModel || saving}>
<Button onClick={handleSave} disabled={saving}>
{saving ? (
<><Loader2 className="size-4 animate-spin mr-2" />Saving...</>
) : (
@ -2094,7 +2234,9 @@ export function SettingsDialog({ children, defaultTab = "account", open: control
})
}, [open])
const visibleTabs = useMemo(() => rowboatConnected ? tabs.filter(t => t.id !== "models") : tabs, [rowboatConnected])
// Hybrid mode: the Models tab is shown in both modes — signed-in users can
// pick gateway models AND bring their own providers/models alongside.
const visibleTabs = tabs
const activeTabConfig = visibleTabs.find((t) => t.id === activeTab) ?? visibleTabs[0]
const isJsonTab = activeTab === "mcp" || activeTab === "security"
@ -2216,7 +2358,7 @@ export function SettingsDialog({ children, defaultTab = "account", open: control
</div>
{/* Content */}
<div className={cn("flex-1 p-4 min-h-0", (activeTab === "models" || activeTab === "connections" || activeTab === "account" || activeTab === "code-mode" || activeTab === "notifications") ? "overflow-y-auto" : activeTab === "note-tagging" ? "overflow-hidden flex flex-col" : "overflow-hidden")}>
<div className={cn("flex-1 p-4 min-h-0", (activeTab === "models" || activeTab === "connections" || activeTab === "mobile" || activeTab === "account" || activeTab === "code-mode" || activeTab === "notifications") ? "overflow-y-auto" : activeTab === "note-tagging" ? "overflow-hidden flex flex-col" : "overflow-hidden")}>
{activeTab === "account" ? (
<AccountSettings dialogOpen={open} />
) : activeTab === "connections" ? (
@ -2231,9 +2373,23 @@ export function SettingsDialog({ children, defaultTab = "account", open: control
<ToolsLibrarySettings dialogOpen={open} rowboatConnected={rowboatConnected} />
</div>
</div>
) : activeTab === "mobile" ? (
<MobileChannelsSettings dialogOpen={open} />
) : activeTab === "models" ? (
rowboatConnected
? <RowboatModelSettings dialogOpen={open} />
? (
<div className="space-y-8">
<RowboatModelSettings dialogOpen={open} />
<Separator />
<div className="space-y-2">
<h4 className="text-sm font-semibold">Your own providers</h4>
<p className="text-xs text-muted-foreground">
Connect your own API keys or local runtimes (Ollama, LM Studio). Their models appear in the model pickers above and alongside your Rowboat models, and are billed to you directly.
</p>
<ModelSettings dialogOpen={open} rowboatConnected />
</div>
</div>
)
: <ModelSettings dialogOpen={open} />
) : activeTab === "note-tagging" ? (
<NoteTaggingSettings dialogOpen={open} />

View file

@ -0,0 +1,287 @@
"use client"
import { useCallback, useEffect, useState } from "react"
import type { z } from "zod"
import { Loader2, MessageCircle, Send, Smartphone } from "lucide-react"
import { Button } from "@/components/ui/button"
import { Input } from "@/components/ui/input"
import { Separator } from "@/components/ui/separator"
import { Switch } from "@/components/ui/switch"
import { toast } from "sonner"
import type { ChannelsConfig, ChannelsStatus } from "@x/shared/src/channels.js"
type Config = z.infer<typeof ChannelsConfig>
type Status = z.infer<typeof ChannelsStatus>
// Comma/newline separated entries; each entry keeps digits only, so a
// formatted number like "+1 (415) 555-1234" survives as one entry instead of
// being shattered at the spaces.
function parseIdList(draft: string): string[] {
return draft
.split(/[,;\n]+/)
.map((s) => s.replace(/\D/g, ""))
.filter(Boolean)
}
export function MobileChannelsSettings({ dialogOpen }: { dialogOpen: boolean }) {
const [config, setConfig] = useState<Config | null>(null)
const [status, setStatus] = useState<Status | null>(null)
const [tokenDraft, setTokenDraft] = useState("")
const [waAllowDraft, setWaAllowDraft] = useState("")
const [tgAllowDraft, setTgAllowDraft] = useState("")
const [saving, setSaving] = useState(false)
useEffect(() => {
if (!dialogOpen) return
let cancelled = false
void (async () => {
try {
const [cfg, st] = await Promise.all([
window.ipc.invoke("channels:getConfig", null),
window.ipc.invoke("channels:getStatus", null),
])
if (cancelled) return
setConfig(cfg)
setStatus(st)
setTokenDraft(cfg.telegram.botToken)
setWaAllowDraft(cfg.whatsapp.allowFrom.join(", "))
setTgAllowDraft(cfg.telegram.allowFrom.join(", "))
} catch {
if (!cancelled) toast.error("Failed to load mobile channel settings")
}
})()
const unsubscribe = window.ipc.on("channels:status", (st) => {
setStatus(st)
})
return () => {
cancelled = true
unsubscribe()
}
}, [dialogOpen])
const save = useCallback(async (next: Config) => {
setConfig(next)
setSaving(true)
try {
await window.ipc.invoke("channels:setConfig", next)
} catch (error) {
toast.error(error instanceof Error ? error.message : "Failed to save channel settings")
} finally {
setSaving(false)
}
}, [])
if (!config) {
return (
<div className="flex items-center justify-center py-8 text-muted-foreground">
<Loader2 className="size-4 animate-spin" />
</div>
)
}
const wa = status?.whatsapp
const tg = status?.telegram
return (
<div className="space-y-6">
<div className="flex items-start gap-2.5 rounded-md bg-muted/50 px-3 py-2.5">
<Smartphone className="size-4 mt-0.5 shrink-0 text-muted-foreground" />
<p className="text-xs text-muted-foreground">
Chat with Rowboat from your phone. Send <span className="font-mono">help</span> for
commands (<span className="font-mono">list</span>, <span className="font-mono">resume 2</span>,{" "}
<span className="font-mono">new</span>, <span className="font-mono">stop</span>) anything
else is a message to the current chat. Your computer must be on with Rowboat running.
</p>
</div>
{/* WhatsApp */}
<div className="space-y-3">
<div className="flex items-center justify-between">
<div className="flex items-center gap-2.5">
<div className="flex size-8 items-center justify-center rounded-md bg-muted">
<MessageCircle className="size-4" />
</div>
<div className="flex flex-col">
<span className="text-sm font-medium">WhatsApp</span>
<span className="text-xs text-muted-foreground">
{wa?.state === "connected"
? `Linked as +${wa.self ?? "?"} — message yourself to use it`
: wa?.state === "qr"
? "Scan the QR below with your phone"
: wa?.state === "starting"
? "Connecting…"
: wa?.state === "error"
? wa.error ?? "Error"
: "Links your own WhatsApp as a companion device"}
</span>
</div>
</div>
<div className="flex items-center gap-2">
{wa?.state === "connected" && (
<Button
variant="outline"
size="sm"
className="h-7 px-3 text-xs"
onClick={() => {
void window.ipc.invoke("channels:whatsappLogout", null).catch(() => {
toast.error("Failed to unlink WhatsApp")
})
}}
>
Unlink
</Button>
)}
<Switch
checked={config.whatsapp.enabled}
disabled={saving}
onCheckedChange={(enabled) =>
void save({ ...config, whatsapp: { ...config.whatsapp, enabled } })
}
/>
</div>
</div>
{config.whatsapp.enabled && wa?.state === "qr" && wa.qrDataUrl && (
<div className="flex items-center gap-4 rounded-md border p-3">
<img src={wa.qrDataUrl} alt="WhatsApp pairing QR" className="size-40 rounded" />
<div className="text-xs text-muted-foreground space-y-1">
<p className="font-medium text-foreground">Link Rowboat to WhatsApp</p>
<p>1. Open WhatsApp on your phone</p>
<p>2. Settings Linked Devices Link a Device</p>
<p>3. Scan this code</p>
<p className="pt-1">
Then message <span className="font-medium">yourself</span> (your own contact) to talk
to Rowboat.
</p>
</div>
</div>
)}
{config.whatsapp.enabled && (
<div className="space-y-1.5">
<label className="text-xs font-medium">Additional allowed numbers</label>
<div className="flex gap-2">
<Input
value={waAllowDraft}
onChange={(e) => setWaAllowDraft(e.target.value)}
placeholder="e.g. 14155551234, 919876543210"
className="h-8 text-xs"
/>
<Button
variant="outline"
size="sm"
className="h-8 px-3 text-xs"
disabled={saving}
onClick={() =>
void save({
...config,
whatsapp: { ...config.whatsapp, allowFrom: parseIdList(waAllowDraft) },
})
}
>
Save
</Button>
</div>
<p className="text-xs text-muted-foreground">
Your own number (self-chat) is always allowed. Digits only, with country code.
</p>
</div>
)}
</div>
<Separator />
{/* Telegram */}
<div className="space-y-3">
<div className="flex items-center justify-between">
<div className="flex items-center gap-2.5">
<div className="flex size-8 items-center justify-center rounded-md bg-muted">
<Send className="size-4" />
</div>
<div className="flex flex-col">
<span className="text-sm font-medium">Telegram</span>
<span className="text-xs text-muted-foreground">
{tg?.state === "polling"
? `Listening${tg.botUsername ? ` as @${tg.botUsername}` : ""}`
: tg?.state === "starting"
? "Connecting…"
: tg?.state === "error"
? tg.error ?? "Error"
: "Uses your own bot — create one with @BotFather"}
</span>
</div>
</div>
<Switch
checked={config.telegram.enabled}
disabled={saving}
onCheckedChange={(enabled) =>
void save({ ...config, telegram: { ...config.telegram, enabled } })
}
/>
</div>
{config.telegram.enabled && (
<>
<div className="space-y-1.5">
<label className="text-xs font-medium">Bot token</label>
<div className="flex gap-2">
<Input
type="password"
value={tokenDraft}
onChange={(e) => setTokenDraft(e.target.value)}
placeholder="123456789:AAF…"
className="h-8 text-xs font-mono"
/>
<Button
variant="outline"
size="sm"
className="h-8 px-3 text-xs"
disabled={saving}
onClick={() =>
void save({
...config,
telegram: { ...config.telegram, botToken: tokenDraft.trim() },
})
}
>
Save
</Button>
</div>
<p className="text-xs text-muted-foreground">
Message @BotFather on Telegram /newbot paste the token here.
</p>
</div>
<div className="space-y-1.5">
<label className="text-xs font-medium">Allowed chat IDs</label>
<div className="flex gap-2">
<Input
value={tgAllowDraft}
onChange={(e) => setTgAllowDraft(e.target.value)}
placeholder="e.g. 123456789"
className="h-8 text-xs"
/>
<Button
variant="outline"
size="sm"
className="h-8 px-3 text-xs"
disabled={saving}
onClick={() =>
void save({
...config,
telegram: { ...config.telegram, allowFrom: parseIdList(tgAllowDraft) },
})
}
>
Save
</Button>
</div>
<p className="text-xs text-muted-foreground">
Message your bot once it replies with your chat ID to add here.
</p>
</div>
</>
)}
</div>
</div>
)
}

View file

@ -13,6 +13,7 @@ import {
Globe,
AlertTriangle,
Home,
LayoutGrid,
Mic,
SquarePen,
Plug,
@ -22,6 +23,8 @@ import {
Settings,
Square,
Video,
CircleAlert,
X,
} from "lucide-react"
import {
AlertDialog,
@ -51,6 +54,7 @@ import {
Popover,
PopoverContent,
PopoverTrigger,
PopoverArrow,
} from "@/components/ui/popover"
import {
Tooltip,
@ -58,7 +62,9 @@ import {
TooltipTrigger,
} from "@/components/ui/tooltip"
import { cn } from "@/lib/utils"
import { isOutOfCredits, CREDIT_EXHAUSTED_EVENT, CREDIT_REPLENISHED_EVENT } from "@/lib/credit-status"
import { SettingsDialog } from "@/components/settings-dialog"
import { MascotFaceIcon } from "@/components/talking-head"
import { extractConferenceLink } from "@/lib/calendar-event"
import { useBilling } from "@/hooks/useBilling"
import { toast } from "@/lib/toast"
@ -161,6 +167,7 @@ type SidebarContentPanelProps = {
onOpenMeetings?: () => void
onOpenCode?: () => void
onOpenBgTasks?: () => void
onOpenApps?: () => void
onOpenAgent?: (slug: string) => void
recentRuns?: { id: string; title?: string; createdAt: string; modifiedAt?: string }[]
onOpenRun?: (runId: string) => void
@ -170,8 +177,10 @@ type SidebarContentPanelProps = {
onNewChat?: () => void
onToggleBrowser?: () => void
onVoiceNoteCreated?: (path: string) => void
/** Starts the mascot-guided product tour. */
onStartTour?: () => void
/** Which primary destination is currently active, for nav highlighting. */
activeNav?: 'home' | 'email' | 'meetings' | 'code' | 'knowledge' | 'agents' | 'workspaces' | null
activeNav?: 'home' | 'email' | 'meetings' | 'code' | 'knowledge' | 'agents' | 'apps' | 'workspaces' | null
/** Live meeting recording state, so the recording row can show its indicator/stop. */
meetingRecordingState?: 'idle' | 'connecting' | 'recording' | 'stopping'
recordingMeetingSource?: string | null
@ -410,6 +419,7 @@ export function SidebarContentPanel({
onOpenMeetings,
onOpenCode,
onOpenBgTasks,
onOpenApps,
recentRuns = [],
onOpenRun,
onOpenChatHistory,
@ -418,6 +428,7 @@ export function SidebarContentPanel({
onNewChat,
onToggleBrowser,
onVoiceNoteCreated,
onStartTour,
activeNav,
meetingRecordingState = 'idle',
recordingMeetingSource = null,
@ -430,9 +441,13 @@ export function SidebarContentPanel({
const [openConnectionsAfterClose, setOpenConnectionsAfterClose] = useState(false)
const connectorsButtonRef = useRef<HTMLButtonElement | null>(null)
const [isRowboatConnected, setIsRowboatConnected] = useState(false)
const [creditPopoverOpen, setCreditPopoverOpen] = useState(false)
const [outOfCredits, setOutOfCredits] = useState(false)
const outOfCreditsRef = useRef(false)
const creditPopoverAutoShownRef = useRef(false)
const [loggingIn, setLoggingIn] = useState(false)
const [appUrl, setAppUrl] = useState<string | null>(null)
const { billing } = useBilling(isRowboatConnected)
const { billing, refresh: refreshBilling } = useBilling(isRowboatConnected)
const currentBillingPlan = billing ? getBillingPlanData(billing.catalog, billing.subscriptionPlanId) : null
// Nav previews: unread important emails + next upcoming meetings (top 2 each).
@ -654,6 +669,50 @@ export function SidebarContentPanel({
}
}, [])
// Re-anchor the warning whenever billing (re)loads — billing is authoritative.
useEffect(() => {
if (billing) {
const next = isOutOfCredits(billing)
outOfCreditsRef.current = next
setOutOfCredits(next)
}
}, [billing])
// Live signals: a usage API error flips it on; a successful cost-incurring
// call flips it off and triggers a single billing refresh to reconcile.
useEffect(() => {
const onExhausted = () => {
outOfCreditsRef.current = true
setOutOfCredits(true)
}
const onReplenished = () => {
const wasOut = outOfCreditsRef.current
outOfCreditsRef.current = false
setOutOfCredits(false)
if (wasOut) void refreshBilling()
}
window.addEventListener(CREDIT_EXHAUSTED_EVENT, onExhausted)
window.addEventListener(CREDIT_REPLENISHED_EVENT, onReplenished)
return () => {
window.removeEventListener(CREDIT_EXHAUSTED_EVENT, onExhausted)
window.removeEventListener(CREDIT_REPLENISHED_EVENT, onReplenished)
}
}, [refreshBilling])
// Auto-open the popover the first time we go out of credits; reset when
// credits return so it can auto-open again on a future episode.
useEffect(() => {
if (outOfCredits) {
if (!creditPopoverAutoShownRef.current) {
creditPopoverAutoShownRef.current = true
setCreditPopoverOpen(true)
}
} else {
creditPopoverAutoShownRef.current = false
setCreditPopoverOpen(false)
}
}, [outOfCredits])
// Single preview shown as a sublabel on the Email / Meetings nav buttons.
const previewEmail = emailThreads[0]
const previewMeeting = meetings[0]
@ -695,13 +754,14 @@ export function SidebarContentPanel({
<SidebarGroupContent>
<SidebarMenu>
<SidebarMenuItem>
<SidebarMenuButton isActive={activeNav === 'home'} onClick={onOpenHome}>
<SidebarMenuButton data-tour-id="nav-home" isActive={activeNav === 'home'} onClick={onOpenHome}>
<Home className="size-4 shrink-0" />
<span className="flex-1 truncate">Home</span>
</SidebarMenuButton>
</SidebarMenuItem>
<SidebarMenuItem>
<SidebarMenuButton
data-tour-id="nav-email"
isActive={activeNav === 'email'}
onClick={() => onOpenEmail?.()}
className={previewEmail ? 'h-auto py-1.5' : undefined}
@ -724,6 +784,7 @@ export function SidebarContentPanel({
</SidebarMenuItem>
<SidebarMenuItem>
<SidebarMenuButton
data-tour-id="nav-meetings"
isActive={activeNav === 'meetings'}
onClick={onOpenMeetings}
className={meetingSublabel ? 'h-auto py-1.5' : undefined}
@ -802,7 +863,7 @@ export function SidebarContentPanel({
</SidebarMenuItem>
{codeModeEnabled && (
<SidebarMenuItem>
<SidebarMenuButton isActive={activeNav === 'code'} onClick={onOpenCode}>
<SidebarMenuButton data-tour-id="nav-code" isActive={activeNav === 'code'} onClick={onOpenCode}>
<Code2 className="size-4 shrink-0" />
<span className="flex-1 truncate">Code</span>
</SidebarMenuButton>
@ -810,6 +871,7 @@ export function SidebarContentPanel({
)}
<SidebarMenuItem>
<SidebarMenuButton
data-tour-id="nav-knowledge"
isActive={activeNav === 'knowledge'}
onClick={() => knowledgeActions.openKnowledgeView()}
className={knowledgeUpdatedLabel ? 'h-auto py-1.5' : undefined}
@ -830,6 +892,7 @@ export function SidebarContentPanel({
<SidebarMenu>
<SidebarMenuItem>
<SidebarMenuButton
data-tour-id="nav-agents"
isActive={activeNav === 'agents'}
onClick={onOpenBgTasks}
className={bgAgentsLabel ? 'h-auto py-1.5' : undefined}
@ -850,6 +913,16 @@ export function SidebarContentPanel({
</SidebarMenuItem>
<SidebarMenuItem>
<SidebarMenuButton
isActive={activeNav === 'apps'}
onClick={onOpenApps}
>
<LayoutGrid className="size-4 shrink-0 text-muted-foreground" />
<span className="flex-1 truncate text-muted-foreground">Apps</span>
</SidebarMenuButton>
</SidebarMenuItem>
<SidebarMenuItem>
<SidebarMenuButton
data-tour-id="nav-workspaces"
isActive={activeNav === 'workspaces'}
onClick={() => knowledgeActions.openWorkspaceAt()}
className="h-auto py-1.5"
@ -874,6 +947,7 @@ export function SidebarContentPanel({
<SidebarGroupContent>
<button
type="button"
data-tour-id="nav-chats"
onClick={() => setChatsExpanded((v) => !v)}
className="flex w-full items-center gap-1.5 px-3 py-1 text-[10.5px] font-semibold uppercase tracking-wider text-muted-foreground"
>
@ -913,31 +987,89 @@ export function SidebarContentPanel({
</SidebarGroup>
</SidebarContent>
{/* Billing / upgrade CTA or Log in CTA */}
{isRowboatConnected && billing ? (
<div className="px-3 py-2">
<div className="flex items-center justify-between rounded-lg border border-sidebar-border bg-sidebar-accent/20 px-3 py-2">
<div className="min-w-0">
<span className="text-xs font-medium capitalize text-sidebar-foreground">
{currentBillingPlan?.displayName ?? (billing.subscriptionPlanId ? 'Unknown' : 'No plan')}
</span>
{billing.subscriptionStatus === 'trialing' && billing.trialExpiresAt && (() => {
const days = Math.max(0, Math.ceil((new Date(billing.trialExpiresAt).getTime() - Date.now()) / (1000 * 60 * 60 * 24)))
return (
<p className="text-[10px] text-sidebar-foreground/60">
{days === 0 ? 'Trial expires today' : days === 1 ? '1 day left' : `${days} days left`}
{isRowboatConnected && billing ? (() => {
const upgradeLabel = !billing.subscriptionPlanId || currentBillingPlan?.category === 'free' || currentBillingPlan?.category === 'starter' ? 'Upgrade' : 'Manage'
if (outOfCredits) {
return (
<div className="px-3 py-2">
<Popover open={creditPopoverOpen} onOpenChange={setCreditPopoverOpen}>
<div className="flex items-center justify-between rounded-lg border border-red-500/50 bg-red-500/10 px-3 py-2">
<PopoverTrigger asChild>
<button type="button" className="flex min-w-0 flex-1 items-center gap-2 text-left">
<AlertTriangle className="size-4 shrink-0 text-red-500" />
<div className="min-w-0">
<span className="text-xs font-medium capitalize text-sidebar-foreground">
{currentBillingPlan?.displayName ?? (billing.subscriptionPlanId ? 'Unknown' : 'No plan')}
</span>
<p className="text-[10px] text-red-500">Out of credits</p>
</div>
</button>
</PopoverTrigger>
<button
onClick={() => appUrl && window.open(`${appUrl}?intent=upgrade`)}
className="shrink-0 rounded-md bg-sidebar-foreground/10 px-2.5 py-1 text-[11px] font-medium text-sidebar-foreground transition-colors hover:bg-sidebar-foreground/20"
>
{upgradeLabel}
</button>
</div>
<PopoverContent side="top" align="start" sideOffset={10} className="w-72">
<PopoverArrow className="fill-popover" />
<div className="flex items-start justify-between gap-2">
<div className="flex items-center gap-2">
<span className="flex size-8 shrink-0 items-center justify-center rounded-md bg-red-500/15 text-red-500">
<CircleAlert className="size-4" />
</span>
<h4 className="text-sm font-bold text-foreground">You&apos;ve run out of credits</h4>
</div>
<button
type="button"
aria-label="Close"
onClick={() => setCreditPopoverOpen(false)}
className="rounded-md p-0.5 text-muted-foreground hover:bg-accent hover:text-foreground"
>
<X className="size-4" />
</button>
</div>
<p className="mt-2 text-xs text-muted-foreground">
Upgrade your plan to continue using all features.
</p>
)
})()}
<button
onClick={() => { appUrl && window.open(`${appUrl}?intent=upgrade`); setCreditPopoverOpen(false) }}
className="mt-3 w-full rounded-md bg-red-500 px-3 py-1.5 text-xs font-medium text-white transition-colors hover:bg-red-600"
>
Upgrade now
</button>
</PopoverContent>
</Popover>
</div>
)
}
return (
<div className="px-3 py-2">
<div className="flex items-center justify-between rounded-lg border border-sidebar-border bg-sidebar-accent/20 px-3 py-2">
<div className="min-w-0">
<span className="text-xs font-medium capitalize text-sidebar-foreground">
{currentBillingPlan?.displayName ?? (billing.subscriptionPlanId ? 'Unknown' : 'No plan')}
</span>
{billing.subscriptionStatus === 'trialing' && billing.trialExpiresAt && (() => {
const days = Math.max(0, Math.ceil((new Date(billing.trialExpiresAt).getTime() - Date.now()) / (1000 * 60 * 60 * 24)))
return (
<p className="text-[10px] text-sidebar-foreground/60">
{days === 0 ? 'Trial expires today' : days === 1 ? '1 day left' : `${days} days left`}
</p>
)
})()}
</div>
<button
onClick={() => appUrl && window.open(`${appUrl}?intent=upgrade`)}
className="shrink-0 rounded-md bg-sidebar-foreground/10 px-2.5 py-1 text-[11px] font-medium text-sidebar-foreground transition-colors hover:bg-sidebar-foreground/20"
>
{upgradeLabel}
</button>
</div>
<button
onClick={() => appUrl && window.open(`${appUrl}?intent=upgrade`)}
className="shrink-0 rounded-md bg-sidebar-foreground/10 px-2.5 py-1 text-[11px] font-medium text-sidebar-foreground transition-colors hover:bg-sidebar-foreground/20"
>
{!billing.subscriptionPlanId || currentBillingPlan?.category === 'free' || currentBillingPlan?.category === 'starter' ? 'Upgrade' : 'Manage'}
</button>
</div>
</div>
) : null}
)
})() : null}
{/* Sign in CTA */}
{!isRowboatConnected && (
<div className="px-3 py-2">
@ -1015,6 +1147,15 @@ export function SidebarContentPanel({
</AlertDialog>
)}
</div>
{onStartTour && (
<button
onClick={onStartTour}
className="flex w-full items-center gap-2 rounded-md px-2 py-1 text-xs text-sidebar-foreground/70 hover:bg-sidebar-accent hover:text-sidebar-accent-foreground transition-colors"
>
<MascotFaceIcon className="size-4" />
<span>Take a tour</span>
</button>
)}
<SettingsDialog>
<button className="flex w-full items-center gap-2 rounded-md px-2 py-1 text-xs text-sidebar-foreground/70 hover:bg-sidebar-accent hover:text-sidebar-accent-foreground transition-colors">
<Settings className="size-4" />

View file

@ -0,0 +1,487 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { X } from 'lucide-react'
import type { TTSState } from '@/hooks/useVoiceTTS'
import { cn } from '@/lib/utils'
const POSITION_STORAGE_KEY = 'talking-head-position'
// Must match the overlay's `bottom-28 right-8` anchor classes.
const ANCHOR_RIGHT_PX = 32
const ANCHOR_BOTTOM_PX = 112
const VIEWPORT_MARGIN_PX = 8
// Palette pulled from the mascot artwork: pale lavender body, dark walnut boat.
const BODY_FILL = '#E8E9F5'
const BODY_STROKE = '#17171B'
const CHEEK_FILL = '#F2B8BE'
const BOAT_DARK = '#3E2E24'
const BOAT_MID = '#54402F'
const BOAT_LIGHT = '#6B5138'
const MOUTH_FILL = '#2A1E19'
export type MascotHat =
| 'mailcap'
| 'headphones'
| 'hardhat'
| 'gradcap'
| 'captain'
| 'explorer'
| 'party'
type TalkingHeadProps = {
ttsState: TTSState
getLevel: () => number
size?: number
/** Costume piece drawn on the head (used by the product tour). */
hat?: MascotHat
/** Paddle hard: fast bobbing + oar strokes, e.g. while traveling in the tour. */
rowing?: boolean
}
/**
* The Rowboat mascot as an animated inline SVG: a round pale character sitting
* in a wooden rowboat holding an oar. The mouth is driven every animation
* frame from the live TTS audio level; eyes blink on a randomized timer.
*/
export function TalkingHead({ ttsState, getLevel, size = 160, hat, rowing = false }: TalkingHeadProps) {
const mouthOpenRef = useRef<SVGEllipseElement>(null)
const mouthSmileRef = useRef<SVGPathElement>(null)
const oarRef = useRef<SVGGElement>(null)
const smoothedRef = useRef(0)
const [blinking, setBlinking] = useState(false)
const speaking = ttsState === 'speaking'
const thinking = ttsState === 'synthesizing'
// Lip sync + oar paddle loop. Writes SVG attributes directly to avoid
// re-rendering React at 60fps. Stops itself once speech has ended and the
// mouth has settled closed; restarts when `speaking` flips this effect.
useEffect(() => {
let raf = 0
let t = 0
const tick = () => {
const target = speaking ? getLevel() : 0
const prev = smoothedRef.current
// Fast attack, slower decay reads as natural mouth movement
const smoothed = target > prev ? prev + (target - prev) * 0.5 : prev + (target - prev) * 0.2
const settled = !speaking && !rowing && smoothed < 0.005
smoothedRef.current = settled ? 0 : smoothed
const open = settled ? 0 : Math.min(1, smoothed * 1.6)
const mouthOpen = mouthOpenRef.current
const mouthSmile = mouthSmileRef.current
if (mouthOpen && mouthSmile) {
if (open > 0.06) {
mouthOpen.setAttribute('rx', String(6.5 + open * 4))
mouthOpen.setAttribute('ry', String(1.5 + open * 9))
mouthOpen.style.opacity = '1'
mouthSmile.style.opacity = '0'
} else {
mouthOpen.style.opacity = '0'
mouthSmile.style.opacity = '1'
}
}
const oar = oarRef.current
if (oar) {
if (rowing) {
t += 0.14
const angle = Math.sin(t) * 13
oar.setAttribute('transform', `rotate(${angle.toFixed(2)} 128 118)`)
} else if (speaking) {
t += 0.045
const angle = Math.sin(t) * 7
oar.setAttribute('transform', `rotate(${angle.toFixed(2)} 128 118)`)
} else {
oar.setAttribute('transform', 'rotate(0 128 118)')
}
}
if (!settled) {
raf = requestAnimationFrame(tick)
}
}
raf = requestAnimationFrame(tick)
return () => cancelAnimationFrame(raf)
}, [speaking, rowing, getLevel])
// Randomized blinking
useEffect(() => {
let timeout: ReturnType<typeof setTimeout>
let cancelled = false
const scheduleBlink = () => {
timeout = setTimeout(() => {
if (cancelled) return
setBlinking(true)
setTimeout(() => {
if (cancelled) return
setBlinking(false)
scheduleBlink()
}, 140)
}, 2400 + Math.random() * 2600)
}
scheduleBlink()
return () => {
cancelled = true
clearTimeout(timeout)
}
}, [])
return (
<div
className="talking-head-bob relative select-none"
style={{
width: size,
height: size,
animationDuration: rowing ? '0.8s' : speaking ? '1.6s' : '3.2s',
}}
>
<style>{`
@keyframes talking-head-bob {
0%, 100% { transform: translateY(0) rotate(-1.6deg); }
50% { transform: translateY(-4px) rotate(1.6deg); }
}
.talking-head-bob {
animation-name: talking-head-bob;
animation-timing-function: ease-in-out;
animation-iteration-count: infinite;
}
@keyframes talking-head-ripple {
0% { transform: scale(0.6); opacity: 0.5; }
100% { transform: scale(1.25); opacity: 0; }
}
.talking-head-ripple {
transform-origin: center;
transform-box: fill-box;
animation: talking-head-ripple 2.6s ease-out infinite;
}
@keyframes talking-head-bubble {
0%, 100% { opacity: 0.25; transform: translateY(0); }
50% { opacity: 1; transform: translateY(-2px); }
}
.talking-head-bubble {
animation: talking-head-bubble 1.2s ease-in-out infinite;
}
`}</style>
<svg viewBox="0 0 200 190" width={size} height={size} aria-hidden="true">
{/* water ripples under the boat */}
<g>
<ellipse className="talking-head-ripple" cx="100" cy="168" rx="62" ry="9" fill="none" stroke="#8FB6D9" strokeWidth="2" style={{ animationDelay: '0s' }} />
<ellipse className="talking-head-ripple" cx="100" cy="168" rx="62" ry="9" fill="none" stroke="#8FB6D9" strokeWidth="2" style={{ animationDelay: '1.3s' }} />
<ellipse cx="100" cy="168" rx="52" ry="7" fill="#8FB6D9" opacity="0.18" />
</g>
{/* thinking bubbles while synthesizing */}
{thinking && (
<g fill={BODY_STROKE} opacity="0.75">
<circle className="talking-head-bubble" cx="146" cy="34" r="3" style={{ animationDelay: '0s' }} />
<circle className="talking-head-bubble" cx="157" cy="26" r="4.2" style={{ animationDelay: '0.2s' }} />
<circle className="talking-head-bubble" cx="170" cy="16" r="5.4" style={{ animationDelay: '0.4s' }} />
</g>
)}
{/* character: head + body blob */}
<g>
<path
d="M 100 22
C 129 22 148 43 148 68
C 148 82 141 93 131 100
C 141 107 147 117 148 128
L 52 128
C 53 115 60 105 69 99
C 59 92 52 81 52 68
C 52 43 71 22 100 22 Z"
fill={BODY_FILL}
stroke={BODY_STROKE}
strokeWidth="5"
strokeLinejoin="round"
/>
{/* eyes */}
<g style={{ transform: thinking ? 'translateY(-2.5px)' : undefined, transition: 'transform 0.3s' }}>
<ellipse
cx="84" cy="64" rx="5" ry={blinking ? 0.8 : 7}
fill={BODY_STROKE}
style={{ transition: 'ry 0.06s' }}
/>
<ellipse
cx="116" cy="64" rx="5" ry={blinking ? 0.8 : 7}
fill={BODY_STROKE}
style={{ transition: 'ry 0.06s' }}
/>
<circle cx="86" cy="61" r="1.6" fill="#FFFFFF" opacity={blinking ? 0 : 0.9} />
<circle cx="118" cy="61" r="1.6" fill="#FFFFFF" opacity={blinking ? 0 : 0.9} />
</g>
{/* cheeks */}
<ellipse cx="72" cy="76" rx="6.5" ry="4" fill={CHEEK_FILL} opacity="0.85" />
<ellipse cx="128" cy="76" rx="6.5" ry="4" fill={CHEEK_FILL} opacity="0.85" />
{/* mouth: smile when quiet, open ellipse driven by audio level */}
<path
ref={mouthSmileRef}
d="M 91 80 Q 100 88 109 80"
fill="none"
stroke={BODY_STROKE}
strokeWidth="4"
strokeLinecap="round"
/>
<ellipse
ref={mouthOpenRef}
cx="100" cy="84" rx="7" ry="2"
fill={MOUTH_FILL}
stroke={BODY_STROKE}
strokeWidth="3"
style={{ opacity: 0 }}
/>
{hat && <MascotHatArt hat={hat} />}
</g>
{/* oar (rotates while speaking) */}
<g ref={oarRef}>
<line x1="158" y1="88" x2="88" y2="152" stroke={BODY_STROKE} strokeWidth="12" strokeLinecap="round" />
<line x1="158" y1="88" x2="88" y2="152" stroke={BOAT_MID} strokeWidth="7" strokeLinecap="round" />
<path
d="M 84 148 L 56 170 C 52 173 52 178 57 178 L 90 165 Z"
fill={BOAT_DARK}
stroke={BODY_STROKE}
strokeWidth="4"
strokeLinejoin="round"
/>
</g>
{/* hand resting over the oar */}
<ellipse cx="121" cy="120" rx="10" ry="8" fill={BODY_FILL} stroke={BODY_STROKE} strokeWidth="4" />
{/* boat hull (drawn last so it overlaps the body) */}
<g>
<path
d="M 30 120
C 50 132 150 132 170 120
C 168 142 152 160 100 160
C 48 160 32 142 30 120 Z"
fill={BOAT_MID}
stroke={BODY_STROKE}
strokeWidth="5"
strokeLinejoin="round"
/>
{/* plank lines */}
<path d="M 36 133 C 60 143 140 143 164 133" fill="none" stroke={BOAT_DARK} strokeWidth="3" strokeLinecap="round" />
<path d="M 44 145 C 66 153 134 153 156 145" fill="none" stroke={BOAT_DARK} strokeWidth="3" strokeLinecap="round" />
{/* gunwale highlight */}
<path d="M 33 121 C 52 131 148 131 167 121" fill="none" stroke={BOAT_LIGHT} strokeWidth="4" strokeLinecap="round" />
</g>
</svg>
</div>
)
}
type TalkingHeadOverlayProps = {
ttsState: TTSState
getLevel: () => number
onDismiss?: () => void
}
// Keep the widget fully on-screen relative to its bottom-right CSS anchor.
// Falls back to the default render size when the element isn't mounted yet.
function clampPositionToViewport(pos: { x: number; y: number }, el: HTMLDivElement | null): { x: number; y: number } {
const width = el?.offsetWidth ?? 160
const height = el?.offsetHeight ?? 160
const baseLeft = window.innerWidth - ANCHOR_RIGHT_PX - width
const baseTop = window.innerHeight - ANCHOR_BOTTOM_PX - height
const clamp = (v: number, min: number, max: number) => Math.max(min, Math.min(max, v))
return {
x: clamp(pos.x, VIEWPORT_MARGIN_PX - baseLeft, window.innerWidth - VIEWPORT_MARGIN_PX - width - baseLeft),
y: clamp(pos.y, VIEWPORT_MARGIN_PX - baseTop, window.innerHeight - VIEWPORT_MARGIN_PX - height - baseTop),
}
}
function loadStoredPosition(): { x: number; y: number } {
try {
const raw = localStorage.getItem(POSITION_STORAGE_KEY)
if (raw) {
const parsed = JSON.parse(raw)
if (typeof parsed?.x === 'number' && typeof parsed?.y === 'number') {
return parsed
}
}
} catch {
// ignore corrupt stored position
}
return { x: 0, y: 0 }
}
/**
* Floating, draggable widget that hosts the talking head. Anchored to the
* bottom-right of the window (above the composer) and offset by a persisted
* drag position, so it hovers over whatever view is active.
*/
export function TalkingHeadOverlay({ ttsState, getLevel, onDismiss }: TalkingHeadOverlayProps) {
// Clamp the stored offset at init so a stale position (e.g. saved on a
// bigger window) can't leave the widget stranded off-screen.
const [offset, setOffset] = useState(() => clampPositionToViewport(loadStoredPosition(), null))
const [dragging, setDragging] = useState(false)
const dragStartRef = useRef<{ pointerX: number; pointerY: number; x: number; y: number } | null>(null)
const containerRef = useRef<HTMLDivElement>(null)
const clampToViewport = useCallback(
(pos: { x: number; y: number }) => clampPositionToViewport(pos, containerRef.current),
[]
)
// Re-clamp when the window shrinks
useEffect(() => {
const handleResize = () => setOffset(prev => clampToViewport(prev))
window.addEventListener('resize', handleResize)
return () => window.removeEventListener('resize', handleResize)
}, [clampToViewport])
const handlePointerDown = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
if (e.button !== 0) return
dragStartRef.current = { pointerX: e.clientX, pointerY: e.clientY, x: offset.x, y: offset.y }
setDragging(true)
e.currentTarget.setPointerCapture(e.pointerId)
}, [offset])
const handlePointerMove = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
const start = dragStartRef.current
if (!start) return
setOffset({
x: start.x + (e.clientX - start.pointerX),
y: start.y + (e.clientY - start.pointerY),
})
}, [])
const handlePointerUp = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
if (!dragStartRef.current) return
dragStartRef.current = null
setDragging(false)
setOffset(prev => clampToViewport(prev))
e.currentTarget.releasePointerCapture(e.pointerId)
}, [clampToViewport])
useEffect(() => {
if (dragging) return
try {
localStorage.setItem(POSITION_STORAGE_KEY, JSON.stringify(offset))
} catch {
// best-effort persistence
}
}, [offset, dragging])
return (
<div
ref={containerRef}
className={cn(
'group fixed bottom-28 right-8 z-50 touch-none',
dragging ? 'cursor-grabbing' : 'cursor-grab'
)}
style={{
transform: `translate(${offset.x}px, ${offset.y}px)`,
// Constant value so the entrance animation runs once on mount and
// never restarts (re-applying it after a drag would replay the pop).
animation: 'talking-head-pop 0.35s cubic-bezier(0.34, 1.56, 0.64, 1)',
}}
onPointerDown={handlePointerDown}
onPointerMove={handlePointerMove}
onPointerUp={handlePointerUp}
onPointerCancel={handlePointerUp}
role="img"
aria-label="Rowboat talking head"
>
<style>{`
@keyframes talking-head-pop {
0% { opacity: 0; scale: 0.4; }
100% { opacity: 1; scale: 1; }
}
`}</style>
{onDismiss && (
<button
type="button"
onPointerDown={(e) => e.stopPropagation()}
onClick={onDismiss}
className="absolute -right-1 -top-1 z-10 flex h-5 w-5 items-center justify-center rounded-full border border-border bg-background text-muted-foreground opacity-0 shadow-sm transition-opacity hover:text-foreground group-hover:opacity-100"
aria-label="Hide talking head"
>
<X className="h-3 w-3" />
</button>
)}
<TalkingHead ttsState={ttsState} getLevel={getLevel} />
</div>
)
}
/** Costume pieces for the product tour, drawn on the mascot's head. */
function MascotHatArt({ hat }: { hat: MascotHat }) {
const outline = { stroke: BODY_STROKE, strokeWidth: 4, strokeLinejoin: 'round' as const }
switch (hat) {
case 'mailcap':
return (
<g>
<path d="M 68 36 Q 100 4 132 36 Q 100 26 68 36 Z" fill="#4A7DDB" {...outline} />
<path d="M 126 33 Q 148 31 150 39 Q 132 42 124 39 Z" fill="#3D68B8" {...outline} />
</g>
)
case 'headphones':
return (
<g>
<path d="M 64 58 Q 100 2 136 58" fill="none" stroke={BODY_STROKE} strokeWidth="11" strokeLinecap="round" />
<path d="M 64 58 Q 100 2 136 58" fill="none" stroke="#4B5563" strokeWidth="6" strokeLinecap="round" />
<ellipse cx="64" cy="61" rx="8" ry="11" fill="#4B5563" {...outline} />
<ellipse cx="136" cy="61" rx="8" ry="11" fill="#4B5563" {...outline} />
</g>
)
case 'hardhat':
return (
<g>
<path d="M 66 38 Q 100 0 134 38 Z" fill="#F6C445" {...outline} />
<path d="M 96 10 Q 94 22 96 36 L 106 36 Q 107 22 105 9 Z" fill="#E0AC2C" stroke="none" />
<path d="M 56 38 L 144 38" fill="none" stroke={BODY_STROKE} strokeWidth="9" strokeLinecap="round" />
<path d="M 58 38 L 142 38" fill="none" stroke="#F6C445" strokeWidth="5" strokeLinecap="round" />
</g>
)
case 'gradcap':
return (
<g>
<path d="M 80 28 Q 100 42 120 28 L 120 38 Q 100 50 80 38 Z" fill="#232838" {...outline} />
<path d="M 100 2 L 144 20 L 100 38 L 56 20 Z" fill="#2E3450" {...outline} />
<path d="M 100 20 Q 124 26 136 34" fill="none" stroke="#E8B94A" strokeWidth="3" strokeLinecap="round" />
<circle cx="137" cy="38" r="4" fill="#E8B94A" stroke={BODY_STROKE} strokeWidth="3" />
</g>
)
case 'captain':
return (
<g>
<path d="M 68 36 Q 100 -2 132 36 Q 100 28 68 36 Z" fill="#F4F5F9" {...outline} />
<path d="M 66 36 Q 100 46 134 36 L 134 42 Q 100 52 66 42 Z" fill="#22262E" {...outline} />
<path d="M 122 42 Q 146 42 148 48 Q 130 52 120 48 Z" fill="#22262E" {...outline} />
<circle cx="100" cy="38" r="4.5" fill="#E8B94A" stroke={BODY_STROKE} strokeWidth="3" />
</g>
)
case 'explorer':
return (
<g>
<ellipse cx="100" cy="35" rx="46" ry="9" fill="#C9A46B" {...outline} />
<path d="M 72 34 Q 100 4 128 34 Z" fill="#C9A46B" {...outline} />
<path d="M 76 30 Q 100 38 124 30" fill="none" stroke="#8A6B3D" strokeWidth="4" strokeLinecap="round" />
</g>
)
case 'party':
return (
<g>
<path d="M 100 -2 L 84 34 L 116 34 Z" fill="#F2699C" {...outline} />
<path d="M 92 16 L 111 22 M 88 26 L 114 31" fill="none" stroke="#FFD166" strokeWidth="3.5" strokeLinecap="round" />
<circle cx="100" cy="0" r="5" fill="#FFD166" stroke={BODY_STROKE} strokeWidth="3" />
</g>
)
}
}
/** Small static mascot face used as the toolbar toggle icon. */
export function MascotFaceIcon({ className }: { className?: string }) {
return (
<svg viewBox="0 0 24 24" width="16" height="16" className={className} aria-hidden="true">
<circle cx="12" cy="12" r="9.5" fill="none" stroke="currentColor" strokeWidth="1.8" />
<ellipse cx="8.6" cy="10.5" rx="1.3" ry="1.8" fill="currentColor" />
<ellipse cx="15.4" cy="10.5" rx="1.3" ry="1.8" fill="currentColor" />
<path d="M 9 14.5 Q 12 17 15 14.5" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" />
</svg>
)
}

View file

@ -0,0 +1,262 @@
import { useEffect } from 'react'
export type MascotVignetteKind = 'email' | 'meetings' | 'brain'
export type TourVignetteKind = MascotVignetteKind | 'agents'
/**
* Little looping "shows" staged around the mascot while it presents a section
* during the product tour. Purely decorative: everything is pointer-events-none
* and rendered on the tour's own layers, never inside the section's real UI.
*/
export function MascotVignette({ kind, playDing }: { kind: MascotVignetteKind; playDing?: () => void }) {
// One round of dings as the first envelopes land, then let the loop run silent
useEffect(() => {
if (kind !== 'email' || !playDing) return
const timers = [900, 1900, 2900].map((ms) => setTimeout(playDing, ms))
return () => timers.forEach(clearTimeout)
}, [kind, playDing])
return (
<div className="pointer-events-none absolute left-1/2 top-0 z-0 -translate-x-1/2" aria-hidden="true">
<style>{`
@keyframes tour-env-left {
0% { opacity: 0; transform: translate(-150px, -130px) rotate(-26deg); }
10% { opacity: 1; }
52% { opacity: 1; transform: translate(0px, 0px) rotate(5deg); }
64% { opacity: 0; transform: translate(2px, 12px) rotate(5deg); }
100% { opacity: 0; transform: translate(2px, 12px) rotate(5deg); }
}
@keyframes tour-env-right {
0% { opacity: 0; transform: translate(150px, -140px) rotate(26deg); }
10% { opacity: 1; }
52% { opacity: 1; transform: translate(0px, 0px) rotate(-5deg); }
64% { opacity: 0; transform: translate(-2px, 12px) rotate(-5deg); }
100% { opacity: 0; transform: translate(-2px, 12px) rotate(-5deg); }
}
@keyframes tour-badge-pulse {
0%, 100% { transform: scale(1); }
50% { transform: scale(1.15); }
}
@keyframes tour-note-line {
0% { width: 0; }
18% { width: var(--w); }
80% { width: var(--w); opacity: 1; }
92%, 100% { width: var(--w); opacity: 0; }
}
@keyframes tour-quill-bob {
0% { transform: translate(4px, 26px) rotate(-8deg); }
25% { transform: translate(46px, 30px) rotate(4deg); }
50% { transform: translate(6px, 40px) rotate(-8deg); }
75% { transform: translate(48px, 44px) rotate(4deg); }
100% { transform: translate(4px, 26px) rotate(-8deg); }
}
@keyframes tour-voice-bar {
0%, 100% { transform: scaleY(0.35); }
50% { transform: scaleY(1); }
}
@keyframes tour-orb {
0% { opacity: 0; transform: translate(var(--from-x), var(--from-y)) scale(0.5); }
15% { opacity: 0.9; }
70% { opacity: 0.9; transform: translate(0, 0) scale(1); }
85%, 100% { opacity: 0; transform: translate(0, 6px) scale(0.3); }
}
@keyframes tour-node-pulse {
0%, 100% { opacity: 0.55; }
50% { opacity: 1; }
}
@keyframes tour-edge-draw {
from { stroke-dashoffset: 60; }
to { stroke-dashoffset: 0; }
}
`}</style>
{kind === 'email' && (
<div className="relative" style={{ width: 240, height: 150 }}>
{[
{ anim: 'tour-env-left', delay: 0, x: 78, y: 96 },
{ anim: 'tour-env-right', delay: 1.0, x: 128, y: 102 },
{ anim: 'tour-env-left', delay: 2.0, x: 104, y: 92 },
{ anim: 'tour-env-right', delay: 3.0, x: 90, y: 104 },
].map((e, i) => (
<div
key={i}
className="absolute"
style={{ left: e.x, top: e.y, animation: `${e.anim} 4s ease-in-out ${e.delay}s infinite both` }}
>
<svg width="34" height="24" viewBox="0 0 34 24">
<rect x="1.5" y="1.5" width="31" height="21" rx="3" fill="#FFF8E7" stroke="#17171B" strokeWidth="2.5" />
<path d="M 2 4 L 17 14 L 32 4" fill="none" stroke="#17171B" strokeWidth="2.5" strokeLinejoin="round" />
</svg>
</div>
))}
<div
className="absolute"
style={{ left: 148, top: 78, animation: 'tour-badge-pulse 2s ease-in-out infinite' }}
>
<svg width="22" height="22" viewBox="0 0 22 22">
<circle cx="11" cy="11" r="9.5" fill="#3FA95C" stroke="#17171B" strokeWidth="2.5" />
<path d="M 6.5 11.5 L 9.5 14.5 L 15.5 8" fill="none" stroke="#FFFFFF" strokeWidth="2.5" strokeLinecap="round" strokeLinejoin="round" />
</svg>
</div>
</div>
)}
{kind === 'meetings' && (
<div className="relative" style={{ width: 220, height: 160, top: -110 }}>
{/* someone's talking */}
<div className="absolute left-1/2 flex -translate-x-1/2 items-end gap-1" style={{ top: 0, height: 26 }}>
{[0.9, 1.1, 0.7, 1.3, 0.8].map((dur, i) => (
<div
key={i}
className="w-1.5 origin-bottom rounded-full bg-[#8FB6D9]"
style={{ height: 22, animation: `tour-voice-bar ${dur}s ease-in-out ${i * 0.12}s infinite` }}
/>
))}
</div>
{/* ...and the notepad writes itself */}
<div
className="absolute left-1/2 rounded-md border-2 border-[#17171B] bg-[#FFFDF6] shadow-md"
style={{ top: 36, width: 96, height: 104, transform: 'translateX(-50%) rotate(-3deg)' }}
>
<div className="mx-2 mt-2 h-1.5 rounded bg-[#D9534F]/70" />
{[64, 52, 60, 44].map((w, i) => (
<div
key={i}
className="ml-2 mt-3 h-1.5 rounded bg-[#9AA1AE]"
style={{ ['--w' as string]: `${w}px`, animation: `tour-note-line 5s ease-out ${i * 1.1}s infinite both` }}
/>
))}
<svg
className="absolute left-0 top-0"
width="26"
height="26"
viewBox="0 0 26 26"
style={{ animation: 'tour-quill-bob 5s ease-in-out infinite' }}
>
<path d="M 4 22 L 10 12 Q 14 4 22 2 Q 18 10 12 14 Z" fill="#6B5138" stroke="#17171B" strokeWidth="2" strokeLinejoin="round" />
</svg>
</div>
</div>
)}
{kind === 'brain' && (
<div className="relative" style={{ width: 260, height: 180, top: -128 }}>
{/* constellation assembling above the head */}
<svg className="absolute left-1/2 -translate-x-1/2" width="140" height="80" viewBox="0 0 140 80" style={{ top: 0 }}>
{[
'M 22 58 L 52 24',
'M 52 24 L 88 40',
'M 88 40 L 120 18',
'M 88 40 L 70 66',
].map((d, i) => (
<path
key={i}
d={d}
fill="none"
stroke="#9CCBEA"
strokeWidth="2"
strokeDasharray="60"
style={{ animation: `tour-edge-draw 1.2s ease-out ${0.4 + i * 0.35}s both` }}
/>
))}
{[
[22, 58], [52, 24], [88, 40], [120, 18], [70, 66],
].map(([cx, cy], i) => (
<circle
key={i}
cx={cx}
cy={cy}
r={5}
fill="#BFE0F5"
stroke="#17171B"
strokeWidth="2"
style={{ animation: `tour-node-pulse 2.4s ease-in-out ${i * 0.3}s infinite` }}
/>
))}
</svg>
{/* thought-orbs drifting into the head */}
{[
{ fx: '-120px', fy: '-30px', delay: 0 },
{ fx: '120px', fy: '-40px', delay: 1.1 },
{ fx: '-90px', fy: '50px', delay: 2.2 },
{ fx: '110px', fy: '46px', delay: 3.3 },
].map((o, i) => (
<div
key={i}
className="absolute rounded-full"
style={{
left: 122,
top: 118,
width: 16,
height: 16,
background: 'radial-gradient(circle, #FFF3B8 20%, #FFD166 60%, transparent 75%)',
['--from-x' as string]: o.fx,
['--from-y' as string]: o.fy,
animation: `tour-orb 4.4s ease-in-out ${o.delay}s infinite both`,
}}
/>
))}
</div>
)}
</div>
)
}
/**
* Background-agents vignette: a fleet of tiny mascots rowing across the water
* at the bottom of the screen while the big one takes a break.
*/
export function AgentsFleet() {
return (
<div className="pointer-events-none fixed inset-x-0 bottom-0 z-[67] h-28 overflow-hidden" aria-hidden="true">
<style>{`
@keyframes tour-fleet-right {
from { transform: translateX(-18vw); }
to { transform: translateX(112vw); }
}
@keyframes tour-fleet-left {
from { transform: translateX(112vw); }
to { transform: translateX(-18vw); }
}
@keyframes tour-fleet-bob {
0%, 100% { transform: translateY(0) rotate(-2deg); }
50% { transform: translateY(-3px) rotate(2deg); }
}
`}</style>
{[
{ size: 46, bottom: 4, dur: 16, delay: 0, dir: 'tour-fleet-right' },
{ size: 34, bottom: 22, dur: 22, delay: -8, dir: 'tour-fleet-left' },
{ size: 28, bottom: 14, dur: 27, delay: -3, dir: 'tour-fleet-right' },
].map((b, i) => (
<div
key={i}
className="absolute left-0"
style={{ bottom: b.bottom, animation: `${b.dir} ${b.dur}s linear ${b.delay}s infinite` }}
>
<div style={{ animation: 'tour-fleet-bob 1.4s ease-in-out infinite', transform: b.dir === 'tour-fleet-left' ? 'scaleX(-1)' : undefined }}>
<MiniRower size={b.size} flipped={b.dir === 'tour-fleet-left'} />
</div>
</div>
))}
</div>
)
}
function MiniRower({ size, flipped }: { size: number; flipped: boolean }) {
return (
<svg
width={size}
height={size * 0.75}
viewBox="0 0 60 45"
style={{ transform: flipped ? 'scaleX(-1)' : undefined }}
>
<circle cx="27" cy="13" r="10" fill="#E8E9F5" stroke="#17171B" strokeWidth="3" />
<circle cx="23.5" cy="11.5" r="1.4" fill="#17171B" />
<circle cx="30.5" cy="11.5" r="1.4" fill="#17171B" />
<path d="M 23 16 Q 27 19 31 16" fill="none" stroke="#17171B" strokeWidth="1.6" strokeLinecap="round" />
<line x1="40" y1="14" x2="20" y2="38" stroke="#17171B" strokeWidth="4.5" strokeLinecap="round" />
<line x1="40" y1="14" x2="20" y2="38" stroke="#54402F" strokeWidth="2.5" strokeLinecap="round" />
<path d="M 7 25 C 17 31 43 31 53 25 C 51 37 43 42 30 42 C 17 42 9 37 7 25 Z" fill="#54402F" stroke="#17171B" strokeWidth="3" strokeLinejoin="round" />
</svg>
)
}

View file

@ -43,4 +43,17 @@ function PopoverAnchor({
return <PopoverPrimitive.Anchor data-slot="popover-anchor" {...props} />
}
export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor }
function PopoverArrow({
className,
...props
}: React.ComponentProps<typeof PopoverPrimitive.Arrow>) {
return (
<PopoverPrimitive.Arrow
data-slot="popover-arrow"
className={cn("fill-popover", className)}
{...props}
/>
)
}
export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor, PopoverArrow }

View file

@ -0,0 +1,274 @@
import { useEffect, useRef, useState } from 'react'
import { Mic, MicOff, Minimize2, MonitorUp, PhoneOff, Presentation, Square, User, Video, VideoOff } from 'lucide-react'
import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
import type { TTSState } from '@/hooks/useVoiceTTS'
import { cn } from '@/lib/utils'
export type VideoCallStatus = 'listening' | 'thinking' | 'speaking'
interface VideoCallViewProps {
/** Live camera stream from useVideoMode — attached to the user's tile. */
streamRef: React.MutableRefObject<MediaStream | null>
cameraOn: boolean
onToggleCamera: () => void
/** User mute = full input pause: no mic audio AND no frame capture. */
micMuted: boolean
onToggleMic: () => void
/** Starting a share collapses this view into the floating popout (the
* surface is derived from devices see App.tsx). */
onToggleScreenShare: () => void
/** Practice preset: the assistant is coaching this session. */
practiceMode?: boolean
/** Shrink to the floating pill without touching any devices. */
onMinimize: () => void
/** Stop the assistant: silence speech and abort the run if still going. */
onInterrupt: () => void
ttsState: TTSState
/** Live TTS output level — drives the mascot's mouth animation. */
getTtsLevel: () => number
status: VideoCallStatus
/** Live transcript of the user's in-progress utterance. */
interimText?: string
/** The assistant line currently being spoken aloud. */
assistantCaption?: string
onLeave: () => void
}
const STATUS_DISPLAY: Record<VideoCallStatus, { label: string; dotClass: string }> = {
listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' },
thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' },
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
}
/**
* Full-screen call surface: a Meet-style two-tile layout with the user's
* webcam on one side and the mascot as the other participant. Shown only
* while the camera is on with no screen share (the derived-surface rule in
* App.tsx) sharing or muting the camera moves the call into the floating
* popout. The mascot animates with the assistant's speech; dismissing it
* swaps in a Meet-style letter avatar ("R"). Live captions run along the
* bottom.
*/
export function VideoCallView({
streamRef,
cameraOn,
onToggleCamera,
micMuted,
onToggleMic,
onToggleScreenShare,
practiceMode,
onMinimize,
onInterrupt,
ttsState,
getTtsLevel,
status,
interimText,
assistantCaption,
onLeave,
}: VideoCallViewProps) {
const videoRef = useRef<HTMLVideoElement | null>(null)
const [mascotVisible, setMascotVisible] = useState(true)
useEffect(() => {
if (!cameraOn) return
const videoEl = videoRef.current
if (!videoEl) return
videoEl.srcObject = streamRef.current
videoEl.play().catch(() => {})
return () => {
videoEl.srcObject = null
}
}, [streamRef, cameraOn])
const userSpeaking = status === 'listening' && Boolean(interimText)
const assistantSpeaking = ttsState === 'speaking'
const caption = assistantSpeaking && assistantCaption
? { who: 'Rowboat', text: assistantCaption }
: interimText
? { who: 'You', text: interimText }
: null
return (
<div className="fixed inset-0 z-[100] flex flex-col bg-neutral-950">
{practiceMode && (
<span className="absolute left-4 top-4 z-10 flex items-center gap-1.5 rounded-full bg-violet-600/90 px-3 py-1 text-xs font-medium text-white">
<Presentation className="h-3.5 w-3.5" />
Practice session
</span>
)}
<button
type="button"
onClick={onMinimize}
className="absolute right-4 top-4 z-10 flex h-8 w-8 items-center justify-center rounded-full bg-neutral-800 text-white/80 transition-colors hover:bg-neutral-700 hover:text-white"
aria-label="Minimize call (shares your screen)"
title="Minimize — shares your screen so it can help you work"
>
<Minimize2 className="h-4 w-4" />
</button>
{/* Participant tiles */}
<div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
{/* User */}
<div
className={cn(
'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
userSpeaking && 'ring-2 ring-green-500/80'
)}
>
{cameraOn ? (
<video
ref={videoRef}
muted
playsInline
className="h-full w-full object-cover"
style={{ transform: 'scaleX(-1)' }}
/>
) : (
<span className="flex h-40 w-40 items-center justify-center rounded-full bg-neutral-700 text-neutral-400" aria-label="Camera off">
<User className="h-20 w-20" />
</span>
)}
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
You
</span>
{micMuted && (
<span className="absolute bottom-3 right-3 flex items-center gap-1.5 rounded-md bg-red-600/90 px-2 py-0.5 text-sm font-medium text-white">
<MicOff className="h-3.5 w-3.5" />
Muted nothing is heard or captured
</span>
)}
</div>
{/* Assistant */}
<div
className={cn(
'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
assistantSpeaking && 'ring-2 ring-sky-400/80'
)}
>
{mascotVisible ? (
<TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={220} />
) : (
<span
className="flex h-40 w-40 items-center justify-center rounded-full bg-sky-600 text-7xl font-medium text-white"
aria-label="Rowboat"
>
R
</span>
)}
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
Rowboat
</span>
{status !== 'listening' && (
<button
type="button"
onClick={onInterrupt}
className="absolute bottom-3 right-3 flex items-center gap-1.5 rounded-md bg-red-600/90 px-2.5 py-1 text-sm font-medium text-white transition-colors hover:bg-red-500"
aria-label="Stop the assistant"
title={status === 'speaking' ? 'Stop speaking' : 'Stop responding'}
>
<Square className="h-3 w-3 fill-current" />
Stop
</button>
)}
<button
type="button"
onClick={() => setMascotVisible((v) => !v)}
className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
>
{mascotVisible ? 'Hide mascot' : 'Show mascot'}
</button>
</div>
</div>
{/* Captions */}
<div className="flex h-14 items-center justify-center px-6">
{caption && (
<div className="max-w-3xl truncate rounded-lg bg-black/60 px-4 py-2 text-sm text-white/90">
<span className="mr-2 font-semibold text-white">{caption.who}:</span>
{caption.text}
</div>
)}
</div>
{/* Control bar */}
<div className="flex items-center justify-center gap-4 pb-5">
<span className="flex items-center gap-2 rounded-full bg-neutral-800 px-3 py-1.5 text-xs font-medium text-white/90">
{/* Muted overrides "Listening" the green pulse would be a lie.
Thinking/speaking still show: output continues while muted. */}
{micMuted && status === 'listening' ? (
<>
<span className="block h-2 w-2 rounded-full bg-red-500" />
Muted
</>
) : (
<>
<span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
{STATUS_DISPLAY[status].label}
</>
)}
</span>
<button
type="button"
onClick={onToggleMic}
className={cn(
'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
micMuted
? 'bg-red-600 text-white hover:bg-red-500'
: 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
)}
aria-label={micMuted ? 'Unmute' : 'Mute (pauses mic and frame capture)'}
title={micMuted ? 'Unmute' : 'Mute — pauses your mic and all frame capture'}
>
{micMuted ? <MicOff className="h-5 w-5" /> : <Mic className="h-5 w-5" />}
</button>
<button
type="button"
onClick={onToggleCamera}
className={cn(
'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
cameraOn
? 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
: 'bg-red-600 text-white hover:bg-red-500'
)}
aria-label={cameraOn ? 'Turn off camera' : 'Turn on camera'}
title={cameraOn ? 'Turn off camera' : 'Turn on camera'}
>
{cameraOn ? <Video className="h-5 w-5" /> : <VideoOff className="h-5 w-5" />}
</button>
<button
type="button"
onClick={onToggleScreenShare}
className="flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700"
aria-label="Present your screen"
title="Present your screen"
>
<MonitorUp className="h-5 w-5" />
</button>
<button
type="button"
onClick={() => setMascotVisible((v) => !v)}
className="relative flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700"
aria-label={mascotVisible ? 'Hide mascot' : 'Show mascot'}
>
<MascotFaceIcon />
{!mascotVisible && (
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
<span className="block h-[1.5px] w-6 -rotate-45 rounded-full bg-white/80" />
</span>
)}
</button>
<button
type="button"
onClick={onLeave}
className="flex h-10 w-14 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
aria-label="End call"
>
<PhoneOff className="h-5 w-5" />
</button>
</div>
</div>
)
}

View file

@ -0,0 +1,239 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { Maximize2, Mic, MicOff, MonitorUp, PhoneOff, Square, User, Video, VideoOff } from 'lucide-react'
import { TalkingHead } from '@/components/talking-head'
type PopoutState = {
ttsState: 'idle' | 'synthesizing' | 'speaking'
status: 'listening' | 'thinking' | 'speaking' | null
cameraOn: boolean
/** User mute = full input pause: no mic audio AND no frame capture. */
micMuted: boolean
screenSharing: boolean
interimText: string | null
}
const STATUS_DISPLAY: Record<NonNullable<PopoutState['status']>, { label: string; dotClass: string }> = {
listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' },
thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' },
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
}
const dragRegion = { WebkitAppRegion: 'drag' } as React.CSSProperties
const noDragRegion = { WebkitAppRegion: 'no-drag' } as React.CSSProperties
/**
* Content of the always-on-top popout window shown for the whole duration of
* a screen share (Meet-style floating mini-call) it floats over every app,
* including Rowboat itself, and is the call's control surface while sharing:
* camera toggle, share toggle, end-call. Rendered in its own BrowserWindow
* (see `video:setPopout` in the main process); call state streams in over
* the `video:popout-state` push channel and control actions round-trip back
* through `video:popoutAction`. Captures its own webcam feed MediaStreams
* can't cross windows.
*/
export function VideoPopout() {
// Camera defaults OFF: guessing "on" would flash the user's video for a
// beat before the real state arrives — which reads as a bug. The true
// state is fetched immediately below.
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: false, micMuted: false, screenSharing: false, interimText: null })
const videoRef = useRef<HTMLVideoElement | null>(null)
useEffect(() => {
const cleanup = window.ipc.on('video:popout-state', (next) => setState(next))
// The main process replays the cached state on did-finish-load, but that
// can race this listener's registration — fetch it explicitly too.
window.ipc
.invoke('video:getPopoutState', null)
.then(({ state: cached }) => {
if (cached) setState(cached)
})
.catch(() => {})
return cleanup
}, [])
// Own camera feed, following the main window's camera-on/off state.
useEffect(() => {
if (!state.cameraOn) return
let stream: MediaStream | null = null
let cancelled = false
navigator.mediaDevices
.getUserMedia({ video: { width: { ideal: 640 }, facingMode: 'user' }, audio: false })
.then((s) => {
if (cancelled) {
s.getTracks().forEach((t) => t.stop())
return
}
stream = s
if (videoRef.current) {
videoRef.current.srcObject = s
videoRef.current.play().catch(() => {})
}
})
.catch((err) => console.error('[popout] camera failed:', err))
return () => {
cancelled = true
stream?.getTracks().forEach((t) => t.stop())
if (videoRef.current) videoRef.current.srcObject = null
}
}, [state.cameraOn])
// The popout has no TTS audio pipeline — synthesize a plausible mouth level
// so the mascot still animates while the assistant speaks in the main window.
const getLevel = useCallback(() => 0.45 + 0.35 * Math.sin(performance.now() / 90), [])
const sendAction = useCallback((action: 'toggle-mic' | 'toggle-camera' | 'toggle-share' | 'stop-speaking' | 'end-call' | 'expand') => {
void window.ipc.invoke('video:popoutAction', { action }).catch(() => {})
}, [])
const statusDisplay = state.status ? STATUS_DISPLAY[state.status] : null
return (
<div
className="relative flex h-screen w-screen select-none flex-col gap-1.5 bg-neutral-900 p-1.5"
style={dragRegion}
>
<div className="flex min-h-0 flex-1 gap-1.5">
<div className="relative flex-1 overflow-hidden rounded-lg bg-neutral-800">
{state.cameraOn ? (
<video
ref={videoRef}
muted
playsInline
className="h-full w-full object-cover"
style={{ transform: 'scaleX(-1)' }}
/>
) : (
<div className="flex h-full w-full items-center justify-center">
<span className="flex h-12 w-12 items-center justify-center rounded-full bg-neutral-700 text-neutral-400">
<User className="h-6 w-6" />
</span>
</div>
)}
<span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
You
</span>
{/* Persistent consent badge the user must always be able to see
at a glance that their screen is going out. Muted pauses frame
capture while keeping the share stream open, so say so. */}
{state.screenSharing && (
<span className="absolute left-1.5 top-1.5 flex items-center gap-1 rounded-full bg-sky-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
<span className={`block h-1.5 w-1.5 rounded-full bg-white ${state.micMuted ? '' : 'animate-pulse'}`} />
{state.micMuted ? 'Sharing paused' : 'Sharing screen'}
</span>
)}
{state.micMuted && (
<span className="absolute bottom-1 right-1.5 flex items-center gap-1 rounded bg-red-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
<MicOff className="h-2.5 w-2.5" />
Muted
</span>
)}
</div>
<div className="relative flex flex-1 items-center justify-center overflow-hidden rounded-lg bg-neutral-800">
<TalkingHead ttsState={state.ttsState} getLevel={getLevel} size={84} />
<span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
Rowboat
</span>
{statusDisplay && (
<span className="absolute right-1.5 top-1.5 flex items-center gap-1 rounded-full bg-black/50 px-1.5 py-0.5 text-[10px] font-medium text-white">
{/* Muted overrides "Listening" — the green pulse would be a lie. */}
{state.micMuted && state.status === 'listening' ? (
<>
<span className="block h-1.5 w-1.5 rounded-full bg-red-500" />
Muted
</>
) : (
<>
<span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
{statusDisplay.label}
</>
)}
</span>
)}
{(state.status === 'speaking' || state.status === 'thinking') && (
<button
type="button"
onClick={() => sendAction('stop-speaking')}
className="absolute bottom-1 right-1.5 flex items-center gap-1 rounded bg-red-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white hover:bg-red-500"
style={noDragRegion}
aria-label="Stop the assistant"
title={state.status === 'speaking' ? 'Stop speaking' : 'Stop responding'}
>
<Square className="h-2.5 w-2.5 fill-current" />
Stop
</button>
)}
</div>
{/* Live caption of the in-progress utterance, floating over the tiles */}
{state.interimText && (
<div className="pointer-events-none absolute inset-x-1.5 bottom-9 flex justify-center">
<span className="max-w-full truncate rounded bg-black/70 px-1.5 py-0.5 text-[10px] text-white/90">
{state.interimText}
</span>
</div>
)}
</div>
{/* Control bar — actions execute in the main app window */}
<div className="flex h-7 shrink-0 items-center justify-center gap-2" style={noDragRegion}>
<button
type="button"
onClick={() => sendAction('toggle-mic')}
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
state.micMuted
? 'bg-red-600 text-white hover:bg-red-500'
: 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
}`}
aria-label={state.micMuted ? 'Unmute' : 'Mute (pauses mic and frame capture)'}
title={state.micMuted ? 'Unmute' : 'Mute — pauses your mic and all frame capture'}
>
{state.micMuted ? <MicOff className="h-3.5 w-3.5" /> : <Mic className="h-3.5 w-3.5" />}
</button>
<button
type="button"
onClick={() => sendAction('toggle-camera')}
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
state.cameraOn
? 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
: 'bg-red-600 text-white hover:bg-red-500'
}`}
aria-label={state.cameraOn ? 'Turn off camera' : 'Turn on camera'}
title={state.cameraOn ? 'Turn off camera' : 'Turn on camera'}
>
{state.cameraOn ? <Video className="h-3.5 w-3.5" /> : <VideoOff className="h-3.5 w-3.5" />}
</button>
<button
type="button"
onClick={() => sendAction('toggle-share')}
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
state.screenSharing
? 'bg-sky-600 text-white hover:bg-sky-500'
: 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
}`}
aria-label={state.screenSharing ? 'Stop sharing screen' : 'Share your screen'}
title={state.screenSharing ? 'Stop sharing screen' : 'Share your screen'}
>
<MonitorUp className="h-3.5 w-3.5" />
</button>
<button
type="button"
onClick={() => sendAction('end-call')}
className="flex h-6 w-8 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
aria-label="End call"
title="End call"
>
<PhoneOff className="h-3.5 w-3.5" />
</button>
<button
type="button"
onClick={() => sendAction('expand')}
className="flex h-6 w-6 items-center justify-center rounded-full bg-neutral-700 text-white/90 transition-colors hover:bg-neutral-600"
aria-label="Expand to full screen (stops screen sharing)"
title="Expand to full screen (stops sharing)"
>
<Maximize2 className="h-3.5 w-3.5" />
</button>
</div>
</div>
)
}

View file

@ -0,0 +1,326 @@
import { useCallback, useEffect, useRef, useState } from 'react';
export type VideoModeState = 'idle' | 'starting' | 'live';
export type ScreenShareState = 'idle' | 'starting' | 'live';
export interface CapturedVideoFrame {
/** base64-encoded JPEG bytes (no data: prefix) — shape of the UserImagePart wire format */
data: string;
mediaType: string;
capturedAt: string; // ISO timestamp
/** data: URL of the same frame, for direct display in the transcript */
dataUrl: string;
source: 'camera' | 'screen';
}
// Frames are grabbed once per second — dense enough to catch expression and
// posture changes while the user talks. Per message we attach at most
// MAX_*_FRAMES_PER_MESSAGE frames, evenly sampled across the window since the
// last send, so long monologues don't balloon the request.
const CAPTURE_INTERVAL_MS = 1000;
const MAX_CAMERA_FRAMES_PER_MESSAGE = 12;
// Screen frames are ~4x the resolution (and tokens) of camera frames, and the
// latest view matters far more than the trajectory — keep the cap small.
const MAX_SCREEN_FRAMES_PER_MESSAGE = 4;
// Rolling buffer bound (~2 minutes). The buffer only needs to cover the gap
// between two sends; anything older is stale context anyway.
const MAX_BUFFERED_FRAMES = 120;
// Downscale targets. 512px wide JPEG keeps a webcam frame around 20-40KB —
// cheap enough to inline a dozen per message as multimodal image parts.
// Screen captures keep 1280px so on-screen text stays legible to the model.
const CAMERA_FRAME_WIDTH = 512;
const SCREEN_FRAME_WIDTH = 1280;
const CAMERA_JPEG_QUALITY = 0.65;
const SCREEN_JPEG_QUALITY = 0.7;
interface BufferedFrame {
dataUrl: string;
capturedAt: string;
ts: number;
}
// One capture pipeline: stream → offscreen <video> → canvas JPEG → ring buffer.
interface CapturePipe {
stream: MediaStream | null;
videoEl: HTMLVideoElement | null;
canvas: HTMLCanvasElement | null;
interval: ReturnType<typeof setInterval> | null;
frames: BufferedFrame[];
lastCollectTs: number;
}
const emptyPipe = (): CapturePipe => ({
stream: null,
videoEl: null,
canvas: null,
interval: null,
frames: [],
lastCollectTs: 0,
});
function capturePipeFrame(pipe: CapturePipe, width: number, quality: number) {
const videoEl = pipe.videoEl;
if (!videoEl || videoEl.readyState < 2 || videoEl.videoWidth === 0) return;
if (!pipe.canvas) {
pipe.canvas = document.createElement('canvas');
}
const canvas = pipe.canvas;
const scale = Math.min(1, width / videoEl.videoWidth);
canvas.width = Math.round(videoEl.videoWidth * scale);
canvas.height = Math.round(videoEl.videoHeight * scale);
const ctx = canvas.getContext('2d');
if (!ctx) return;
ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
const dataUrl = canvas.toDataURL('image/jpeg', quality);
// A near-empty data URL means the frame was blank (source still warming up)
if (dataUrl.length < 100) return;
pipe.frames.push({ dataUrl, capturedAt: new Date().toISOString(), ts: Date.now() });
if (pipe.frames.length > MAX_BUFFERED_FRAMES) {
pipe.frames.splice(0, pipe.frames.length - MAX_BUFFERED_FRAMES);
}
}
function attachPipeSource(pipe: CapturePipe, stream: MediaStream, grab: () => void) {
pipe.stream = stream;
// Offscreen <video> that feeds the capture canvas; any visible preview
// attaches to the same MediaStream separately.
const videoEl = document.createElement('video');
videoEl.muted = true;
videoEl.playsInline = true;
videoEl.srcObject = stream;
pipe.videoEl = videoEl;
videoEl.play().catch(() => {});
// First frame as soon as the source delivers data, then steady-state cadence.
videoEl.addEventListener('loadeddata', () => grab(), { once: true });
pipe.interval = setInterval(grab, CAPTURE_INTERVAL_MS);
}
function teardownPipe(pipe: CapturePipe) {
if (pipe.interval) {
clearInterval(pipe.interval);
pipe.interval = null;
}
if (pipe.videoEl) {
pipe.videoEl.srcObject = null;
pipe.videoEl = null;
}
if (pipe.stream) {
pipe.stream.getTracks().forEach((t) => t.stop());
pipe.stream = null;
}
pipe.frames = [];
pipe.lastCollectTs = 0;
}
/**
* Drain frames captured since the previous collection, evenly sampled down to
* `max` (always keeping the newest). Falls back to the single most recent
* frame when nothing new accumulated (rapid-fire messages), so every message
* carries at least one frame once the source has warmed up.
*/
function drainPipe(pipe: CapturePipe, max: number, source: CapturedVideoFrame['source']): CapturedVideoFrame[] {
const all = pipe.frames;
if (all.length === 0) return [];
let window_ = all.filter((f) => f.ts > pipe.lastCollectTs);
if (window_.length === 0) {
window_ = [all[all.length - 1]];
}
pipe.lastCollectTs = window_[window_.length - 1].ts;
let sampled: BufferedFrame[];
if (window_.length <= max) {
sampled = window_;
} else {
sampled = [];
const step = (window_.length - 1) / (max - 1);
for (let i = 0; i < max; i++) {
sampled.push(window_[Math.round(i * step)]);
}
}
return sampled.map((f) => ({
data: f.dataUrl.slice(f.dataUrl.indexOf(',') + 1),
mediaType: 'image/jpeg',
capturedAt: f.capturedAt,
dataUrl: f.dataUrl,
source,
}));
}
export function useVideoMode() {
const [state, setState] = useState<VideoModeState>('idle');
const [screenState, setScreenState] = useState<ScreenShareState>('idle');
// Camera can be turned off mid-session (Meet-style) while the mode — and
// any screen share — keeps running. Resets to on for the next session.
const [cameraOn, setCameraOn] = useState(true);
// In-call mute pauses capture entirely: nothing lands in the ring buffers
// and collectFrames() returns nothing, so a muted stretch can never ride
// along with a later message. Streams stay open for instant resume.
const capturePausedRef = useRef(false);
const cameraPipeRef = useRef<CapturePipe>(emptyPipe());
const screenPipeRef = useRef<CapturePipe>(emptyPipe());
// Stable stream refs for preview components (<video srcObject>).
const streamRef = useRef<MediaStream | null>(null);
const screenStreamRef = useRef<MediaStream | null>(null);
const stateRef = useRef<VideoModeState>('idle');
stateRef.current = state;
const screenStateRef = useRef<ScreenShareState>('idle');
screenStateRef.current = screenState;
const captureCameraFrame = useCallback(() => {
if (capturePausedRef.current) return;
capturePipeFrame(cameraPipeRef.current, CAMERA_FRAME_WIDTH, CAMERA_JPEG_QUALITY);
}, []);
const captureScreenFrame = useCallback(() => {
if (capturePausedRef.current) return;
capturePipeFrame(screenPipeRef.current, SCREEN_FRAME_WIDTH, SCREEN_JPEG_QUALITY);
}, []);
const setCapturePaused = useCallback((paused: boolean) => {
capturePausedRef.current = paused;
}, []);
const stopScreenShare = useCallback(() => {
teardownPipe(screenPipeRef.current);
screenStreamRef.current = null;
setScreenState('idle');
}, []);
const stop = useCallback(() => {
teardownPipe(cameraPipeRef.current);
streamRef.current = null;
setState('idle');
setCameraOn(true);
capturePausedRef.current = false;
stopScreenShare();
}, [stopScreenShare]);
// Acquire the webcam and start its capture pipeline. Shared by start()
// and by re-enabling the camera mid-session.
const acquireCamera = useCallback(async (): Promise<boolean> => {
// Settle the macOS TCC camera permission before getUserMedia, same as
// voice mode does for the mic — otherwise the first click silently
// fails while the native prompt is still up.
const access = await window.ipc
.invoke('voice:ensureCameraAccess', null)
.catch(() => ({ granted: true }));
if (!access.granted) {
console.error('[video] Camera access denied');
return false;
}
let stream: MediaStream | null = null;
try {
stream = await navigator.mediaDevices.getUserMedia({
video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: 'user' },
audio: false,
});
} catch (err) {
console.error('[video] Camera access failed:', err);
return false;
}
streamRef.current = stream;
attachPipeSource(cameraPipeRef.current, stream, captureCameraFrame);
return true;
}, [captureCameraFrame]);
/**
* Turn the camera off/on without leaving video mode (Meet-style). While
* off, no webcam frames are captured or attached; screen-share frames
* (if presenting) keep flowing.
*/
const setCameraEnabled = useCallback(async (enabled: boolean): Promise<boolean> => {
if (stateRef.current !== 'live') return false;
if (enabled) {
const ok = await acquireCamera();
if (ok) setCameraOn(true);
return ok;
}
teardownPipe(cameraPipeRef.current);
streamRef.current = null;
setCameraOn(false);
return true;
}, [acquireCamera]);
/**
* Start video mode. `camera: false` starts a camera-less session (voice
* call / screen-share-only) the mode is live so frames can flow from
* other sources, and the camera can be enabled later via setCameraEnabled.
*/
const start = useCallback(async ({ camera = true }: { camera?: boolean } = {}): Promise<boolean> => {
if (stateRef.current !== 'idle') return true;
setState('starting');
if (camera) {
const ok = await acquireCamera();
if (!ok) {
setState('idle');
return false;
}
}
setCameraOn(camera);
setState('live');
return true;
}, [acquireCamera]);
/**
* Share the screen. The main process auto-approves getDisplayMedia with
* the primary screen (see setDisplayMediaRequestHandler in main.ts), so
* no source picker appears. Returns false if capture couldn't start
* (usually the macOS Screen Recording permission).
*/
const startScreenShare = useCallback(async (): Promise<boolean> => {
if (screenStateRef.current !== 'idle') return true;
setScreenState('starting');
// Surfaces the macOS Screen Recording permission state and, on first
// use, registers the app in System Settings (same flow meetings use).
await window.ipc.invoke('meeting:checkScreenPermission', null).catch(() => null);
let stream: MediaStream | null = null;
try {
stream = await navigator.mediaDevices.getDisplayMedia({
video: { frameRate: { ideal: 5 } },
audio: false,
});
} catch (err) {
console.error('[video] Screen share failed:', err);
setScreenState('idle');
return false;
}
screenStreamRef.current = stream;
// The capture can end outside our UI (display unplugged, OS revokes) —
// tear down cleanly so the UI doesn't show a dead share.
stream.getVideoTracks()[0]?.addEventListener('ended', () => stopScreenShare(), { once: true });
attachPipeSource(screenPipeRef.current, stream, captureScreenFrame);
setScreenState('live');
return true;
}, [captureScreenFrame, stopScreenShare]);
/**
* Drain webcam + screen-share frames buffered since the last send, tagged
* by source. Webcam frames come first, then screen frames.
*/
const collectFrames = useCallback((): CapturedVideoFrame[] => {
if (stateRef.current !== 'live') return [];
// Muted: no frames at all — not even pre-mute buffered ones — so a
// typed message during a mute carries nothing captured around it.
if (capturePausedRef.current) return [];
// Grab a frame right now so the message always includes the moment of send.
captureCameraFrame();
const frames = drainPipe(cameraPipeRef.current, MAX_CAMERA_FRAMES_PER_MESSAGE, 'camera');
if (screenStateRef.current === 'live') {
captureScreenFrame();
frames.push(...drainPipe(screenPipeRef.current, MAX_SCREEN_FRAMES_PER_MESSAGE, 'screen'));
}
return frames;
}, [captureCameraFrame, captureScreenFrame]);
// Release the camera/screen if the component unmounts with video mode on.
useEffect(() => stop, [stop]);
return { state, screenState, cameraOn, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, setCameraEnabled, setCapturePaused, collectFrames };
}

View file

@ -19,7 +19,35 @@ const DEEPGRAM_PARAMS = new URLSearchParams({
endpointing: '100',
no_delay: 'true',
});
const DEEPGRAM_LISTEN_URL = `wss://api.deepgram.com/v1/listen?${DEEPGRAM_PARAMS.toString()}`;
// Hands-free (continuous) mode: Deepgram's endpoint fires FAST (600ms of
// silence) and we apply smart hold logic on our side — if the transcript
// already reads as a complete thought (terminal punctuation) the utterance
// fires immediately, otherwise we hold INCOMPLETE_HOLD_MS longer in case the
// user was mid-thought. Net effect: complete sentences turn around ~1.2s
// faster than the old fixed 1800ms endpoint, while thinking pauses still get
// the same total grace (~1.8s).
const CONTINUOUS_ENDPOINTING_MS = 600;
const INCOMPLETE_HOLD_MS = 1200;
// While the mic is paused (assistant speaking), keep the idle Deepgram socket
// alive — it closes after ~10s without audio otherwise.
const KEEPALIVE_INTERVAL_MS = 5000;
// Deepgram punctuates finals (punctuate=true) — a transcript ending in
// terminal punctuation (optionally inside a closing quote/paren) is treated
// as a complete thought.
const COMPLETE_THOUGHT_RE = /[.!?…]["')\]]*\s*$/;
function deepgramParams(continuous: boolean): URLSearchParams {
if (!continuous) return DEEPGRAM_PARAMS;
const params = new URLSearchParams(DEEPGRAM_PARAMS);
params.set('endpointing', String(CONTINUOUS_ENDPOINTING_MS));
// Second end-of-speech signal: speech_final can be missed (it often rides
// on a result with an empty transcript, or never fires when background
// noise keeps the endpointer engaged). UtteranceEnd is word-timing based
// and arrives as its own message type, so we listen for both.
params.set('utterance_end_ms', '1000');
return params;
}
// Cap on retained per-frame amplitude samples (~64ms/frame ⇒ ~5 min of history).
// The waveform only ever displays the most recent window, so older samples are dropped.
@ -53,6 +81,13 @@ export function useVoiceMode() {
const audioLevelsRef = useRef<number[]>([]);
// Running peak amplitude for the waveform auto-gain (see PEAK_DECAY/MIN_PEAK).
const audioPeakRef = useRef(0);
// Hands-free mode: invoked with each completed utterance (speech_final).
const continuousCbRef = useRef<((text: string) => void) | null>(null);
// While true (assistant is speaking), mic audio is dropped instead of streamed.
const pausedRef = useRef(false);
const keepAliveTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
// Pending mid-thought hold (smart endpointing) — see maybeEndUtterance.
const holdTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
// Refresh cached auth details (called on warmup, not on mic click)
const refreshAuth = useCallback(async () => {
@ -71,9 +106,44 @@ export function useVoiceMode() {
}
}, [refreshRowboatAccount]);
// Hands-free mode: flush the accumulated utterance to the callback.
// Both end-of-speech signals may fire for the same utterance — the second
// finds an empty buffer and is a no-op.
const fireContinuousUtterance = useCallback(() => {
if (holdTimerRef.current) {
clearTimeout(holdTimerRef.current);
holdTimerRef.current = null;
}
if (!continuousCbRef.current || pausedRef.current) return;
const utterance = transcriptBufferRef.current.trim();
transcriptBufferRef.current = '';
interimRef.current = '';
setInterimText('');
if (utterance) continuousCbRef.current(utterance);
}, []);
// Smart endpoint: Deepgram's endpoint fires fast (600ms). If the
// transcript reads as a complete thought, hand it off immediately; if it
// trails off mid-sentence ("so what I want is…"), hold a little longer —
// resumed speech cancels the hold and the utterance keeps growing.
const maybeEndUtterance = useCallback(() => {
if (!continuousCbRef.current || pausedRef.current) return;
const buffered = transcriptBufferRef.current.trim();
if (!buffered) return;
if (COMPLETE_THOUGHT_RE.test(buffered)) {
fireContinuousUtterance();
return;
}
if (holdTimerRef.current) clearTimeout(holdTimerRef.current);
holdTimerRef.current = setTimeout(() => {
holdTimerRef.current = null;
fireContinuousUtterance();
}, INCOMPLETE_HOLD_MS);
}, [fireContinuousUtterance]);
// Create and connect a Deepgram WebSocket using cached auth.
// Starts the connection and returns immediately (does not wait for open).
const connectWs = useCallback(async () => {
const connectWs = useCallback(async (continuous = false) => {
if (wsRef.current && (wsRef.current.readyState === WebSocket.OPEN || wsRef.current.readyState === WebSocket.CONNECTING)) return;
// Refresh auth if we don't have it cached yet
@ -82,12 +152,13 @@ export function useVoiceMode() {
}
if (!cachedAuth) return;
const params = deepgramParams(continuous);
let ws: WebSocket;
if (cachedAuth.type === 'rowboat') {
const listenUrl = buildDeepgramListenUrl(cachedAuth.url, DEEPGRAM_PARAMS);
const listenUrl = buildDeepgramListenUrl(cachedAuth.url, params);
ws = new WebSocket(listenUrl, ['bearer', cachedAuth.token]);
} else {
ws = new WebSocket(DEEPGRAM_LISTEN_URL, ['token', cachedAuth.apiKey]);
ws = new WebSocket(`wss://api.deepgram.com/v1/listen?${params.toString()}`, ['token', cachedAuth.apiKey]);
}
wsRef.current = ws;
@ -103,16 +174,43 @@ export function useVoiceMode() {
ws.onmessage = (event) => {
const data = JSON.parse(event.data);
if (!data.channel?.alternatives?.[0]) return;
// Hands-free mode: word-timing based end-of-speech marker.
if (data.type === 'UtteranceEnd') {
maybeEndUtterance();
return;
}
if (!data.channel?.alternatives?.[0]) return;
const transcript = data.channel.alternatives[0].transcript;
if (!transcript) return;
// The user resumed speaking — cancel any pending mid-thought hold
// so the utterance keeps growing instead of firing under them.
if (transcript && holdTimerRef.current) {
clearTimeout(holdTimerRef.current);
holdTimerRef.current = null;
}
if (data.is_final) {
transcriptBufferRef.current += (transcriptBufferRef.current ? ' ' : '') + transcript;
interimRef.current = '';
setInterimText(transcriptBufferRef.current);
} else {
// NOTE: the endpoint marker (speech_final) usually arrives on a
// result whose transcript is EMPTY — the silence after the user
// stops talking. Empty finals must still reach the speech_final
// check below or hands-free utterances never complete.
if (transcript) {
transcriptBufferRef.current += (transcriptBufferRef.current ? ' ' : '') + transcript;
interimRef.current = '';
}
// Hands-free mode: an endpoint may complete the utterance —
// immediately for complete thoughts, after a short hold for
// mid-sentence trails.
if (continuousCbRef.current && data.speech_final) {
maybeEndUtterance();
return;
}
if (transcript) {
setInterimText(transcriptBufferRef.current);
}
} else if (transcript) {
interimRef.current = transcript;
setInterimText(transcriptBufferRef.current + (transcriptBufferRef.current ? ' ' : '') + transcript);
}
@ -127,8 +225,17 @@ export function useVoiceMode() {
ws.onclose = () => {
console.log('[voice] WebSocket closed');
wsRef.current = null;
// A hands-free call is long-lived — if the socket drops while the
// call is still on, reconnect instead of silently going deaf.
if (continuousCbRef.current) {
setTimeout(() => {
if (continuousCbRef.current && !wsRef.current) {
void connectWs(true);
}
}, 1000);
}
};
}, [refreshAuth]);
}, [refreshAuth, maybeEndUtterance]);
const waitForWsOpen = useCallback(async (timeoutMs = 1500): Promise<boolean> => {
const ws = wsRef.current;
@ -191,6 +298,16 @@ export function useVoiceMode() {
wsRef.current.close();
wsRef.current = null;
}
continuousCbRef.current = null;
pausedRef.current = false;
if (holdTimerRef.current) {
clearTimeout(holdTimerRef.current);
holdTimerRef.current = null;
}
if (keepAliveTimerRef.current) {
clearInterval(keepAliveTimerRef.current);
keepAliveTimerRef.current = null;
}
audioBufferRef.current = [];
audioLevelsRef.current = [];
audioPeakRef.current = 0;
@ -200,7 +317,7 @@ export function useVoiceMode() {
setState('idle');
}, [stopInputCapture]);
const start = useCallback(async () => {
const start = useCallback(async (continuous = false) => {
if (state !== 'idle') return;
transcriptBufferRef.current = '';
@ -235,7 +352,7 @@ export function useVoiceMode() {
console.error('Microphone access denied:', err);
return null;
}),
connectWs(),
connectWs(continuous),
]);
if (!stream) {
@ -259,6 +376,9 @@ export function useVoiceMode() {
processorRef.current = processor;
processor.onaudioprocess = (e) => {
// Paused (assistant is speaking in a call): drop mic audio so the
// assistant's own TTS never gets transcribed back at it.
if (pausedRef.current) return;
const float32 = e.inputBuffer.getChannelData(0);
const int16 = new Int16Array(float32.length);
let sumSquares = 0;
@ -283,8 +403,13 @@ export function useVoiceMode() {
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(buffer);
} else {
// WebSocket still connecting — buffer the audio
// WebSocket still connecting (or reconnecting mid-call) —
// buffer the audio, bounded so an unreachable server during a
// long call can't grow it without limit (~30s at 64ms/chunk).
audioBufferRef.current.push(buffer);
if (audioBufferRef.current.length > 500) {
audioBufferRef.current.shift();
}
}
};
@ -318,10 +443,46 @@ export function useVoiceMode() {
stopAudioCapture();
}, [stopAudioCapture]);
/**
* Hands-free (call) mode: listen continuously and invoke `onUtterance`
* with each completed utterance. Runs until cancel()/stop.
*/
const startContinuous = useCallback(async (onUtterance: (text: string) => void) => {
continuousCbRef.current = onUtterance;
await start(true);
}, [start]);
/**
* Mute/unmute the continuous stream (used while the assistant is
* thinking/speaking). Keeps the Deepgram socket alive with KeepAlives and
* discards any half-heard utterance from before the pause.
*/
const setPaused = useCallback((paused: boolean) => {
if (pausedRef.current === paused) return;
pausedRef.current = paused;
if (paused) {
if (holdTimerRef.current) {
clearTimeout(holdTimerRef.current);
holdTimerRef.current = null;
}
transcriptBufferRef.current = '';
interimRef.current = '';
setInterimText('');
keepAliveTimerRef.current = setInterval(() => {
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ type: 'KeepAlive' }));
}
}, KEEPALIVE_INTERVAL_MS);
} else if (keepAliveTimerRef.current) {
clearInterval(keepAliveTimerRef.current);
keepAliveTimerRef.current = null;
}
}, []);
/** Pre-cache auth details so mic click skips IPC round-trips */
const warmup = useCallback(() => {
refreshAuth().catch(() => {});
}, [refreshAuth]);
return { state, interimText, audioLevelsRef, start, submit, cancel, warmup };
return { state, interimText, audioLevelsRef, start, submit, cancel, warmup, startContinuous, setPaused };
}

View file

@ -1,4 +1,5 @@
import { useCallback, useRef, useState } from 'react';
import { useCallback, useEffect, useRef, useState } from 'react';
import { dispatchCreditReplenished } from '@/lib/credit-status';
export type TTSState = 'idle' | 'synthesizing' | 'speaking';
@ -8,20 +9,32 @@ interface SynthesizedAudio {
function synthesize(text: string): Promise<SynthesizedAudio> {
return window.ipc.invoke('voice:synthesize', { text }).then(
(result: { audioBase64: string; mimeType: string }) => ({
dataUrl: `data:${result.mimeType};base64,${result.audioBase64}`,
})
(result: { audioBase64: string; mimeType: string }) => {
// A successful Rowboat voice synth is a cost-incurring call that
// returned OK, so it proves credits are available again.
dispatchCreditReplenished();
return { dataUrl: `data:${result.mimeType};base64,${result.audioBase64}` };
}
);
}
function playAudio(dataUrl: string, audioRef: React.MutableRefObject<HTMLAudioElement | null>): Promise<void> {
function playAudio(
dataUrl: string,
audioRef: React.MutableRefObject<HTMLAudioElement | null>,
onAudioElement?: (audio: HTMLAudioElement) => void
): Promise<void> {
return new Promise<void>((resolve, reject) => {
const audio = new Audio(dataUrl);
audioRef.current = audio;
onAudioElement?.(audio);
audio.onended = () => {
console.log('[tts] audio ended');
resolve();
};
// pause() (from cancel) must settle this promise too, or the queue
// loop stays parked on it forever. Natural end also fires 'pause'
// just before 'ended'; double-resolve is harmless.
audio.onpause = () => resolve();
audio.onerror = (e) => {
console.error('[tts] audio error:', e);
reject(new Error('Audio playback failed'));
@ -35,47 +48,282 @@ function playAudio(dataUrl: string, audioRef: React.MutableRefObject<HTMLAudioEl
});
}
/** A queue entry: text to synthesize, or a ready-to-play audio URL (e.g. a bundled clip). */
type QueueItem = { text: string } | { url: string };
type TtsChunkMsg = { requestId: string; chunkBase64?: string; done: boolean; error?: string };
export function useVoiceTTS() {
const [state, setState] = useState<TTSState>('idle');
const audioRef = useRef<HTMLAudioElement | null>(null);
const queueRef = useRef<string[]>([]);
const queueRef = useRef<QueueItem[]>([]);
const processingRef = useRef(false);
// Pre-fetched audio ready to play immediately
const prefetchedRef = useRef<Promise<SynthesizedAudio> | null>(null);
// Streaming synthesis: per-request chunk handlers + the in-flight request
// id (so cancel() can abort the main-process fetch).
const streamHandlersRef = useRef<Map<string, (msg: TtsChunkMsg) => void>>(new Map());
const activeStreamIdRef = useRef<string | null>(null);
// Bumped by cancel(). A queue loop that awaited across a cancel sees a
// stale generation and exits instead of playing audio that was cancelled
// while still synthesizing (which would overlap the next utterance).
const generationRef = useRef(0);
// Web Audio analyser tap for lip-sync (talking head)
const audioCtxRef = useRef<AudioContext | null>(null);
const analyserRef = useRef<AnalyserNode | null>(null);
const levelBufferRef = useRef<Uint8Array<ArrayBuffer> | null>(null);
// Route playback through an AnalyserNode so consumers can read the live
// output level. If Web Audio wiring fails, the element still plays directly.
const connectAnalyser = useCallback((audio: HTMLAudioElement) => {
try {
let ctx = audioCtxRef.current;
if (!ctx) {
ctx = new AudioContext();
audioCtxRef.current = ctx;
const analyser = ctx.createAnalyser();
analyser.fftSize = 512;
analyser.smoothingTimeConstant = 0.5;
analyser.connect(ctx.destination);
analyserRef.current = analyser;
}
if (ctx.state === 'suspended') {
void ctx.resume();
}
const source = ctx.createMediaElementSource(audio);
source.connect(analyserRef.current!);
// Detach once this chunk is done (ended, cancelled via pause, or
// failed) so source nodes don't accumulate over a long session.
const disconnect = () => {
try {
source.disconnect();
} catch {
// already disconnected
}
};
audio.addEventListener('ended', disconnect, { once: true });
audio.addEventListener('pause', disconnect, { once: true });
audio.addEventListener('error', disconnect, { once: true });
} catch (err) {
console.error('[tts] analyser hookup failed:', err);
}
}, []);
// Current output level, 0..1. Safe to call every animation frame.
// Release the audio graph when the owning component unmounts
useEffect(() => () => {
audioCtxRef.current?.close().catch(() => {});
audioCtxRef.current = null;
analyserRef.current = null;
}, []);
// Route streaming TTS chunks to whichever request is waiting for them.
useEffect(() => {
return window.ipc.on('voice:tts-chunk', (msg) => {
streamHandlersRef.current.get(msg.requestId)?.(msg);
});
}, []);
/**
* Streaming synthesis + playback via MediaSource: audio starts on the
* first chunk instead of after the full body. Rejects (for caller
* fallback to non-streaming synth) if the stream fails before any audio
* arrived; resolves when playback finishes.
*/
const streamSynthesizeAndPlay = useCallback((text: string, onStarted: () => void): Promise<void> => {
return new Promise<void>((resolve, reject) => {
if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported('audio/mpeg')) {
reject(new Error('MSE audio/mpeg unsupported'));
return;
}
const requestId = `tts-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
const mediaSource = new MediaSource();
const audio = new Audio();
audio.src = URL.createObjectURL(mediaSource);
audioRef.current = audio;
connectAnalyser(audio);
activeStreamIdRef.current = requestId;
let sourceBuffer: SourceBuffer | null = null;
const pending: Uint8Array[] = [];
let streamDone = false;
let gotAudio = false;
let settled = false;
const cleanup = () => {
streamHandlersRef.current.delete(requestId);
if (activeStreamIdRef.current === requestId) activeStreamIdRef.current = null;
URL.revokeObjectURL(audio.src);
};
const finish = (err?: Error) => {
if (settled) return;
settled = true;
cleanup();
if (err) reject(err);
else resolve();
};
// Drain pending chunks into the SourceBuffer one at a time
// (appendBuffer is async; only one append may be in flight).
const pump = () => {
if (!sourceBuffer || sourceBuffer.updating || settled) return;
const chunk = pending.shift();
if (chunk) {
try {
sourceBuffer.appendBuffer(chunk as BufferSource);
} catch (e) {
finish(e as Error);
}
return;
}
if (streamDone && mediaSource.readyState === 'open') {
try {
mediaSource.endOfStream();
} catch { /* already ended */ }
}
};
mediaSource.addEventListener('sourceopen', () => {
try {
sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg');
} catch (e) {
finish(e as Error);
return;
}
sourceBuffer.addEventListener('updateend', pump);
pump();
}, { once: true });
streamHandlersRef.current.set(requestId, (msg) => {
if (msg.error && !gotAudio) {
streamDone = true;
finish(new Error(msg.error));
return;
}
if (msg.chunkBase64) {
gotAudio = true;
const bin = atob(msg.chunkBase64);
const bytes = new Uint8Array(bin.length);
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
pending.push(bytes);
pump();
}
if (msg.done) {
streamDone = true;
pump();
}
});
audio.addEventListener('playing', () => onStarted(), { once: true });
audio.onended = () => finish();
// pause() (from cancel) must settle this promise too; natural end
// also fires 'pause' just before 'ended'; double-settle is a no-op.
audio.onpause = () => finish();
audio.onerror = () => finish(new Error('stream playback failed'));
window.ipc
.invoke('voice:synthesizeStreamStart', { requestId, text })
.then((res) => {
if (!res.ok) finish(new Error(res.error || 'stream start failed'));
})
.catch((e) => finish(e as Error));
// Starts as soon as the first appended data is decodable.
audio.play().catch(() => { /* surfaced via onerror / chunk error */ });
// Nothing arrived at all — bail so the caller can fall back.
setTimeout(() => {
if (!gotAudio && !settled) finish(new Error('stream timeout'));
}, 10_000);
});
}, [connectAnalyser]);
const getLevel = useCallback((): number => {
const analyser = analyserRef.current;
if (!analyser) return 0;
let buffer = levelBufferRef.current;
if (!buffer || buffer.length !== analyser.fftSize) {
buffer = new Uint8Array(analyser.fftSize);
levelBufferRef.current = buffer;
}
analyser.getByteTimeDomainData(buffer);
let sum = 0;
for (let i = 0; i < buffer.length; i++) {
const d = (buffer[i] - 128) / 128;
sum += d * d;
}
const rms = Math.sqrt(sum / buffer.length);
return Math.min(1, rms * 4);
}, []);
const processQueue = useCallback(async () => {
if (processingRef.current) return;
processingRef.current = true;
const gen = generationRef.current;
// Kick off full-body pre-fetch for the next queued text while the
// current one plays — keeps sentence-to-sentence playback gapless.
const prefetchNext = () => {
const next = queueRef.current[0];
if (next && 'text' in next && next.text.trim() && !prefetchedRef.current) {
console.log('[tts] pre-fetching next:', next.text.substring(0, 80));
prefetchedRef.current = synthesize(next.text);
}
};
while (queueRef.current.length > 0) {
const text = queueRef.current.shift()!;
if (!text.trim()) continue;
const item = queueRef.current.shift()!;
if ('text' in item && !item.text.trim()) continue;
// Cold start (nothing playing, nothing pre-fetched): stream the
// synthesis so audio begins on the first chunk instead of after
// the full body — this is where first-response latency lives.
if ('text' in item && !prefetchedRef.current) {
setState('synthesizing');
console.log('[tts] stream-synthesizing:', item.text.substring(0, 80));
try {
await streamSynthesizeAndPlay(item.text, () => {
if (generationRef.current !== gen) return;
setState('speaking');
prefetchNext();
});
if (generationRef.current !== gen) return;
continue;
} catch (err) {
if (generationRef.current !== gen) return;
console.error('[tts] stream failed, falling back to full synth:', err);
// fall through to the non-streaming path below
}
}
try {
// Use pre-fetched result if available, otherwise synthesize now
// Pre-recorded URL plays as-is; text uses the pre-fetched
// result if available, otherwise synthesizes now.
let audioPromise: Promise<SynthesizedAudio>;
if (prefetchedRef.current) {
if ('url' in item) {
audioPromise = Promise.resolve({ dataUrl: item.url });
} else if (prefetchedRef.current) {
console.log('[tts] using pre-fetched audio');
audioPromise = prefetchedRef.current;
prefetchedRef.current = null;
} else {
setState('synthesizing');
console.log('[tts] synthesizing:', text.substring(0, 80));
audioPromise = synthesize(text);
console.log('[tts] synthesizing:', item.text.substring(0, 80));
audioPromise = synthesize(item.text);
}
const audio = await audioPromise;
// Cancelled while synthesizing — cancel() already reset all
// state (and a new loop may be running), so just bail.
if (generationRef.current !== gen) return;
setState('speaking');
// Kick off pre-fetch for next chunk while this one plays
const nextText = queueRef.current[0];
if (nextText?.trim()) {
console.log('[tts] pre-fetching next:', nextText.substring(0, 80));
prefetchedRef.current = synthesize(nextText);
}
prefetchNext();
await playAudio(audio.dataUrl, audioRef);
await playAudio(audio.dataUrl, audioRef, connectAnalyser);
if (generationRef.current !== gen) return;
} catch (err) {
if (generationRef.current !== gen) return;
console.error('[tts] error:', err);
prefetchedRef.current = null;
}
@ -85,17 +333,33 @@ export function useVoiceTTS() {
prefetchedRef.current = null;
processingRef.current = false;
setState('idle');
}, []);
}, [connectAnalyser, streamSynthesizeAndPlay]);
const speak = useCallback((text: string) => {
console.log('[tts] speak() called:', text.substring(0, 80));
queueRef.current.push(text);
queueRef.current.push({ text });
processQueue();
}, [processQueue]);
// Play a pre-recorded clip (e.g. bundled tour narration) through the same
// queue, so lip-sync levels, state, and cancel() all work unchanged.
const speakUrl = useCallback((url: string) => {
console.log('[tts] speakUrl() called:', url.substring(0, 120));
queueRef.current.push({ url });
processQueue();
}, [processQueue]);
const cancel = useCallback(() => {
generationRef.current++;
queueRef.current = [];
prefetchedRef.current = null;
// Abort any in-flight streaming synthesis in the main process.
if (activeStreamIdRef.current) {
void window.ipc
.invoke('voice:synthesizeStreamCancel', { requestId: activeStreamIdRef.current })
.catch(() => {});
activeStreamIdRef.current = null;
}
if (audioRef.current) {
audioRef.current.pause();
audioRef.current = null;
@ -104,5 +368,5 @@ export function useVoiceTTS() {
setState('idle');
}, []);
return { state, speak, cancel };
return { state, speak, speakUrl, cancel, getLevel };
}

View file

@ -53,6 +53,10 @@ export function chatMessageSent(props: {
})
}
export function appOpened(folder: string) {
posthog.capture('app_opened', { folder })
}
export function oauthConnected(provider: string) {
posthog.capture('oauth_connected', { provider })
}
@ -65,6 +69,26 @@ export function voiceInputStarted() {
posthog.capture('voice_input_started')
}
export function callStarted(preset: 'voice' | 'video' | 'share' | 'practice') {
posthog.capture('call_started', { preset })
}
// Voice-to-voice latency breakdown for one call turn (all milliseconds):
// utterance accepted → message submitted → first TTS speak() → audio playing.
export function callTurnLatency(props: {
endpointToSubmitMs: number
submitToSpeakMs: number
speakToAudioMs: number
totalMs: number
}) {
posthog.capture('call_turn_latency', {
endpoint_to_submit_ms: Math.round(props.endpointToSubmitMs),
submit_to_speak_ms: Math.round(props.submitToSpeakMs),
speak_to_audio_ms: Math.round(props.speakToAudioMs),
total_ms: Math.round(props.totalMs),
})
}
export function searchExecuted(types: string[]) {
posthog.capture('search_executed', { types })
}

View file

@ -1,17 +1,20 @@
export const BILLING_ERROR_PATTERNS = [
{
kind: 'subscription_required',
pattern: /upgrade required/i,
title: 'A subscription is required',
subtitle: 'Get started with a plan to access AI features in Rowboat.',
cta: 'Subscribe',
},
{
kind: 'out_of_credits',
pattern: /not enough credits/i,
title: "You've run out of credits",
subtitle: 'Upgrade your plan for more usage. Daily usage resets at 00:00 UTC.',
cta: 'Upgrade plan',
},
{
kind: 'subscription_inactive',
pattern: /subscription not active/i,
title: 'Your subscription is inactive',
subtitle: 'Reactivate your subscription to continue using AI features.',

View file

@ -0,0 +1,30 @@
// Tiny synthesized UI sounds for calls — no audio assets, one lazy context.
let ctx: AudioContext | null = null
/**
* Soft rising blip played the instant an utterance is accepted sub-second
* acknowledgment makes the (still ongoing) model turn feel responsive
* instead of dead air.
*/
export function playAckCue() {
try {
if (!ctx) ctx = new AudioContext()
if (ctx.state === 'suspended') void ctx.resume()
const t = ctx.currentTime
const osc = ctx.createOscillator()
const gain = ctx.createGain()
osc.type = 'sine'
osc.frequency.setValueAtTime(880, t)
osc.frequency.exponentialRampToValueAtTime(1320, t + 0.08)
gain.gain.setValueAtTime(0.0001, t)
gain.gain.exponentialRampToValueAtTime(0.08, t + 0.015)
gain.gain.exponentialRampToValueAtTime(0.0001, t + 0.12)
osc.connect(gain)
gain.connect(ctx.destination)
osc.start(t)
osc.stop(t + 0.13)
} catch {
// cosmetic — never let a sound failure affect the call
}
}

View file

@ -10,6 +10,9 @@ export interface MessageAttachment {
mimeType: string
size?: number
thumbnailUrl?: string
/** Live webcam frame from video chat mode rendered as a compact filmstrip.
* Carries no path; thumbnailUrl holds the frame as a data: URL. */
isVideoFrame?: boolean
}
export interface ChatMessage {
@ -118,6 +121,19 @@ export const normalizeToolOutput = (
return output
}
export const getToolErrorText = (tool: ToolCall): string | undefined => {
if (tool.status !== 'error') return undefined
if (typeof tool.result === 'string' && tool.result.trim()) return tool.result
if (tool.result !== undefined) {
try {
return JSON.stringify(tool.result, null, 2)
} catch {
// Fall through to the generic label for non-serializable legacy values.
}
}
return 'Tool error'
}
export type WebSearchCardResult = { title: string; url: string; description: string }
export type WebSearchCardData = {
@ -198,6 +214,22 @@ const summarizeFilterUpdates = (updates: Record<string, unknown>): string => {
return parts.length > 0 ? parts.join(', ') : 'Updated view'
}
const APP_VIEW_LABELS: Record<string, string> = {
home: 'home',
email: 'email',
meetings: 'meetings',
'live-notes': 'live notes',
'bg-tasks': 'background agents',
'chat-history': 'chat history',
knowledge: 'knowledge',
workspace: 'workspace',
code: 'code',
bases: 'bases',
graph: 'graph',
}
const appViewLabel = (view: unknown): string => APP_VIEW_LABELS[view as string] ?? String(view ?? 'view')
export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null => {
if (tool.name !== 'app-navigation') return null
const result = tool.result as Record<string, unknown> | undefined
@ -209,7 +241,10 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null =
const action = input.action as string
switch (action) {
case 'open-note': return { action, label: `Opening ${(input.path as string || '').split('/').pop()?.replace(/\.md$/, '') || 'note'}...` }
case 'open-view': return { action, label: `Opening ${input.view} view...` }
case 'open-view': return { action, label: `Opening ${appViewLabel(input.view)}...` }
case 'open-app': return { action, label: `Opening ${input.appId || 'app'}...` }
case 'read-view': return { action, label: `Reading ${appViewLabel(input.view)}...` }
case 'open-item': return { action, label: 'Opening...' }
case 'update-base-view': return { action, label: 'Updating view...' }
case 'create-base': return { action, label: `Creating "${input.name}"...` }
case 'get-base-state': return null // renders as normal tool block
@ -224,7 +259,33 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null =
return { action: 'open-note', label: `Opened ${name}` }
}
case 'open-view':
return { action: 'open-view', label: `Opened ${result.view} view` }
return { action: 'open-view', label: `Opened ${appViewLabel(result.view)}` }
case 'open-app':
return { action: 'open-app', label: `Opened ${result.appName || result.appId || 'app'}` }
case 'read-view': {
const counted =
(result.threads as unknown[] | undefined)?.length ??
(result.agents as unknown[] | undefined)?.length ??
(result.sessions as unknown[] | undefined)?.length
return {
action: 'read-view',
label: counted !== undefined
? `Read ${appViewLabel(result.view)} (${counted} item${counted === 1 ? '' : 's'})`
: `Read ${appViewLabel(result.view)}`,
}
}
case 'open-item': {
switch (result.kind) {
case 'email-thread': return { action: 'open-item', label: 'Opened email thread' }
case 'note': {
const name = (result.path as string || '').split('/').pop()?.replace(/\.md$/, '') || 'note'
return { action: 'open-item', label: `Opened ${name}` }
}
case 'bg-task': return { action: 'open-item', label: `Opened agent "${result.taskName}"` }
case 'session': return { action: 'open-item', label: 'Opened chat' }
default: return { action: 'open-item', label: 'Opened item' }
}
}
case 'update-base-view':
return {
action: 'update-base-view',

View file

@ -0,0 +1,52 @@
import { useSyncExternalStore } from 'react'
import type { CodeRunEvent, CodeRunFeedEvent } from '@x/shared/src/code-mode.js'
// Renderer half of the ephemeral CodeRunFeed side-channel: buffers the live
// `codeRun:events` broadcast per toolCallId so tool cards can render a
// code_agent_run's activity while it streams. Module-level (not tied to any
// session store) so the buffer survives session switches mid-run. Nothing here
// is persisted — on settle the durable code-run-events-batch in the turn
// record supersedes the buffer, which is then dropped.
const buffers = new Map<string, CodeRunEvent[]>()
const listeners = new Set<() => void>()
const EMPTY: CodeRunEvent[] = []
// Backstop so abandoned runs can't grow the map forever (a run's buffer is
// normally dropped explicitly once its durable batch lands).
const MAX_TRACKED_RUNS = 32
let attached = false
function ensureAttached(): void {
if (attached) return
attached = true
window.ipc.on('codeRun:events', ((raw: unknown) => {
const { toolCallId, event } = raw as CodeRunFeedEvent
if (!toolCallId || !event) return
if (!buffers.has(toolCallId) && buffers.size >= MAX_TRACKED_RUNS) {
const oldest = buffers.keys().next().value
if (oldest !== undefined) buffers.delete(oldest)
}
// Immutable append: useSyncExternalStore consumers compare by reference.
buffers.set(toolCallId, [...(buffers.get(toolCallId) ?? EMPTY), event])
for (const listener of [...listeners]) listener()
}) as never)
}
function subscribe(onChange: () => void): () => void {
ensureAttached()
listeners.add(onChange)
return () => {
listeners.delete(onChange)
}
}
export function clearCodeRunBuffer(toolCallId: string): void {
if (buffers.delete(toolCallId)) {
for (const listener of [...listeners]) listener()
}
}
// Live events for one code_agent_run tool call, empty once the durable batch
// takes over (or if the buffer never existed — e.g. after an app reload).
export function useCodeRunFeed(toolCallId: string): CodeRunEvent[] {
return useSyncExternalStore(subscribe, () => buffers.get(toolCallId) ?? EMPTY)
}

View file

@ -0,0 +1,26 @@
import type { BillingInfo } from '@x/shared/dist/billing.js'
/**
* A user is "out of credits" when EITHER the daily or the monthly bucket is
* exhausted. Either exhaustion is what triggers the backend's "not enough
* credits" API error, and this mirrors the two usage bars shown in Settings
* (Plan usage = monthly, Daily use = daily). `availableCredits` already comes
* down in the /v1/me payload, so no extra API work is needed.
*/
export function isOutOfCredits(billing: BillingInfo): boolean {
return billing.daily.availableCredits <= 0 || billing.monthly.availableCredits <= 0
}
/** Fired when we learn the user is out of credits (billing data or a usage API error). */
export const CREDIT_EXHAUSTED_EVENT = 'credit-status:exhausted'
/** Fired when a successful cost-incurring call (LLM / voice) proves credits are available again. */
export const CREDIT_REPLENISHED_EVENT = 'credit-status:replenished'
export function dispatchCreditExhausted(): void {
window.dispatchEvent(new Event(CREDIT_EXHAUSTED_EVENT))
}
export function dispatchCreditReplenished(): void {
window.dispatchEvent(new Event(CREDIT_REPLENISHED_EVENT))
}

View file

@ -41,6 +41,9 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
filename?: string
mimeType?: string
size?: number
data?: string
mediaType?: string
source?: string
toolCallId?: string
toolName?: string
arguments?: unknown
@ -52,13 +55,24 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
.join('')
const attachmentParts = parts.filter((p) => p.type === 'attachment' && p.path)
if (attachmentParts.length > 0) {
msgAttachments = attachmentParts.map((p) => ({
path: p.path!,
filename: p.filename || p.path!.split('/').pop() || p.path!,
mimeType: p.mimeType || 'application/octet-stream',
size: p.size,
}))
// Video-mode webcam frames — inline base64 image parts, shown as a filmstrip
const imageParts = parts.filter((p) => p.type === 'image' && p.data)
if (attachmentParts.length > 0 || imageParts.length > 0) {
msgAttachments = [
...attachmentParts.map((p) => ({
path: p.path!,
filename: p.filename || p.path!.split('/').pop() || p.path!,
mimeType: p.mimeType || 'application/octet-stream',
size: p.size,
})),
...imageParts.map((p, index) => ({
path: '',
filename: `${p.source === 'screen' ? 'screen' : 'camera'}-frame-${index + 1}.jpg`,
mimeType: p.mediaType || 'image/jpeg',
thumbnailUrl: `data:${p.mediaType || 'image/jpeg'};base64,${p.data}`,
isVideoFrame: true,
})),
]
}
if (msg.role === 'assistant') {

View file

@ -96,6 +96,25 @@ describe('voice output', () => {
expect(overlay.voiceSegments).toEqual(['hello there', 'bye'])
})
it('emits an early clause from a long open block, then the remainder on close', () => {
let overlay = emptyOverlay()
const longClause = 'Okay so the first thing I would look at here is the error message,'
overlay = applyOverlay(overlay, delta(`<voice>${longClause} because`))
// Open block crossed the early-speech threshold at a clause boundary.
expect(overlay.voiceSegments).toEqual([longClause])
overlay = applyOverlay(overlay, delta(' it tells you the root cause.</voice>'))
// Remainder only — the early clause is not repeated.
expect(overlay.voiceSegments).toEqual([longClause, 'because it tells you the root cause.'])
})
it('does not emit early clauses from short open blocks', () => {
let overlay = emptyOverlay()
overlay = applyOverlay(overlay, delta('<voice>Sure, one sec'))
expect(overlay.voiceSegments).toEqual([])
overlay = applyOverlay(overlay, delta('.</voice>'))
expect(overlay.voiceSegments).toEqual(['Sure, one sec.'])
})
it('keeps segments but resets the scan on model_call_completed', () => {
let overlay = emptyOverlay()
overlay = applyOverlay(overlay, delta('<voice>one</voice>'))
@ -171,6 +190,117 @@ describe('buildTurnConversation', () => {
expect(items.map((i) => i.status)).toEqual(['pending', 'running'])
})
it('uses the tool result envelope to determine error status', () => {
const state = reduceTurn([
created(T1, S1),
requested(T1, 0),
completed(
T1,
0,
assistantCalls(
toolCallPart('flagged', 'echo'),
toolCallPart('normal', 'echo'),
),
),
invocation(T1, 'flagged', 'echo'),
{
type: 'tool_result',
turnId: T1,
ts: TS,
toolCallId: 'flagged',
toolName: 'echo',
source: 'sync',
result: { output: 'boom', isError: true },
},
invocation(T1, 'normal', 'echo'),
toolResult(T1, 'normal', 'echo', { success: false, message: 'payload data' }),
])
const items = buildTurnConversation(state).filter(isToolCall)
expect(items.map((i) => i.status)).toEqual(['error', 'completed'])
})
it('derives the code-run timeline from the settle-time batch and asks from request events', () => {
const codeProgress = (progress: unknown): TEvent => ({
type: 'tool_progress',
turnId: T1,
ts: TS,
toolCallId: 'cr1',
source: 'sync',
progress: progress as never,
})
const state = reduceTurn([
created(T1, S1),
requested(T1, 0),
completed(T1, 0, assistantCalls(toolCallPart('cr1', 'code_agent_run', { agent: 'codex' }))),
invocation(T1, 'cr1', 'code_agent_run'),
codeProgress({
kind: 'code-run-permission-request',
requestId: 'cpr-1',
ask: { toolCallId: 'x', title: 'write file', options: [] },
}),
codeProgress({
kind: 'code-run-events',
events: [
{ type: 'message', role: 'agent', text: 'hi' },
{ type: 'tool_call', id: 'x', title: 'write file' },
],
}),
])
const tool = buildTurnConversation(state).filter(isToolCall)[0]
expect(tool.status).toBe('running')
expect(tool.codeRunEvents?.map((e) => e.type)).toEqual(['message', 'tool_call'])
expect(tool.pendingCodePermission?.requestId).toBe('cpr-1')
})
it('clears the pending code permission on the resolved marker and on tool result', () => {
const codeProgress = (toolCallId: string, progress: unknown): TEvent => ({
type: 'tool_progress',
turnId: T1,
ts: TS,
toolCallId,
source: 'sync',
progress: progress as never,
})
const ask = { toolCallId: 'x', title: 'write file', options: [] }
// resolved: the durable marker pairs off the request mid-run
const resolvedState = reduceTurn([
created(T1, S1),
requested(T1, 0),
completed(T1, 0, assistantCalls(toolCallPart('cr1', 'code_agent_run'))),
invocation(T1, 'cr1', 'code_agent_run'),
codeProgress('cr1', { kind: 'code-run-permission-request', requestId: 'cpr-1', ask }),
codeProgress('cr1', { kind: 'code-run-permission-resolved' }),
])
const resolved = buildTurnConversation(resolvedState).filter(isToolCall)[0]
expect(resolved.pendingCodePermission).toBeUndefined()
// a second ask after the first resolution is pending again
const secondAskState = reduceTurn([
created(T1, S1),
requested(T1, 0),
completed(T1, 0, assistantCalls(toolCallPart('cr1', 'code_agent_run'))),
invocation(T1, 'cr1', 'code_agent_run'),
codeProgress('cr1', { kind: 'code-run-permission-request', requestId: 'cpr-1', ask }),
codeProgress('cr1', { kind: 'code-run-permission-resolved' }),
codeProgress('cr1', { kind: 'code-run-permission-request', requestId: 'cpr-2', ask }),
])
const secondAsk = buildTurnConversation(secondAskState).filter(isToolCall)[0]
expect(secondAsk.pendingCodePermission?.requestId).toBe('cpr-2')
// settled: an unanswered ask must not survive the tool's terminal result
const settledState = reduceTurn([
created(T1, S1),
requested(T1, 0),
completed(T1, 0, assistantCalls(toolCallPart('cr2', 'code_agent_run'))),
invocation(T1, 'cr2', 'code_agent_run'),
codeProgress('cr2', { kind: 'code-run-permission-request', requestId: 'cpr-2', ask }),
toolResult(T1, 'cr2', 'code_agent_run', { success: false, stopReason: 'cancelled' }),
])
const settled = buildTurnConversation(settledState).filter(isToolCall)[0]
expect(settled.pendingCodePermission).toBeUndefined()
expect(settled.status).toBe('completed')
})
it('renders user attachments and a failed turn as an error item', () => {
const input = {
role: 'user' as const,

View file

@ -13,6 +13,7 @@ import {
type TurnState,
type TurnStreamEvent,
} from '@x/shared/src/turns.js'
import type { CodeRunEvent, PermissionAsk } from '@x/shared/src/code-mode.js'
import type {
ChatMessage,
ConversationItem,
@ -34,14 +35,20 @@ export type LiveOverlay = {
text: string
reasoning: string
toolOutput: Record<string, string>
// Contents of completed <voice>…</voice> blocks seen while streaming, in
// order, monotonically growing for the lifetime of the overlay (i.e. one
// active turn). Consumers speak segments beyond what they've already
// spoken; the overlay reset on turn switch starts a fresh list.
// Speakable segments seen while streaming, in order, monotonically growing
// for the lifetime of the overlay (i.e. one active turn). Usually the
// contents of completed <voice>…</voice> blocks, but a long still-open
// block may emit an early clause (see EARLY_SPEECH_MIN_CHARS) so speech
// can start before the sentence finishes generating. Consumers speak
// segments beyond what they've already spoken; the overlay reset on turn
// switch starts a fresh list.
voiceSegments: string[]
// Scan cursor into `text` — everything before it has been checked for
// complete voice blocks.
voiceScanIndex: number
// Chars of the currently-open voice block's content already emitted as an
// early clause — the block's remainder (on close) excludes them.
voicePartialConsumed: number
}
export const emptyOverlay = (): LiveOverlay => ({
@ -50,6 +57,7 @@ export const emptyOverlay = (): LiveOverlay => ({
toolOutput: {},
voiceSegments: [],
voiceScanIndex: 0,
voicePartialConsumed: 0,
})
// The model emits <voice>…</voice> around speakable text when voice output
@ -59,6 +67,17 @@ export function stripVoiceTags(text: string): string {
}
const VOICE_BLOCK = /<voice>([\s\S]*?)<\/voice>/g
const VOICE_OPEN_TAG = '<voice>'
// Early speech: once an open block has this many unconsumed chars, its last
// complete clause is emitted immediately instead of waiting for </voice> —
// TTS starts on the first clause while the rest of the sentence generates.
const EARLY_SPEECH_MIN_CHARS = 60
// ...but never emit a fragment shorter than this (prosody suffers).
const EARLY_SPEECH_MIN_EMIT = 30
// Clause boundaries (punctuation, optionally inside closing quote/paren,
// followed by whitespace or end-of-buffer).
const CLAUSE_BOUNDARY = /[,;:.!?…—]["')\]]*(?=\s|$)/g
// Accumulates deltas; canonical durable events supersede the buffers (the
// committed transcript now contains what was streaming).
@ -68,15 +87,43 @@ export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): Live
const text = overlay.text + event.delta
// Extract complete voice blocks past the scan cursor. Incomplete
// blocks (opening tag seen, closing not yet) stay unconsumed until a
// later delta completes them.
// later delta completes them. The first complete block may have had an
// early clause emitted while it was open — skip those chars.
const segments: string[] = []
let scanIndex = overlay.voiceScanIndex
let partialConsumed = overlay.voicePartialConsumed
VOICE_BLOCK.lastIndex = scanIndex
for (let m = VOICE_BLOCK.exec(text); m; m = VOICE_BLOCK.exec(text)) {
const content = m[1].trim()
const content = m[1].slice(partialConsumed).trim()
partialConsumed = 0
if (content) segments.push(content)
scanIndex = m.index + m[0].length
}
// Early speech: if a voice block is still open and has accumulated a
// long unconsumed run, emit its last complete clause now — speech can
// start while the rest of the sentence is still generating.
const openIdx = text.indexOf(VOICE_OPEN_TAG, scanIndex)
if (openIdx !== -1) {
const unconsumed = text.slice(openIdx + VOICE_OPEN_TAG.length + partialConsumed)
if (unconsumed.length >= EARLY_SPEECH_MIN_CHARS) {
let lastBoundaryEnd = -1
CLAUSE_BOUNDARY.lastIndex = 0
for (let b = CLAUSE_BOUNDARY.exec(unconsumed); b; b = CLAUSE_BOUNDARY.exec(unconsumed)) {
lastBoundaryEnd = b.index + b[0].length
}
if (lastBoundaryEnd >= EARLY_SPEECH_MIN_EMIT) {
const clause = unconsumed.slice(0, lastBoundaryEnd).trim()
if (clause) segments.push(clause)
partialConsumed += lastBoundaryEnd
}
}
} else {
// No open block — any partial bookkeeping belongs to a block that
// has since closed.
partialConsumed = 0
}
return {
...overlay,
text,
@ -84,12 +131,13 @@ export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): Live
? { voiceSegments: [...overlay.voiceSegments, ...segments] }
: {}),
voiceScanIndex: scanIndex,
voicePartialConsumed: partialConsumed,
}
}
case 'reasoning_delta':
return { ...overlay, reasoning: overlay.reasoning + event.delta }
case 'model_call_completed':
return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0 }
return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0, voicePartialConsumed: 0 }
case 'tool_progress': {
const progress = event.progress
if (
@ -153,11 +201,46 @@ function extractAttachments(content: UserContent): MessageAttachment[] | undefin
}
function toolStatus(tc: ToolCallState): ToolCall['status'] {
if (tc.result) return 'completed'
if (tc.result) return tc.result.result.isError ? 'error' : 'completed'
if (tc.permission && !tc.permission.resolved) return 'pending'
return 'running'
}
// code_agent_run's durable trail in tool_progress (see the publish bridge in
// real-tool-registry.ts): ONE settle-time 'code-run-events' batch carrying the
// whole timeline (the live per-event stream travels over the ephemeral
// CodeRunFeed and never reaches turn state), plus per-ask
// 'code-run-permission-request' / 'code-run-permission-resolved' pairs. An ask
// is pending while requests outnumber resolutions and the tool hasn't settled.
function codeRunViewOf(
tc: ToolCallState,
): Pick<ToolCall, 'codeRunEvents' | 'pendingCodePermission'> {
let events: CodeRunEvent[] | undefined
let pending: { requestId: string; ask: PermissionAsk } | null = null
let unresolved = 0
for (const p of tc.progress) {
const entry = p.progress
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) continue
const kind = (entry as { kind?: unknown }).kind
if (kind === 'code-run-events') {
const batch = (entry as { events?: unknown }).events
if (Array.isArray(batch)) events = batch as CodeRunEvent[]
} else if (kind === 'code-run-permission-request') {
const { requestId, ask } = entry as { requestId?: unknown; ask?: unknown }
if (typeof requestId === 'string' && ask) {
pending = { requestId, ask: ask as PermissionAsk }
unresolved += 1
}
} else if (kind === 'code-run-permission-resolved') {
unresolved -= 1
}
}
return {
...(events && events.length > 0 ? { codeRunEvents: events } : {}),
...(pending && unresolved > 0 && !tc.result ? { pendingCodePermission: pending } : {}),
}
}
// One turn's contribution to the conversation: the user input, then per
// completed model call its text and tool calls (with live status/results).
export function buildTurnConversation(state: TurnState): ConversationItem[] {
@ -211,6 +294,7 @@ export function buildTurnConversation(state: TurnState): ConversationItem[] {
...(tc?.result ? { result: tc.result.result.output as ToolCall['result'] } : {}),
status: tc ? toolStatus(tc) : 'running',
timestamp: ts(),
...(tc ? codeRunViewOf(tc) : {}),
} satisfies ToolCall)
}
}

View file

@ -0,0 +1,109 @@
/**
* Tiny synthesized sound effects for the product tour oar splashes, dock
* bumps, and an arrival fanfare. Everything is generated with Web Audio
* oscillators/noise so no audio assets are needed. All methods fail silently
* if audio is unavailable.
*/
export class TourSounds {
private ctx: AudioContext | null = null
private master: GainNode | null = null
private ensure(): AudioContext | null {
try {
if (!this.ctx) {
this.ctx = new AudioContext()
this.master = this.ctx.createGain()
this.master.gain.value = 0.5
this.master.connect(this.ctx.destination)
}
if (this.ctx.state === 'suspended') void this.ctx.resume()
return this.ctx
} catch {
return null
}
}
/** Short filtered-noise burst with a falling pitch — an oar dipping. */
splash() {
const ctx = this.ensure()
if (!ctx || !this.master) return
const dur = 0.22
const noise = ctx.createBufferSource()
const buffer = ctx.createBuffer(1, Math.ceil(ctx.sampleRate * dur), ctx.sampleRate)
const data = buffer.getChannelData(0)
for (let i = 0; i < data.length; i++) data[i] = Math.random() * 2 - 1
noise.buffer = buffer
const filter = ctx.createBiquadFilter()
filter.type = 'bandpass'
filter.Q.value = 1.2
filter.frequency.setValueAtTime(1600, ctx.currentTime)
filter.frequency.exponentialRampToValueAtTime(350, ctx.currentTime + dur)
const gain = ctx.createGain()
gain.gain.setValueAtTime(0.14, ctx.currentTime)
gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + dur)
noise.connect(filter).connect(gain).connect(this.master)
noise.start()
noise.stop(ctx.currentTime + dur)
}
/** Soft low thump — the boat nudging a dock. */
bump() {
const ctx = this.ensure()
if (!ctx || !this.master) return
const osc = ctx.createOscillator()
osc.type = 'sine'
osc.frequency.setValueAtTime(150, ctx.currentTime)
osc.frequency.exponentialRampToValueAtTime(70, ctx.currentTime + 0.18)
const gain = ctx.createGain()
gain.gain.setValueAtTime(0.2, ctx.currentTime)
gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + 0.25)
osc.connect(gain).connect(this.master)
osc.start()
osc.stop(ctx.currentTime + 0.3)
}
/** Gentle high blip — an email landing in the boat. */
ding() {
const ctx = this.ensure()
if (!ctx || !this.master) return
const osc = ctx.createOscillator()
osc.type = 'sine'
osc.frequency.setValueAtTime(880, ctx.currentTime)
osc.frequency.exponentialRampToValueAtTime(1320, ctx.currentTime + 0.05)
const gain = ctx.createGain()
gain.gain.setValueAtTime(0.08, ctx.currentTime)
gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + 0.3)
osc.connect(gain).connect(this.master)
osc.start()
osc.stop(ctx.currentTime + 0.35)
}
/** Little four-note arpeggio for the tour finale. */
fanfare() {
const ctx = this.ensure()
if (!ctx || !this.master) return
const notes = [523.25, 659.25, 783.99, 1046.5] // C5 E5 G5 C6
notes.forEach((freq, i) => {
const start = ctx.currentTime + i * 0.13
const osc = ctx.createOscillator()
osc.type = 'triangle'
osc.frequency.value = freq
const gain = ctx.createGain()
gain.gain.setValueAtTime(0.0001, start)
gain.gain.exponentialRampToValueAtTime(0.16, start + 0.02)
gain.gain.exponentialRampToValueAtTime(0.001, start + (i === notes.length - 1 ? 0.7 : 0.3))
osc.connect(gain).connect(this.master!)
osc.start(start)
osc.stop(start + 0.8)
})
}
dispose() {
this.ctx?.close().catch(() => {})
this.ctx = null
this.master = null
}
}

View file

@ -6,6 +6,7 @@ import { PostHogProvider } from 'posthog-js/react'
import type { CaptureResult } from 'posthog-js'
import { ThemeProvider } from '@/contexts/theme-context'
import { configureAnalyticsContext } from './lib/analytics'
import { VideoPopout } from '@/components/video-popout'
// Fetch the stable installation ID from main so renderer + main share one
// PostHog distinct_id. Falls back to PostHog's auto-generated anonymous ID
@ -57,4 +58,14 @@ async function bootstrap() {
// The loaded callback applies api_url/app_version once PostHog has initialized.
}
bootstrap()
// The video-mode popout window loads the same bundle with a hash route and
// renders only the mini-call UI — no analytics or app bootstrap needed.
if (window.location.hash === '#video-popout') {
createRoot(document.getElementById('root')!).render(
<StrictMode>
<VideoPopout />
</StrictMode>,
)
} else {
bootstrap()
}