mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-12 21:02:17 +02:00
Merge remote-tracking branch 'upstream/dev' into feat/disk-skills
# Conflicts: # apps/x/apps/main/src/main.ts # apps/x/packages/core/src/application/assistant/instructions.ts
This commit is contained in:
commit
43590ca6d9
159 changed files with 18109 additions and 2589 deletions
|
|
@ -199,6 +199,7 @@ module.exports = {
|
|||
],
|
||||
extendInfo: {
|
||||
NSAudioCaptureUsageDescription: 'Rowboat needs access to system audio to transcribe meetings from other apps (Zoom, Meet, etc.)',
|
||||
NSCameraUsageDescription: 'Rowboat uses your camera in video chat mode so the assistant can see you and give feedback (e.g. pitch practice).',
|
||||
},
|
||||
osxSign: {
|
||||
batchCodesignCalls: true,
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@
|
|||
"make": "electron-forge make"
|
||||
},
|
||||
"dependencies": {
|
||||
"@agentclientprotocol/claude-agent-acp": "^0.39.0",
|
||||
"@agentclientprotocol/codex-acp": "^0.0.44",
|
||||
"@agentclientprotocol/claude-agent-acp": "^0.55.0",
|
||||
"@agentclientprotocol/codex-acp": "^1.1.0",
|
||||
"@x/core": "workspace:*",
|
||||
"@x/shared": "workspace:*",
|
||||
"agent-slack": "0.9.3",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { BrowserWindow } from 'electron';
|
||||
import { ipc } from '@x/shared';
|
||||
import { browserViewManager, type BrowserState } from './view.js';
|
||||
import { browserViewManager, type BrowserState, type HttpAuthRequest } from './view.js';
|
||||
|
||||
type IPCChannels = ipc.IPCChannels;
|
||||
|
||||
|
|
@ -20,6 +20,7 @@ type BrowserHandlers = {
|
|||
'browser:forward': InvokeHandler<'browser:forward'>;
|
||||
'browser:reload': InvokeHandler<'browser:reload'>;
|
||||
'browser:getState': InvokeHandler<'browser:getState'>;
|
||||
'browser:httpAuthResponse': InvokeHandler<'browser:httpAuthResponse'>;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -62,6 +63,9 @@ export const browserIpcHandlers: BrowserHandlers = {
|
|||
'browser:getState': async () => {
|
||||
return browserViewManager.getState();
|
||||
},
|
||||
'browser:httpAuthResponse': async (_event, args) => {
|
||||
return browserViewManager.respondToHttpAuth(args);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -70,12 +74,26 @@ export const browserIpcHandlers: BrowserHandlers = {
|
|||
* window is created so the manager has a window to attach to.
|
||||
*/
|
||||
export function setupBrowserEventForwarding(): void {
|
||||
browserViewManager.on('state-updated', (state: BrowserState) => {
|
||||
const windows = BrowserWindow.getAllWindows();
|
||||
for (const win of windows) {
|
||||
if (!win.isDestroyed() && win.webContents) {
|
||||
win.webContents.send('browser:didUpdateState', state);
|
||||
}
|
||||
// Only send to app windows, never to OAuth/SSO popup windows created by
|
||||
// page window.open() — those render untrusted web content, and browsing
|
||||
// state / auth-challenge metadata must not cross into them.
|
||||
const broadcast = (channel: string, payload: unknown) => {
|
||||
for (const win of BrowserWindow.getAllWindows()) {
|
||||
if (win.isDestroyed() || !win.webContents) continue;
|
||||
if (browserViewManager.isPopupWindow(win)) continue;
|
||||
win.webContents.send(channel, payload);
|
||||
}
|
||||
};
|
||||
|
||||
browserViewManager.on('state-updated', (state: BrowserState) => {
|
||||
broadcast('browser:didUpdateState', state);
|
||||
});
|
||||
|
||||
browserViewManager.on('http-auth-request', (request: HttpAuthRequest) => {
|
||||
broadcast('browser:httpAuthRequest', request);
|
||||
});
|
||||
|
||||
browserViewManager.on('http-auth-resolved', (requestId: string) => {
|
||||
broadcast('browser:httpAuthResolved', { requestId });
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { BrowserWindow, WebContentsView, session, shell, type Session } from 'electron';
|
||||
import { BrowserWindow, WebContentsView, session, shell, type Session, type WebContents } from 'electron';
|
||||
import type {
|
||||
BrowserPageElement,
|
||||
BrowserPageSnapshot,
|
||||
BrowserState,
|
||||
BrowserTabState,
|
||||
HttpAuthRequest,
|
||||
} from '@x/shared/dist/browser-control.js';
|
||||
import { normalizeNavigationTarget } from './navigation.js';
|
||||
import {
|
||||
|
|
@ -20,7 +21,7 @@ import {
|
|||
type RawBrowserPageSnapshot,
|
||||
} from './page-scripts.js';
|
||||
|
||||
export type { BrowserPageSnapshot, BrowserState, BrowserTabState };
|
||||
export type { BrowserPageSnapshot, BrowserState, BrowserTabState, HttpAuthRequest };
|
||||
|
||||
/**
|
||||
* Embedded browser pane implementation.
|
||||
|
|
@ -36,13 +37,33 @@ export type { BrowserPageSnapshot, BrowserState, BrowserTabState };
|
|||
|
||||
export const BROWSER_PARTITION = 'persist:rowboat-browser';
|
||||
|
||||
// Claims Chrome 130 on macOS — close enough to recent stable for OAuth servers
|
||||
// that sniff the UA looking for "real browser" shapes.
|
||||
const SPOOF_UA =
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36';
|
||||
// Spoof a real Chrome UA so OAuth servers don't reject the embedded browser.
|
||||
// The Chrome major version is derived from the running Chromium at startup:
|
||||
// pinning a fixed version goes stale as Electron upgrades, and Chromium keeps
|
||||
// emitting Sec-CH-UA client hints with the *real* version — a UA/client-hint
|
||||
// version mismatch is a classic bot-detection signal (Google sign-in,
|
||||
// Cloudflare). Minor version is frozen at 0.0.0, exactly like real Chrome's
|
||||
// reduced UA. The platform token matches the actual OS for the same reason.
|
||||
function getChromeMajorVersion(): number {
|
||||
const major = Number.parseInt(process.versions.chrome ?? '', 10);
|
||||
return Number.isFinite(major) && major > 0 ? major : 130;
|
||||
}
|
||||
|
||||
function buildChromeUserAgent(): string {
|
||||
const platformToken =
|
||||
process.platform === 'darwin'
|
||||
? 'Macintosh; Intel Mac OS X 10_15_7'
|
||||
: process.platform === 'win32'
|
||||
? 'Windows NT 10.0; Win64; x64'
|
||||
: 'X11; Linux x86_64';
|
||||
return `Mozilla/5.0 (${platformToken}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${getChromeMajorVersion()}.0.0.0 Safari/537.36`;
|
||||
}
|
||||
|
||||
const SPOOF_UA = buildChromeUserAgent();
|
||||
|
||||
const HOME_URL = 'https://www.google.com';
|
||||
const NAVIGATION_TIMEOUT_MS = 10000;
|
||||
const HTTP_AUTH_TIMEOUT_MS = 120000;
|
||||
const POST_ACTION_IDLE_MS = 400;
|
||||
const POST_ACTION_MAX_ELEMENTS = 25;
|
||||
const POST_ACTION_MAX_TEXT_LENGTH = 4000;
|
||||
|
|
@ -68,6 +89,13 @@ type CachedSnapshot = {
|
|||
elements: Array<{ index: number; selector: string }>;
|
||||
};
|
||||
|
||||
type PendingHttpAuth = {
|
||||
callback: (username?: string, password?: string) => void;
|
||||
timer: NodeJS.Timeout;
|
||||
// The webContents that raised the challenge, so its teardown can cancel it.
|
||||
webContents: WebContents;
|
||||
};
|
||||
|
||||
const EMPTY_STATE: BrowserState = {
|
||||
activeTabId: null,
|
||||
tabs: [],
|
||||
|
|
@ -109,6 +137,12 @@ export class BrowserViewManager extends EventEmitter {
|
|||
private visible = false;
|
||||
private bounds: BrowserBounds = { x: 0, y: 0, width: 0, height: 0 };
|
||||
private snapshotCache = new Map<string, CachedSnapshot>();
|
||||
private pendingHttpAuth = new Map<string, PendingHttpAuth>();
|
||||
// Child windows created by page window.open() (OAuth/SSO popups). Tracked so
|
||||
// they can be closed when the host window goes away — otherwise an orphaned
|
||||
// popup keeps BrowserWindow.getAllWindows() non-empty and, on macOS, blocks
|
||||
// the app from reopening via the Dock (see main.ts 'activate' handler).
|
||||
private popupWindows = new Set<BrowserWindow>();
|
||||
private cleanupWindowListeners: (() => void) | null = null;
|
||||
|
||||
attach(window: BrowserWindow): void {
|
||||
|
|
@ -137,6 +171,7 @@ export class BrowserViewManager extends EventEmitter {
|
|||
if (this.window !== window) return;
|
||||
|
||||
const tabs = [...this.tabs.values()];
|
||||
const popups = [...this.popupWindows];
|
||||
this.cleanupWindowListeners = null;
|
||||
this.window = null;
|
||||
this.browserSession = null;
|
||||
|
|
@ -150,6 +185,14 @@ export class BrowserViewManager extends EventEmitter {
|
|||
this.attachedTabId = null;
|
||||
this.visible = false;
|
||||
this.snapshotCache.clear();
|
||||
for (const requestId of [...this.pendingHttpAuth.keys()]) {
|
||||
this.finishHttpAuth(requestId);
|
||||
}
|
||||
// Close any OAuth/SSO popups so they don't outlive the app window.
|
||||
for (const popup of popups) {
|
||||
if (!popup.isDestroyed()) popup.close();
|
||||
}
|
||||
this.popupWindows.clear();
|
||||
};
|
||||
|
||||
hostWebContents.on('did-start-loading', handleDidStartLoading);
|
||||
|
|
@ -171,6 +214,36 @@ export class BrowserViewManager extends EventEmitter {
|
|||
if (this.browserSession) return this.browserSession;
|
||||
const browserSession = session.fromPartition(BROWSER_PARTITION);
|
||||
browserSession.setUserAgent(SPOOF_UA);
|
||||
|
||||
// Electron's Sec-CH-UA client hints only carry the "Chromium" brand;
|
||||
// real Chrome also sends "Google Chrome". Some sign-in flows (notably
|
||||
// Google's) distinguish the two, so rewrite the brand list to match what
|
||||
// Chrome sends. Both the low-entropy header (`sec-ch-ua`, major versions)
|
||||
// and the high-entropy one (`sec-ch-ua-full-version-list`, requested via
|
||||
// Accept-CH and carrying full versions) must be rewritten together — a
|
||||
// header that claims "Google Chrome" alongside one that doesn't is a
|
||||
// stronger bot signal than the original. Only headers Chromium already
|
||||
// attached are rewritten — none are added. (navigator.userAgentData JS
|
||||
// brands still report only Chromium; there is no reliable hook to spoof
|
||||
// that under sandbox+contextIsolation, and header-based detection is the
|
||||
// common case.)
|
||||
const chromeMajor = getChromeMajorVersion();
|
||||
const chromeFull = process.versions.chrome ?? `${chromeMajor}.0.0.0`;
|
||||
const brandLists: Record<string, string> = {
|
||||
'sec-ch-ua': `"Chromium";v="${chromeMajor}", "Google Chrome";v="${chromeMajor}", "Not-A.Brand";v="99"`,
|
||||
'sec-ch-ua-full-version-list': `"Chromium";v="${chromeFull}", "Google Chrome";v="${chromeFull}", "Not-A.Brand";v="99.0.0.0"`,
|
||||
};
|
||||
browserSession.webRequest.onBeforeSendHeaders((details, callback) => {
|
||||
const requestHeaders = details.requestHeaders;
|
||||
for (const name of Object.keys(requestHeaders)) {
|
||||
const replacement = brandLists[name.toLowerCase()];
|
||||
if (replacement !== undefined) {
|
||||
requestHeaders[name] = replacement;
|
||||
}
|
||||
}
|
||||
callback({ requestHeaders });
|
||||
});
|
||||
|
||||
this.browserSession = browserSession;
|
||||
return browserSession;
|
||||
}
|
||||
|
|
@ -196,17 +269,34 @@ export class BrowserViewManager extends EventEmitter {
|
|||
return /^https?:\/\//i.test(url) || url === 'about:blank';
|
||||
}
|
||||
|
||||
/**
|
||||
* webPreferences shared by browser tabs and OAuth popups. Kept in one place
|
||||
* so the security-sensitive popup surface can never drift from tabs.
|
||||
*/
|
||||
private browserWebPreferences(): Electron.WebPreferences {
|
||||
return {
|
||||
session: this.getSession(),
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
nodeIntegration: false,
|
||||
// Chromium's built-in PDFium viewer, so PDFs render inline instead
|
||||
// of showing a blank page.
|
||||
plugins: true,
|
||||
// Remove the WebAuthn API from the embedded browser only. Electron ships
|
||||
// the API but not Chrome's authenticator UI (Touch ID sheet, QR/phone
|
||||
// hybrid), so passkey challenges hang forever on "Verifying it's you...".
|
||||
// With the API absent, sites feature-detect it and fall back to
|
||||
// password/other verification. Scoped here (not app-wide) so the app's
|
||||
// own renderer keeps WebAuthn.
|
||||
disableBlinkFeatures: 'WebAuth',
|
||||
};
|
||||
}
|
||||
|
||||
private createView(): WebContentsView {
|
||||
const view = new WebContentsView({
|
||||
webPreferences: {
|
||||
session: this.getSession(),
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
nodeIntegration: false,
|
||||
},
|
||||
webPreferences: this.browserWebPreferences(),
|
||||
});
|
||||
|
||||
view.webContents.setUserAgent(SPOOF_UA);
|
||||
return view;
|
||||
}
|
||||
|
||||
|
|
@ -266,16 +356,145 @@ export class BrowserViewManager extends EventEmitter {
|
|||
});
|
||||
wc.on('page-title-updated', this.emitState.bind(this));
|
||||
|
||||
wc.setWindowOpenHandler(({ url }) => {
|
||||
if (this.isEmbeddedTabUrl(url)) {
|
||||
void this.newTab(url);
|
||||
} else {
|
||||
void shell.openExternal(url);
|
||||
this.wireWindowPolicy(wc);
|
||||
}
|
||||
|
||||
/**
|
||||
* Window-open, popup, and HTTP-auth wiring shared by tabs and popups.
|
||||
*/
|
||||
private wireWindowPolicy(wc: WebContents): void {
|
||||
wc.setWindowOpenHandler((details) => this.handleWindowOpen(details));
|
||||
wc.on('did-create-window', (child) => this.wirePopupWindow(child));
|
||||
this.wireHttpAuth(wc);
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared window.open / target=_blank policy for tabs and popups.
|
||||
*
|
||||
* An open that hands a handle back to the opener must become a real child
|
||||
* window so window.opener / postMessage survive — this is how OAuth/SSO
|
||||
* popups (Google, Microsoft, Plaid, ...) return their result; denying them
|
||||
* also makes sites report "popup blocked". Those are: a sized popup
|
||||
* (disposition 'new-window'), a *named* window.open(url, 'name') (non-empty
|
||||
* frameName), or a scripted blank window the opener will populate
|
||||
* (about:blank). A nameless target=_blank link (foreground-tab, empty
|
||||
* frameName) has no opener contract and opens as a tab, matching browser
|
||||
* behavior. Non-web schemes go to the system handler.
|
||||
*
|
||||
* Residual gap: a nameless, featureless window.open(url) is indistinguishable
|
||||
* from a _blank link (both foreground-tab + empty frameName) and opens as a
|
||||
* tab, losing its opener — rare for OAuth, which virtually always names or
|
||||
* sizes its popup.
|
||||
*/
|
||||
private handleWindowOpen(details: Electron.HandlerDetails): Electron.WindowOpenHandlerResponse {
|
||||
const { url, disposition, frameName } = details;
|
||||
|
||||
if (this.isEmbeddedTabUrl(url)) {
|
||||
const needsOpener =
|
||||
disposition === 'new-window' || frameName !== '' || url === 'about:blank';
|
||||
if (needsOpener) {
|
||||
return {
|
||||
action: 'allow',
|
||||
overrideBrowserWindowOptions: {
|
||||
autoHideMenuBar: true,
|
||||
webPreferences: this.browserWebPreferences(),
|
||||
},
|
||||
};
|
||||
}
|
||||
void this.newTab(url);
|
||||
return { action: 'deny' };
|
||||
}
|
||||
|
||||
void shell.openExternal(url);
|
||||
return { action: 'deny' };
|
||||
}
|
||||
|
||||
private wirePopupWindow(child: BrowserWindow): void {
|
||||
this.popupWindows.add(child);
|
||||
child.once('closed', () => this.popupWindows.delete(child));
|
||||
this.wireWindowPolicy(child.webContents);
|
||||
}
|
||||
|
||||
/** True if `win` is an OAuth/SSO popup created by page window.open(). */
|
||||
isPopupWindow(win: BrowserWindow): boolean {
|
||||
return this.popupWindows.has(win);
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP basic/proxy auth. Chromium's default is to cancel the challenge, so
|
||||
* 401-protected sites and authenticating proxies dead-end. When the browser
|
||||
* pane is on screen to answer, forward the challenge to it as a credential
|
||||
* prompt (cancelled after a timeout if unanswered). When the pane is closed
|
||||
* — e.g. agent-driven navigation — don't preventDefault, so Chromium cancels
|
||||
* immediately and the 401 page is readable rather than hanging.
|
||||
*/
|
||||
private wireHttpAuth(wc: WebContents): void {
|
||||
wc.on('login', (event, _details, authInfo, callback) => {
|
||||
if (!this.visible || !this.window) return;
|
||||
event.preventDefault();
|
||||
|
||||
const requestId = randomUUID();
|
||||
const timer = setTimeout(() => {
|
||||
this.finishHttpAuth(requestId);
|
||||
}, HTTP_AUTH_TIMEOUT_MS);
|
||||
this.pendingHttpAuth.set(requestId, { callback, timer, webContents: wc });
|
||||
// If the challenging contents dies before an answer, resolve now so the
|
||||
// native callback and timer don't leak (backstop for paths other than
|
||||
// destroyTab, which cancels explicitly before removeAllListeners()).
|
||||
wc.once('destroyed', () => this.finishHttpAuth(requestId));
|
||||
|
||||
const request: HttpAuthRequest = {
|
||||
requestId,
|
||||
host: authInfo.host,
|
||||
isProxy: authInfo.isProxy,
|
||||
...(authInfo.realm ? { realm: authInfo.realm } : {}),
|
||||
};
|
||||
this.emit('http-auth-request', request);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a pending auth challenge. `username === undefined` cancels it; an
|
||||
* empty-string username is a valid submission (token-style Basic auth).
|
||||
* Always notifies the renderer so a dialog it may still be showing (e.g.
|
||||
* after a timeout or tab close) is pruned.
|
||||
*/
|
||||
private finishHttpAuth(requestId: string, username?: string, password?: string): boolean {
|
||||
const pending = this.pendingHttpAuth.get(requestId);
|
||||
if (!pending) return false;
|
||||
this.pendingHttpAuth.delete(requestId);
|
||||
clearTimeout(pending.timer);
|
||||
try {
|
||||
if (username == null) {
|
||||
pending.callback();
|
||||
} else {
|
||||
pending.callback(username, password ?? '');
|
||||
}
|
||||
} catch {
|
||||
// The challenged webContents may already be destroyed.
|
||||
}
|
||||
this.emit('http-auth-resolved', requestId);
|
||||
return true;
|
||||
}
|
||||
|
||||
private cancelHttpAuthForWebContents(wc: WebContents): void {
|
||||
const ids: string[] = [];
|
||||
for (const [requestId, pending] of this.pendingHttpAuth) {
|
||||
if (pending.webContents === wc) ids.push(requestId);
|
||||
}
|
||||
for (const requestId of ids) {
|
||||
this.finishHttpAuth(requestId);
|
||||
}
|
||||
}
|
||||
|
||||
respondToHttpAuth(input: {
|
||||
requestId: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
}): { ok: boolean } {
|
||||
return { ok: this.finishHttpAuth(input.requestId, input.username, input.password) };
|
||||
}
|
||||
|
||||
private snapshotTabState(tab: BrowserTab): BrowserTabState {
|
||||
const wc = tab.view.webContents;
|
||||
return {
|
||||
|
|
@ -364,6 +583,10 @@ export class BrowserViewManager extends EventEmitter {
|
|||
|
||||
private destroyTab(tab: BrowserTab): void {
|
||||
this.invalidateSnapshot(tab.id);
|
||||
// Cancel any auth challenge this tab raised before we drop its listeners,
|
||||
// so the native callback + timer don't leak and the renderer prunes its
|
||||
// dialog (removeAllListeners() below would kill the 'destroyed' backstop).
|
||||
this.cancelHttpAuthForWebContents(tab.view.webContents);
|
||||
tab.view.webContents.removeAllListeners();
|
||||
if (!tab.view.webContents.isDestroyed()) {
|
||||
tab.view.webContents.close();
|
||||
|
|
|
|||
|
|
@ -315,3 +315,50 @@ export async function listToolkits() {
|
|||
totalItems: filtered.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a Composio tool by slug on behalf of a Mini App. The toolkit must be
|
||||
* connected (ACTIVE). Mirrors the agent's composio-execute-tool builtin.
|
||||
*/
|
||||
export async function executeTool(
|
||||
toolkitSlug: string,
|
||||
toolSlug: string,
|
||||
args?: Record<string, unknown>,
|
||||
): Promise<{ successful: boolean; data?: unknown; error?: string }> {
|
||||
const account = composioAccountsRepo.getAccount(toolkitSlug);
|
||||
if (!account || account.status !== 'ACTIVE') {
|
||||
return { successful: false, error: `Toolkit "${toolkitSlug}" is not connected.` };
|
||||
}
|
||||
try {
|
||||
const result = await composioClient.executeAction(toolSlug, {
|
||||
connected_account_id: account.id,
|
||||
user_id: 'rowboat-user',
|
||||
version: 'latest',
|
||||
arguments: args ?? {},
|
||||
});
|
||||
return { successful: result.successful, data: result.data, error: result.error ?? undefined };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`[Composio] Mini App tool execution failed for ${toolSlug}:`, message);
|
||||
return { successful: false, error: `Failed to execute ${toolSlug}: ${message}` };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search Composio tools within a toolkit so a Mini App can discover the right
|
||||
* tool slug + input schema at runtime (how generated apps will wire actions).
|
||||
*/
|
||||
export async function searchToolsInToolkit(
|
||||
toolkitSlug: string,
|
||||
query: string,
|
||||
): Promise<{ tools: Array<{ slug: string; name: string; description?: string }>; error?: string }> {
|
||||
try {
|
||||
const { items } = await composioClient.searchTools(query, [toolkitSlug]);
|
||||
return {
|
||||
tools: items.map((t) => ({ slug: t.slug, name: t.name, description: t.description })),
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return { tools: [], error: message };
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { ipcMain, BrowserWindow, shell, dialog, systemPreferences, desktopCapturer, app } from 'electron';
|
||||
import { ipcMain, BrowserWindow, shell, dialog, systemPreferences, desktopCapturer, app, screen } from 'electron';
|
||||
import { ipc } from '@x/shared';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
connectProvider,
|
||||
disconnectProvider,
|
||||
|
|
@ -37,6 +38,7 @@ import type { IOAuthRepo } from '@x/core/dist/auth/repo.js';
|
|||
import { IGranolaConfigRepo } from '@x/core/dist/knowledge/granola/repo.js';
|
||||
import { ICodeModeConfigRepo } from '@x/core/dist/code-mode/repo.js';
|
||||
import { CodePermissionRegistry } from '@x/core/dist/code-mode/acp/permission-registry.js';
|
||||
import type { CodeRunFeed } from '@x/core/dist/code-mode/feed.js';
|
||||
import { checkCodeModeAgentStatus } from '@x/core/dist/code-mode/status.js';
|
||||
import { ensureEngine } from '@x/core/dist/code-mode/acp/engine-provisioner.js';
|
||||
import type { ICodeProjectsRepo } from '@x/core/dist/code-mode/projects/repo.js';
|
||||
|
|
@ -51,6 +53,8 @@ import type { CodeSession } from '@x/shared/dist/code-sessions.js';
|
|||
import { invalidateCopilotInstructionsCache } from '@x/core/dist/application/assistant/instructions.js';
|
||||
import { triggerSync as triggerGranolaSync } from '@x/core/dist/knowledge/granola/sync.js';
|
||||
import { ISlackConfigRepo } from '@x/core/dist/slack/repo.js';
|
||||
import { IChannelsConfigRepo } from '@x/core/dist/channels/repo.js';
|
||||
import { applyChannelsConfig, getChannelsStatus, logoutWhatsApp, subscribeChannelsStatus } from '@x/core/dist/channels/service.js';
|
||||
import { runAgentSlack, getAgentSlackCliStatus, AgentSlackRunError } from '@x/core/dist/slack/agent-slack-exec.js';
|
||||
import { knowledgeSourcesRepo } from '@x/core/dist/knowledge/sources/repo.js';
|
||||
import { rankSlackHomeMessages } from '@x/core/dist/knowledge/sources/rank_slack_home.js';
|
||||
|
|
@ -58,6 +62,10 @@ import { syncSlackKnowledgeSources, triggerSync as triggerSlackKnowledgeSync, ge
|
|||
import { isOnboardingComplete, markOnboardingComplete } from '@x/core/dist/config/note_creation_config.js';
|
||||
import { loadNotificationSettings, saveNotificationSettings } from '@x/core/dist/config/notification_config.js';
|
||||
import * as composioHandler from './composio-handler.js';
|
||||
import * as appsIndexer from '@x/core/dist/apps/indexer.js';
|
||||
import * as appsServer from '@x/core/dist/apps/server.js';
|
||||
import * as appsAgents from '@x/core/dist/apps/agents.js';
|
||||
import { capture } from '@x/core/dist/analytics/posthog.js';
|
||||
import { consumePendingDeepLink } from './deeplink.js';
|
||||
import { qualifyAndDisconnectComposioGoogle } from '@x/core/dist/migrations/composio-google-migration.js';
|
||||
import { IAgentScheduleRepo } from '@x/core/dist/agent-schedule/repo.js';
|
||||
|
|
@ -74,7 +82,7 @@ import { summarizeMeeting } from '@x/core/dist/knowledge/summarize_meeting.js';
|
|||
import { getAccessToken } from '@x/core/dist/auth/tokens.js';
|
||||
import { getRowboatConfig } from '@x/core/dist/config/rowboat.js';
|
||||
import { runLiveNoteAgent } from '@x/core/dist/knowledge/live-note/runner.js';
|
||||
import { listImportantThreads, listEverythingElseThreads, saveMessageBodyHeight, triggerSync as triggerGmailSync, sendThreadReply, archiveThread, trashThread, markThreadRead, getAccountEmail, getAccountName, getConnectionStatus as getGmailConnectionStatus } from '@x/core/dist/knowledge/sync_gmail.js';
|
||||
import { listImportantThreads, listEverythingElseThreads, saveMessageBodyHeight, triggerSync as triggerGmailSync, sendThreadReply, saveThreadDraft, deleteThreadDraft, listDraftThreads, searchThreads, archiveThread, trashThread, markThreadRead, downloadAttachment, getAccountEmail, getAccountName, getConnectionStatus as getGmailConnectionStatus, setThreadImportance } from '@x/core/dist/knowledge/sync_gmail.js';
|
||||
import { searchContacts as searchGmailContacts, warmContactIndex } from '@x/core/dist/knowledge/gmail_contacts.js';
|
||||
import { searchSentContacts, warmSentContacts } from '@x/core/dist/knowledge/gmail_sent_contacts.js';
|
||||
import { getGoogleDocsConnectionStatus, importGoogleDoc, syncGoogleDocDown, syncGoogleDocUp, getGoogleDocLink } from '@x/core/dist/knowledge/google_docs.js';
|
||||
|
|
@ -381,9 +389,37 @@ type InvokeHandlers = {
|
|||
[K in InvokeChannels]: InvokeHandler<K>;
|
||||
};
|
||||
|
||||
// In-flight streaming TTS requests, keyed by renderer-chosen requestId.
|
||||
const activeTtsStreams = new Map<string, AbortController>();
|
||||
|
||||
// Video-mode popout window (shown for the whole duration of a screen share,
|
||||
// floating over every app including Rowboat itself) and the last call state
|
||||
// pushed by the main window — replayed to the popout when it finishes loading.
|
||||
let videoPopoutWin: BrowserWindow | null = null;
|
||||
let lastVideoPopoutState: {
|
||||
ttsState: 'idle' | 'synthesizing' | 'speaking';
|
||||
status: 'listening' | 'thinking' | 'speaking' | null;
|
||||
cameraOn: boolean;
|
||||
micMuted: boolean;
|
||||
screenSharing: boolean;
|
||||
interimText: string | null;
|
||||
} | null = null;
|
||||
|
||||
// Match only real app windows — getAllWindows() can also contain the popout
|
||||
// itself and hidden utility windows (e.g. PDF-export renderers), which must
|
||||
// not be shown, focused, or sent app events.
|
||||
function findMainAppWindow(): BrowserWindow | undefined {
|
||||
return BrowserWindow.getAllWindows().find((w) => {
|
||||
if (w === videoPopoutWin || w.isDestroyed()) return false;
|
||||
const url = w.webContents.getURL();
|
||||
const isAppWindow = url.startsWith('app://') || url.startsWith('http://localhost');
|
||||
return isAppWindow && !url.includes('#video-popout');
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Register all IPC handlers with type safety and runtime validation
|
||||
*
|
||||
*
|
||||
* This function ensures:
|
||||
* 1. All invoke channels have handlers (exhaustiveness checking)
|
||||
* 2. Handler signatures match channel definitions
|
||||
|
|
@ -592,6 +628,9 @@ function emitServiceEvent(event: z.infer<typeof ServiceEvent>): void {
|
|||
}
|
||||
|
||||
export function emitOAuthEvent(event: { provider: string; success: boolean; error?: string; userId?: string }): void {
|
||||
// Native connection status (e.g. Google) is baked into the Copilot system
|
||||
// prompt, so any OAuth state change must rebuild it.
|
||||
invalidateCopilotInstructionsCache();
|
||||
const windows = BrowserWindow.getAllWindows();
|
||||
for (const win of windows) {
|
||||
if (!win.isDestroyed() && win.webContents) {
|
||||
|
|
@ -646,6 +685,20 @@ function emitSessionEvent(event: SessionBusEvent): void {
|
|||
}
|
||||
}
|
||||
|
||||
// Mobile channels: status changes (QR pairing, connect/disconnect) → renderer.
|
||||
let channelsWatcher: (() => void) | null = null;
|
||||
export function startChannelsWatcher(): void {
|
||||
if (channelsWatcher) return;
|
||||
channelsWatcher = subscribeChannelsStatus((status) => {
|
||||
const windows = BrowserWindow.getAllWindows();
|
||||
for (const win of windows) {
|
||||
if (!win.isDestroyed() && win.webContents) {
|
||||
win.webContents.send('channels:status', status);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let sessionsWatcher: (() => void) | null = null;
|
||||
export function startSessionsWatcher(): void {
|
||||
if (sessionsWatcher) {
|
||||
|
|
@ -655,6 +708,38 @@ export function startSessionsWatcher(): void {
|
|||
sessionsWatcher = sessionBus.subscribe((event) => emitSessionEvent(event));
|
||||
}
|
||||
|
||||
// Ephemeral code-run stream: CodeRunFeed → all renderer windows. A direct
|
||||
// tool→renderer side-channel that bypasses the turn runtime; the durable
|
||||
// record is the settle-time code-run-events-batch tool progress.
|
||||
let codeRunFeedWatcher: (() => void) | null = null;
|
||||
export function startCodeRunFeedWatcher(): void {
|
||||
if (codeRunFeedWatcher) {
|
||||
return;
|
||||
}
|
||||
const feed = container.resolve<CodeRunFeed>('codeRunFeed');
|
||||
codeRunFeedWatcher = feed.subscribe((event) => {
|
||||
const windows = BrowserWindow.getAllWindows();
|
||||
for (const win of windows) {
|
||||
if (!win.isDestroyed() && win.webContents) {
|
||||
win.webContents.send('codeRun:events', event);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// The renderer window is created before the session-index startup scan
|
||||
// finishes, so an early sessions:list could observe a partially built index
|
||||
// (the scan runs oldest-first — exactly the newest chats would be missing).
|
||||
// sessions:list awaits this deferred; main.ts resolves it when the scan
|
||||
// settles (success or failure, so the list never hangs).
|
||||
let resolveSessionsIndexReady: () => void;
|
||||
const sessionsIndexReady = new Promise<void>((resolve) => {
|
||||
resolveSessionsIndexReady = resolve;
|
||||
});
|
||||
export function markSessionsIndexReady(): void {
|
||||
resolveSessionsIndexReady();
|
||||
}
|
||||
|
||||
let servicesWatcher: (() => void) | null = null;
|
||||
export async function startServicesWatcher(): Promise<void> {
|
||||
if (servicesWatcher) {
|
||||
|
|
@ -782,6 +867,18 @@ export function setupIpcHandlers() {
|
|||
'gmail:sendReply': async (_event, args) => {
|
||||
return sendThreadReply(args);
|
||||
},
|
||||
'gmail:saveDraft': async (_event, args) => {
|
||||
return saveThreadDraft(args);
|
||||
},
|
||||
'gmail:deleteDraft': async (_event, args) => {
|
||||
return deleteThreadDraft(args.draftId);
|
||||
},
|
||||
'gmail:getDrafts': async () => {
|
||||
return listDraftThreads();
|
||||
},
|
||||
'gmail:search': async (_event, args) => {
|
||||
return searchThreads(args.query, { limit: args.limit });
|
||||
},
|
||||
'gmail:getConnectionStatus': async () => {
|
||||
return getGmailConnectionStatus();
|
||||
},
|
||||
|
|
@ -791,6 +888,10 @@ export function setupIpcHandlers() {
|
|||
'gmail:getAccountName': async () => {
|
||||
return { name: await getAccountName() };
|
||||
},
|
||||
'gmail:setImportance': async (_event, args) => {
|
||||
const result = setThreadImportance(args.threadId, args.importance);
|
||||
return { ok: result.success, previous: result.previous, error: result.error };
|
||||
},
|
||||
'gmail:archiveThread': async (_event, args) => {
|
||||
return archiveThread(args.threadId);
|
||||
},
|
||||
|
|
@ -798,7 +899,10 @@ export function setupIpcHandlers() {
|
|||
return trashThread(args.threadId);
|
||||
},
|
||||
'gmail:markThreadRead': async (_event, args) => {
|
||||
return markThreadRead(args.threadId);
|
||||
return markThreadRead(args.threadId, args.read);
|
||||
},
|
||||
'gmail:downloadAttachment': async (_event, args) => {
|
||||
return downloadAttachment(args);
|
||||
},
|
||||
'gmail:saveMessageHeight': async (_event, args) => {
|
||||
saveMessageBodyHeight(args.threadId, args.messageId, args.height);
|
||||
|
|
@ -872,6 +976,7 @@ export function setupIpcHandlers() {
|
|||
return { sessionId };
|
||||
},
|
||||
'sessions:list': async () => {
|
||||
await sessionsIndexReady;
|
||||
return { sessions: container.resolve<ISessions>('sessions').listSessions() };
|
||||
},
|
||||
'sessions:get': async (_event, args) => {
|
||||
|
|
@ -999,6 +1104,11 @@ export function setupIpcHandlers() {
|
|||
await repo.setConfig(args);
|
||||
return { success: true };
|
||||
},
|
||||
'models:updateConfig': async (_event, args) => {
|
||||
const repo = container.resolve<IModelConfigRepo>('modelConfigRepo');
|
||||
await repo.updateConfig(args);
|
||||
return { success: true };
|
||||
},
|
||||
'oauth:connect': async (_event, args) => {
|
||||
const credentials = args.clientId && args.clientSecret
|
||||
? { clientId: args.clientId.trim(), clientSecret: args.clientSecret.trim() }
|
||||
|
|
@ -1193,6 +1303,22 @@ export function setupIpcHandlers() {
|
|||
|
||||
return { success: true };
|
||||
},
|
||||
// ── Mobile channels (WhatsApp / Telegram bridge) ─────────────
|
||||
'channels:getConfig': async () => {
|
||||
return container.resolve<IChannelsConfigRepo>('channelsConfigRepo').getConfig();
|
||||
},
|
||||
'channels:setConfig': async (_event, args) => {
|
||||
await container.resolve<IChannelsConfigRepo>('channelsConfigRepo').setConfig(args);
|
||||
await applyChannelsConfig(args);
|
||||
return { success: true };
|
||||
},
|
||||
'channels:getStatus': async () => {
|
||||
return getChannelsStatus();
|
||||
},
|
||||
'channels:whatsappLogout': async () => {
|
||||
await logoutWhatsApp();
|
||||
return { success: true };
|
||||
},
|
||||
'slack:getConfig': async () => {
|
||||
const repo = container.resolve<ISlackConfigRepo>('slackConfigRepo');
|
||||
const config = await repo.getConfig();
|
||||
|
|
@ -1422,9 +1548,56 @@ export function setupIpcHandlers() {
|
|||
'composio:list-toolkits': async () => {
|
||||
return composioHandler.listToolkits();
|
||||
},
|
||||
'composio:execute-tool': async (_event, args) => {
|
||||
return composioHandler.executeTool(args.toolkitSlug, args.toolSlug, args.arguments);
|
||||
},
|
||||
'composio:search-tools': async (_event, args) => {
|
||||
return composioHandler.searchToolsInToolkit(args.toolkitSlug, args.query);
|
||||
},
|
||||
'migration:check-composio-google': async () => {
|
||||
return qualifyAndDisconnectComposioGoogle();
|
||||
},
|
||||
// Rowboat Apps handlers (spec §13)
|
||||
'apps:list': async () => {
|
||||
const status = appsServer.getServerStatus();
|
||||
const apps = await appsIndexer.listApps();
|
||||
// Keep bundled agents materialized (idempotent; disabled by default).
|
||||
for (const app of apps) {
|
||||
if (app.agentSlugs.length) await appsAgents.syncAppAgents(app);
|
||||
}
|
||||
return {
|
||||
serverRunning: status.running,
|
||||
...(status.error ? { serverError: status.error } : {}),
|
||||
apps,
|
||||
};
|
||||
},
|
||||
'apps:get': async (_event, args) => {
|
||||
const app = await appsIndexer.getApp(args.folder);
|
||||
if (!app) throw new Error(`no such app: ${args.folder}`);
|
||||
const readme = await appsIndexer.readAppReadme(args.folder);
|
||||
return {
|
||||
app,
|
||||
...(readme ? { readme } : {}),
|
||||
rollbackAvailable: await appsIndexer.rollbackAvailable(args.folder),
|
||||
};
|
||||
},
|
||||
'apps:create': async (_event, args) => {
|
||||
const app = await appsIndexer.createApp(args);
|
||||
capture('app_created', { folder: app.folder });
|
||||
return { app };
|
||||
},
|
||||
'apps:delete': async (_event, args) => {
|
||||
await appsIndexer.deleteApp(args.folder);
|
||||
// Remove app-owned bg-tasks too — orphaned app--<folder>-- tasks firing
|
||||
// against a deleted app was a painful prototype failure mode.
|
||||
await appsAgents.deleteAppAgents(args.folder);
|
||||
capture('app_deleted', { folder: args.folder });
|
||||
return { ok: true as const };
|
||||
},
|
||||
'apps:setTheme': async (_event, args) => {
|
||||
appsServer.setAppsTheme(args.theme);
|
||||
return { ok: true as const };
|
||||
},
|
||||
// Agent schedule handlers
|
||||
'agent-schedule:getConfig': async () => {
|
||||
const repo = container.resolve<IAgentScheduleRepo>('agentScheduleRepo');
|
||||
|
|
@ -1680,6 +1853,51 @@ export function setupIpcHandlers() {
|
|||
'voice:synthesize': async (_event, args) => {
|
||||
return voice.synthesizeSpeech(args.text);
|
||||
},
|
||||
'voice:synthesizeStreamStart': async (event, args) => {
|
||||
const { requestId, text } = args;
|
||||
const sender = event.sender;
|
||||
const controller = new AbortController();
|
||||
activeTtsStreams.set(requestId, controller);
|
||||
// Fire-and-forget: chunks are pushed to the renderer as they arrive so
|
||||
// playback can begin immediately; the invoke returns once started.
|
||||
void voice
|
||||
.synthesizeSpeechStream(
|
||||
text,
|
||||
(chunk) => {
|
||||
if (!sender.isDestroyed()) {
|
||||
sender.send('voice:tts-chunk', {
|
||||
requestId,
|
||||
chunkBase64: chunk.toString('base64'),
|
||||
done: false,
|
||||
});
|
||||
}
|
||||
},
|
||||
controller.signal,
|
||||
)
|
||||
.then(() => {
|
||||
if (!sender.isDestroyed()) {
|
||||
sender.send('voice:tts-chunk', { requestId, done: true });
|
||||
}
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (!sender.isDestroyed() && !controller.signal.aborted) {
|
||||
sender.send('voice:tts-chunk', {
|
||||
requestId,
|
||||
done: true,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
activeTtsStreams.delete(requestId);
|
||||
});
|
||||
return { ok: true };
|
||||
},
|
||||
'voice:synthesizeStreamCancel': async (_event, args) => {
|
||||
activeTtsStreams.get(args.requestId)?.abort();
|
||||
activeTtsStreams.delete(args.requestId);
|
||||
return {};
|
||||
},
|
||||
'voice:ensureMicAccess': async () => {
|
||||
if (process.platform !== 'darwin') return { granted: true };
|
||||
const status = systemPreferences.getMediaAccessStatus('microphone');
|
||||
|
|
@ -1698,6 +1916,113 @@ export function setupIpcHandlers() {
|
|||
return { granted: false };
|
||||
}
|
||||
},
|
||||
'voice:ensureCameraAccess': async () => {
|
||||
if (process.platform !== 'darwin') return { granted: true };
|
||||
const status = systemPreferences.getMediaAccessStatus('camera');
|
||||
console.log('[video] Camera permission status:', status);
|
||||
if (status === 'granted') return { granted: true };
|
||||
// Same flow as the microphone: settle the native TCC prompt before the
|
||||
// renderer's getUserMedia so the first video click doesn't silently fail.
|
||||
try {
|
||||
const granted = await systemPreferences.askForMediaAccess('camera');
|
||||
console.log('[video] Camera permission after prompt:', granted);
|
||||
return { granted };
|
||||
} catch {
|
||||
return { granted: false };
|
||||
}
|
||||
},
|
||||
'video:setPopout': async (_event, args) => {
|
||||
if (!args.show) {
|
||||
if (videoPopoutWin && !videoPopoutWin.isDestroyed()) videoPopoutWin.destroy();
|
||||
videoPopoutWin = null;
|
||||
return {};
|
||||
}
|
||||
if (videoPopoutWin && !videoPopoutWin.isDestroyed()) return {};
|
||||
|
||||
const workArea = screen.getPrimaryDisplay().workArea;
|
||||
const width = 340;
|
||||
const height = 184;
|
||||
const ipcDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const preloadPath = app.isPackaged
|
||||
? path.join(ipcDir, '../preload/dist/preload.js')
|
||||
: path.join(ipcDir, '../../../preload/dist/preload.js');
|
||||
const win = new BrowserWindow({
|
||||
width,
|
||||
height,
|
||||
x: workArea.x + workArea.width - width - 24,
|
||||
y: workArea.y + 24,
|
||||
frame: false,
|
||||
resizable: false,
|
||||
alwaysOnTop: true,
|
||||
skipTaskbar: true,
|
||||
show: false,
|
||||
hasShadow: true,
|
||||
backgroundColor: '#171717',
|
||||
webPreferences: {
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
preload: preloadPath,
|
||||
},
|
||||
});
|
||||
// Float above other apps on every workspace. Deliberately NOT
|
||||
// `visibleOnFullScreen: true`: on macOS that flag hides the app's Dock
|
||||
// icon for as long as such a window exists (the app becomes an
|
||||
// "agent" app), which reads as Rowboat having vanished. The trade-off
|
||||
// is the popout won't hover over other apps' fullscreen Spaces.
|
||||
win.setAlwaysOnTop(true, 'floating');
|
||||
win.setVisibleOnAllWorkspaces(true);
|
||||
win.webContents.once('did-finish-load', () => {
|
||||
if (lastVideoPopoutState) {
|
||||
win.webContents.send('video:popout-state', lastVideoPopoutState);
|
||||
}
|
||||
// showInactive: appearing must not steal focus from the app the user
|
||||
// switched to — that would immediately re-hide the popout.
|
||||
if (!win.isDestroyed()) win.showInactive();
|
||||
});
|
||||
win.on('closed', () => {
|
||||
if (videoPopoutWin === win) videoPopoutWin = null;
|
||||
});
|
||||
videoPopoutWin = win;
|
||||
if (app.isPackaged) {
|
||||
win.loadURL('app://-/index.html#video-popout');
|
||||
} else {
|
||||
win.loadURL('http://localhost:5173/#video-popout');
|
||||
}
|
||||
return {};
|
||||
},
|
||||
'video:popoutState': async (_event, args) => {
|
||||
lastVideoPopoutState = args;
|
||||
if (videoPopoutWin && !videoPopoutWin.isDestroyed()) {
|
||||
videoPopoutWin.webContents.send('video:popout-state', args);
|
||||
}
|
||||
return {};
|
||||
},
|
||||
'app:focusMainWindow': async () => {
|
||||
const main = findMainAppWindow();
|
||||
if (main) {
|
||||
if (main.isMinimized()) main.restore();
|
||||
main.show();
|
||||
main.focus();
|
||||
}
|
||||
return {};
|
||||
},
|
||||
'video:getPopoutState': async () => {
|
||||
return { state: lastVideoPopoutState };
|
||||
},
|
||||
'video:popoutAction': async (_event, args) => {
|
||||
// Relay a popout control-bar action to the app window, which owns the
|
||||
// call (mic, camera, screen capture) and executes it there. 'expand'
|
||||
// additionally brings the app window back to the foreground.
|
||||
const main = findMainAppWindow();
|
||||
if (args.action === 'expand' && main) {
|
||||
if (main.isMinimized()) main.restore();
|
||||
main.show();
|
||||
main.focus();
|
||||
}
|
||||
main?.webContents.send('video:popout-action', args);
|
||||
return {};
|
||||
},
|
||||
// Live-note handlers
|
||||
'live-note:run': async (_event, args) => {
|
||||
const result = await runLiveNoteAgent(args.filePath, 'manual', args.context);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@ import { app, BrowserWindow, desktopCapturer, protocol, net, shell, session, typ
|
|||
import path from "node:path";
|
||||
import {
|
||||
setupIpcHandlers,
|
||||
startRunsWatcher, startSessionsWatcher,
|
||||
startRunsWatcher, startSessionsWatcher, markSessionsIndexReady,
|
||||
startCodeRunFeedWatcher,
|
||||
startChannelsWatcher,
|
||||
startCodeSessionStatusWatcher,
|
||||
startServicesWatcher,
|
||||
startLiveNoteAgentWatcher,
|
||||
|
|
@ -25,6 +27,7 @@ import { init as initEmailLabeling } from "@x/core/dist/knowledge/label_emails.j
|
|||
import { init as initNoteTagging } from "@x/core/dist/knowledge/tag_notes.js";
|
||||
import { init as initInlineTasks } from "@x/core/dist/knowledge/inline_tasks.js";
|
||||
import { init as initAgentRunner } from "@x/core/dist/agent-schedule/runner.js";
|
||||
import { init as initChannels } from "@x/core/dist/channels/service.js";
|
||||
import { init as initAgentNotes } from "@x/core/dist/knowledge/agent_notes.js";
|
||||
import { init as initCalendarNotifications } from "@x/core/dist/knowledge/notify_calendar_meetings.js";
|
||||
import { init as initMeetingPrep } from "@x/core/dist/knowledge/meeting_prep_scheduler.js";
|
||||
|
|
@ -33,10 +36,12 @@ import { init as initEventProcessor, registerConsumer } from "@x/core/dist/event
|
|||
import { liveNoteEventConsumer } from "@x/core/dist/knowledge/live-note/event-consumer.js";
|
||||
import { init as initBackgroundTaskScheduler } from "@x/core/dist/background-tasks/scheduler.js";
|
||||
import { backgroundTaskEventConsumer } from "@x/core/dist/background-tasks/event-consumer.js";
|
||||
import { init as initLocalSites, shutdown as shutdownLocalSites } from "@x/core/dist/local-sites/server.js";
|
||||
import { startSkillsWatcher, stopSkillsWatcher } from "@x/core/dist/application/assistant/skills/watcher.js";
|
||||
import { init as initAppsServer, shutdown as shutdownAppsServer } from "@x/core/dist/apps/server.js";
|
||||
import { registerAppsHostApi } from "@x/core/dist/apps/host-api.js";
|
||||
import { shutdown as shutdownAnalytics } from "@x/core/dist/analytics/posthog.js";
|
||||
import { identifyIfSignedIn } from "@x/core/dist/analytics/identify.js";
|
||||
import { migrateRuns } from "@x/core/dist/migrations/runs/migrate.js";
|
||||
|
||||
import { initConfigs } from "@x/core/dist/config/initConfigs.js";
|
||||
import { getAgentSlackCliStatus } from "@x/core/dist/slack/agent-slack-exec.js";
|
||||
|
|
@ -390,10 +395,44 @@ app.whenReady().then(async () => {
|
|||
// start runs watcher
|
||||
startRunsWatcher();
|
||||
|
||||
// New runtime: build the in-memory session index (startup scan) before the
|
||||
// renderer can list sessions, then forward the session bus to windows.
|
||||
await container.resolve<ISessions>('sessions').initialize();
|
||||
// One-time: port legacy runs/*.jsonl into the new turn/session runtime.
|
||||
// Must run BEFORE the session index is built so migrated sessions are picked
|
||||
// up by the startup scan. Fully defensive — never blocks boot.
|
||||
try {
|
||||
const migration = migrateRuns();
|
||||
if (migration.scanned > 0) {
|
||||
console.log(
|
||||
`[runs-migration] migrated ${migration.migratedTurns} turn(s) across ` +
|
||||
`${migration.migratedSessions} session(s) from ${migration.scanned} run(s) ` +
|
||||
`(${migration.skipped} skipped, ${migration.failed.length} failed)`,
|
||||
);
|
||||
for (const failure of migration.failed) {
|
||||
console.warn(`[runs-migration] left in place (failed): ${failure.file} — ${failure.error}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[runs-migration] pass failed:', error);
|
||||
}
|
||||
|
||||
// New runtime: build the in-memory session index (startup scan), then
|
||||
// forward the session bus to windows. The renderer window is already up and
|
||||
// may have called sessions:list — that handler blocks on
|
||||
// markSessionsIndexReady, which must fire even if the scan throws so the
|
||||
// list never hangs.
|
||||
try {
|
||||
await container.resolve<ISessions>('sessions').initialize();
|
||||
} finally {
|
||||
markSessionsIndexReady();
|
||||
}
|
||||
startSessionsWatcher();
|
||||
startCodeRunFeedWatcher();
|
||||
|
||||
// Mobile channels (WhatsApp/Telegram bridge): needs the session index, so
|
||||
// start after initialize(). Failures must never block boot.
|
||||
startChannelsWatcher();
|
||||
initChannels().catch((error) => {
|
||||
console.error('[Channels] Failed to start mobile channels:', error);
|
||||
});
|
||||
|
||||
// start code-session status tracker (derives working/needs-you/idle + notifications)
|
||||
startCodeSessionStatusWatcher();
|
||||
|
|
@ -468,9 +507,11 @@ app.whenReady().then(async () => {
|
|||
// start chrome extension sync server
|
||||
initChromeSync();
|
||||
|
||||
// start local sites server for iframe dashboards and other mini apps
|
||||
initLocalSites().catch((error) => {
|
||||
console.error('[LocalSites] Failed to start:', error);
|
||||
// start the Rowboat Apps server (per-app origins on 127.0.0.1:3210) with the
|
||||
// full Host API (tools/fetch/llm/copilot behind the capability gate)
|
||||
registerAppsHostApi();
|
||||
initAppsServer().catch((error) => {
|
||||
console.error('[Apps] Failed to start:', error);
|
||||
});
|
||||
|
||||
app.on("activate", () => {
|
||||
|
|
@ -500,8 +541,8 @@ stopSkillsWatcher();
|
|||
}
|
||||
// Kill embedded terminal shells.
|
||||
disposeAllTerminals();
|
||||
shutdownLocalSites().catch((error) => {
|
||||
console.error('[LocalSites] Failed to shut down cleanly:', error);
|
||||
shutdownAppsServer().catch((error) => {
|
||||
console.error('[Apps] Failed to shut down cleanly:', error);
|
||||
});
|
||||
shutdownAnalytics().catch((error) => {
|
||||
console.error('[Analytics] Failed to flush on quit:', error);
|
||||
|
|
|
|||
62
apps/x/apps/renderer/scripts/generate-tour-audio.mjs
Normal file
62
apps/x/apps/renderer/scripts/generate-tour-audio.mjs
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
/**
|
||||
* Regenerates the bundled product-tour narration clips.
|
||||
*
|
||||
* Parses the TOUR_STEPS texts straight out of product-tour.tsx (so the code
|
||||
* stays the single source of truth), synthesizes each one via @x/core's
|
||||
* synthesizeSpeech (Rowboat proxy when signed in, direct ElevenLabs otherwise,
|
||||
* using the voice id configured there / in ~/.rowboat/config/elevenlabs.json),
|
||||
* and writes MP3s to src/assets/tour/<step-id>.mp3.
|
||||
*
|
||||
* Run whenever a step's narration text or the tour voice changes:
|
||||
* cd apps/x && npm run deps # script imports core's built output
|
||||
* node apps/renderer/scripts/generate-tour-audio.mjs
|
||||
*
|
||||
* Pass step ids to regenerate only those clips (e.g. to re-roll one whose
|
||||
* synthesis came out glitchy):
|
||||
* node apps/renderer/scripts/generate-tour-audio.mjs welcome done
|
||||
*/
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url))
|
||||
const tourSource = path.join(here, '../src/components/product-tour.tsx')
|
||||
const outDir = path.join(here, '../src/assets/tour')
|
||||
const corePath = path.join(here, '../../../packages/core/dist/voice/voice.js')
|
||||
|
||||
const { synthesizeSpeech } = await import(corePath)
|
||||
|
||||
const src = await readFile(tourSource, 'utf8')
|
||||
const start = src.indexOf('const TOUR_STEPS')
|
||||
const end = src.indexOf('\n]', start)
|
||||
if (start === -1 || end === -1) throw new Error('Could not locate TOUR_STEPS in product-tour.tsx')
|
||||
const block = src.slice(start, end)
|
||||
|
||||
const steps = []
|
||||
// voiceText, when present, is the spoken variant of the bubble text.
|
||||
const re = /id: '([^']+)'[\s\S]*?text:\s*("[^"]*"|'[^']*')(?:,\s*voiceText:\s*("[^"]*"|'[^']*'))?/g
|
||||
for (let m; (m = re.exec(block)); ) {
|
||||
// The captures are JS string literals from our own source; evaluate them
|
||||
// to resolve the quoting.
|
||||
steps.push({ id: m[1], text: new Function(`return ${m[3] ?? m[2]}`)() })
|
||||
}
|
||||
if (steps.length === 0) throw new Error('Parsed zero tour steps — regex out of sync with product-tour.tsx?')
|
||||
console.log(`Parsed ${steps.length} tour steps`)
|
||||
|
||||
const only = process.argv.slice(2)
|
||||
if (only.length > 0) {
|
||||
const unknown = only.filter((id) => !steps.some((s) => s.id === id))
|
||||
if (unknown.length > 0) throw new Error(`Unknown step ids: ${unknown.join(', ')}`)
|
||||
steps.splice(0, steps.length, ...steps.filter((s) => only.includes(s.id)))
|
||||
console.log(`Regenerating only: ${only.join(', ')}`)
|
||||
}
|
||||
|
||||
await mkdir(outDir, { recursive: true })
|
||||
for (const step of steps) {
|
||||
process.stdout.write(`synthesizing ${step.id}... `)
|
||||
const { audioBase64 } = await synthesizeSpeech(step.text)
|
||||
const file = path.join(outDir, `${step.id}.mp3`)
|
||||
await writeFile(file, Buffer.from(audioBase64, 'base64'))
|
||||
console.log(`${(audioBase64.length * 0.75 / 1024).toFixed(0)} KB`)
|
||||
}
|
||||
console.log(`Done — ${steps.length} clips in ${path.relative(process.cwd(), outDir)}`)
|
||||
|
|
@ -169,6 +169,25 @@
|
|||
color: var(--gm-placeholder);
|
||||
}
|
||||
|
||||
.gmail-search-clear {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
background: transparent;
|
||||
color: var(--gm-text-muted);
|
||||
cursor: pointer;
|
||||
transition: background 120ms ease, color 120ms ease;
|
||||
}
|
||||
|
||||
.gmail-search-clear:hover {
|
||||
background: var(--gm-icon-hover-bg);
|
||||
color: var(--gm-text);
|
||||
}
|
||||
|
||||
.gmail-icon-button {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
|
|
@ -205,6 +224,46 @@
|
|||
flex-direction: column;
|
||||
}
|
||||
|
||||
/* Native list virtualization: offscreen rows skip layout and paint entirely.
|
||||
Applied only to rows without a mounted ThreadDetail (those hold iframes and
|
||||
composers, which must keep rendering while offscreen). */
|
||||
.gmail-row-group-cv {
|
||||
content-visibility: auto;
|
||||
contain-intrinsic-size: auto 40px;
|
||||
}
|
||||
|
||||
/* While the list is scrolling, rows ignore the pointer so hover restyles and
|
||||
prefetch timers don't compete with frame rendering. */
|
||||
.gmail-shell[data-scrolling] .gmail-row-shell {
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
/* Archived/trashed rows slide out and collapse before removal. Removing the
|
||||
class (failed action) snaps the row back. */
|
||||
.gmail-row-group-leaving {
|
||||
overflow: hidden;
|
||||
pointer-events: none;
|
||||
animation: gmail-row-leave 160ms ease-in forwards;
|
||||
}
|
||||
|
||||
@keyframes gmail-row-leave {
|
||||
0% {
|
||||
opacity: 1;
|
||||
transform: translateX(0);
|
||||
max-height: 48px;
|
||||
}
|
||||
60% {
|
||||
opacity: 0;
|
||||
transform: translateX(32px);
|
||||
max-height: 48px;
|
||||
}
|
||||
100% {
|
||||
opacity: 0;
|
||||
transform: translateX(32px);
|
||||
max-height: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.gmail-list-header {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
|
|
@ -271,6 +330,11 @@
|
|||
opacity: 0;
|
||||
pointer-events: none;
|
||||
transition: opacity 120ms ease;
|
||||
/* The overlay may extend over the subject text; a backdrop matching the
|
||||
row's current background (see --gm-row-actions-bg below) plus a soft left
|
||||
fade keeps it readable instead of colliding. */
|
||||
padding-left: 18px;
|
||||
background: linear-gradient(to right, transparent, var(--gm-row-actions-bg, var(--gm-bg-row-hover)) 18px);
|
||||
}
|
||||
|
||||
.gmail-row-shell:hover .gmail-row-actions {
|
||||
|
|
@ -278,6 +342,19 @@
|
|||
pointer-events: auto;
|
||||
}
|
||||
|
||||
/* Backdrop color tracks the row state underneath the overlay. */
|
||||
.gmail-row-shell:hover {
|
||||
--gm-row-actions-bg: var(--gm-bg-row-hover);
|
||||
}
|
||||
|
||||
.gmail-row-shell:hover:has(.gmail-row-selected) {
|
||||
--gm-row-actions-bg: var(--gm-bg-row-selected-hover);
|
||||
}
|
||||
|
||||
.gmail-row-shell:hover:has(.gmail-row-selected.gmail-row-focused) {
|
||||
--gm-row-actions-bg: var(--gm-bg-row-selected);
|
||||
}
|
||||
|
||||
.gmail-row-shell:hover .gmail-row-date {
|
||||
visibility: hidden;
|
||||
}
|
||||
|
|
@ -305,7 +382,7 @@
|
|||
color: var(--destructive);
|
||||
}
|
||||
|
||||
.gmail-row:hover {
|
||||
.gmail-row-shell:hover .gmail-row {
|
||||
background: var(--gm-bg-row-hover);
|
||||
box-shadow: none;
|
||||
}
|
||||
|
|
@ -315,10 +392,24 @@
|
|||
box-shadow: inset 2px 0 0 var(--gm-accent);
|
||||
}
|
||||
|
||||
.gmail-row-selected:hover {
|
||||
.gmail-row-shell:hover .gmail-row-selected {
|
||||
background: var(--gm-bg-row-selected-hover);
|
||||
}
|
||||
|
||||
/* The j/k keyboard cursor. Declared after the hover rules (same specificity)
|
||||
so the focus ring survives hovering the focused row. */
|
||||
.gmail-row-focused,
|
||||
.gmail-row-shell:hover .gmail-row-focused {
|
||||
background: var(--gm-bg-row-hover);
|
||||
box-shadow: inset 0 0 0 1px var(--gm-accent);
|
||||
}
|
||||
|
||||
.gmail-row-selected.gmail-row-focused,
|
||||
.gmail-row-shell:hover .gmail-row-selected.gmail-row-focused {
|
||||
background: var(--gm-bg-row-selected);
|
||||
box-shadow: inset 2px 0 0 var(--gm-accent), inset 0 0 0 1px var(--gm-accent);
|
||||
}
|
||||
|
||||
.gmail-row-unread {
|
||||
color: var(--gm-text);
|
||||
}
|
||||
|
|
@ -394,12 +485,50 @@
|
|||
border-top: 1px solid var(--gm-border);
|
||||
border-bottom: 1px solid var(--gm-border);
|
||||
box-shadow: inset 2px 0 0 var(--gm-accent);
|
||||
/* Replays whenever the detail is shown — hidden details are display: none,
|
||||
so un-hiding restarts the animation. */
|
||||
animation: gmail-detail-open 140ms ease-out;
|
||||
}
|
||||
|
||||
.gmail-detail-hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
@keyframes gmail-detail-open {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(-6px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: none;
|
||||
}
|
||||
}
|
||||
|
||||
/* The inline reply/forward composer pops in from below the thread. */
|
||||
.gmail-compose-inline {
|
||||
animation: gmail-compose-open 140ms ease-out;
|
||||
}
|
||||
|
||||
@keyframes gmail-compose-open {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(8px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: none;
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.gmail-detail-inline,
|
||||
.gmail-compose-inline,
|
||||
.gmail-row-group-leaving {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
|
||||
.gmail-detail-toolbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
BIN
apps/x/apps/renderer/src/assets/tour/agents.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/agents.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/chats.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/chats.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/code.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/code.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/composer.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/composer.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/done.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/done.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/email.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/email.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/home.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/home.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/knowledge.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/knowledge.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/meetings.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/meetings.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/welcome.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/welcome.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/workspaces.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/workspaces.mp3
Normal file
Binary file not shown.
|
|
@ -22,7 +22,7 @@ import {
|
|||
import { type ComponentProps, type ReactNode, isValidElement, useState } from "react";
|
||||
import { AnimatePresence, motion } from "motion/react";
|
||||
import type { ToolCall, ToolGroup as ToolGroupType } from "@/lib/chat-conversation";
|
||||
import { getToolActionsSummary, getToolDisplayName, getToolGroupSummary, toToolState } from "@/lib/chat-conversation";
|
||||
import { getToolActionsSummary, getToolDisplayName, getToolErrorText, getToolGroupSummary, toToolState } from "@/lib/chat-conversation";
|
||||
|
||||
const formatToolValue = (value: unknown) => {
|
||||
if (typeof value === "string") return value;
|
||||
|
|
@ -329,7 +329,7 @@ export const ToolGroupComponent = ({ group, isToolOpen, onToolOpenChange }: Tool
|
|||
<ToolTabbedContent
|
||||
input={tool.input as ToolUIPart["input"]}
|
||||
output={tool.result as ToolUIPart["output"]}
|
||||
errorText={tool.status === 'error' ? 'Tool error' : undefined}
|
||||
errorText={getToolErrorText(tool)}
|
||||
/>
|
||||
</ToolContent>
|
||||
</Tool>
|
||||
|
|
|
|||
186
apps/x/apps/renderer/src/components/apps/app-detail.tsx
Normal file
186
apps/x/apps/renderer/src/components/apps/app-detail.tsx
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
import { useEffect, useState } from 'react'
|
||||
import { X, RotateCcw, Play } from 'lucide-react'
|
||||
import type { rowboatApp } from '@x/shared'
|
||||
|
||||
// App detail panel (spec §14): manifest info, provenance/publish state,
|
||||
// bundled agents with enable toggles, rollback when available. Update/publish
|
||||
// actions land with M3.
|
||||
|
||||
type AgentRow = {
|
||||
slug: string
|
||||
name: string
|
||||
active: boolean
|
||||
lastRunAt?: string
|
||||
lastRunError?: string
|
||||
}
|
||||
|
||||
export function AppDetail({ folder, onClose }: { folder: string; onClose: () => void }) {
|
||||
const [app, setApp] = useState<rowboatApp.AppSummary | null>(null)
|
||||
const [readme, setReadme] = useState<string | undefined>(undefined)
|
||||
const [rollback, setRollback] = useState(false)
|
||||
const [agents, setAgents] = useState<AgentRow[]>([])
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [reloadNonce, setReloadNonce] = useState(0)
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
;(async () => {
|
||||
try {
|
||||
const r = await window.ipc.invoke('apps:get', { folder })
|
||||
if (cancelled) return
|
||||
setApp(r.app)
|
||||
setReadme(r.readme)
|
||||
setRollback(r.rollbackAvailable)
|
||||
const rows: AgentRow[] = []
|
||||
for (const slug of r.app.agentSlugs) {
|
||||
try {
|
||||
const t = await window.ipc.invoke('bg-task:get', { slug })
|
||||
if (t.task) {
|
||||
rows.push({
|
||||
slug,
|
||||
name: t.task.name,
|
||||
active: t.task.active,
|
||||
lastRunAt: t.task.lastRunAt,
|
||||
lastRunError: t.task.lastRunError,
|
||||
})
|
||||
}
|
||||
} catch { /* not materialized yet */ }
|
||||
}
|
||||
if (!cancelled) setAgents(rows)
|
||||
} catch (e) {
|
||||
if (!cancelled) setError(e instanceof Error ? e.message : String(e))
|
||||
}
|
||||
})()
|
||||
return () => { cancelled = true }
|
||||
}, [folder, reloadNonce])
|
||||
|
||||
const toggleAgent = async (slug: string, active: boolean) => {
|
||||
setAgents((prev) => prev.map((a) => (a.slug === slug ? { ...a, active } : a)))
|
||||
try {
|
||||
await window.ipc.invoke('bg-task:patch', { slug, partial: { active } })
|
||||
} catch {
|
||||
setReloadNonce((n) => n + 1) // revert to truth on failure
|
||||
}
|
||||
}
|
||||
|
||||
const runAgent = async (slug: string) => {
|
||||
try {
|
||||
await window.ipc.invoke('bg-task:run', { slug })
|
||||
} catch { /* surfaced via bg-task UI */ }
|
||||
}
|
||||
|
||||
const manifest = app?.manifest
|
||||
|
||||
return (
|
||||
<div className="flex h-full flex-col overflow-hidden">
|
||||
<div className="flex items-center gap-2 border-b border-border px-4 py-2.5">
|
||||
<span className="flex-1 truncate text-sm font-semibold">{manifest?.name ?? folder}</span>
|
||||
<button type="button" onClick={onClose} className="rounded-md p-1.5 text-muted-foreground hover:bg-accent hover:text-foreground">
|
||||
<X className="size-4" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="min-h-0 flex-1 overflow-y-auto px-4 py-4 text-sm">
|
||||
{error && <div className="mb-3 rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-destructive">{error}</div>}
|
||||
{!app ? (
|
||||
<div className="text-muted-foreground">Loading…</div>
|
||||
) : (
|
||||
<div className="space-y-5">
|
||||
<section className="space-y-1.5">
|
||||
<div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">App</div>
|
||||
{app.status === 'invalid' && (
|
||||
<div className="rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-xs text-destructive">
|
||||
Invalid manifest: {app.manifestError}
|
||||
</div>
|
||||
)}
|
||||
<InfoRow k="Version" v={manifest ? `v${manifest.version}` : '—'} />
|
||||
<InfoRow k="Folder" v={app.folder} />
|
||||
<InfoRow k="Origin" v={app.origin} mono />
|
||||
{manifest?.description ? <p className="pt-1 text-muted-foreground">{manifest.description}</p> : null}
|
||||
</section>
|
||||
|
||||
<section className="space-y-1.5">
|
||||
<div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">Capabilities</div>
|
||||
{manifest && manifest.capabilities.length > 0 ? (
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{manifest.capabilities.map((c) => (
|
||||
<span key={c} className="rounded-full bg-muted px-2.5 py-0.5 text-xs font-medium text-muted-foreground">{c}</span>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<p className="text-muted-foreground">None declared — this app can’t use tools, LLM, or the copilot.</p>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<section className="space-y-1.5">
|
||||
<div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">Source</div>
|
||||
{app.kind === 'installed' && app.install ? (
|
||||
<>
|
||||
<InfoRow k="Installed from" v={app.install.repo ?? app.install.sourceUrl ?? 'unknown'} mono />
|
||||
<InfoRow k="Installed" v={new Date(app.install.installedAt).toLocaleString()} />
|
||||
{app.install.updatedAt && <InfoRow k="Updated" v={new Date(app.install.updatedAt).toLocaleString()} />}
|
||||
</>
|
||||
) : (
|
||||
<p className="text-muted-foreground">Local app — created on this machine{app.publish ? `, published as ${app.publish.repo}` : ''}.</p>
|
||||
)}
|
||||
{rollback && (
|
||||
<button type="button" className="mt-1 flex items-center gap-1.5 rounded-md border border-border px-2.5 py-1 text-xs font-medium hover:bg-accent">
|
||||
<RotateCcw className="size-3.5" /> Roll back to previous version
|
||||
</button>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<section className="space-y-2">
|
||||
<div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">Background agents</div>
|
||||
{agents.length === 0 ? (
|
||||
<p className="text-muted-foreground">No bundled agents.</p>
|
||||
) : agents.map((a) => (
|
||||
<div key={a.slug} className="flex items-center gap-2 rounded-lg border border-border px-3 py-2">
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="truncate text-sm font-medium">{a.name}</div>
|
||||
<div className="truncate text-xs text-muted-foreground">
|
||||
{a.lastRunError ? `Failed: ${a.lastRunError}` : a.lastRunAt ? `Last run ${new Date(a.lastRunAt).toLocaleString()}` : 'Never run'}
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
title="Run now"
|
||||
onClick={() => void runAgent(a.slug)}
|
||||
className="rounded-md p-1.5 text-muted-foreground hover:bg-accent hover:text-foreground"
|
||||
>
|
||||
<Play className="size-3.5" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
role="switch"
|
||||
aria-checked={a.active}
|
||||
onClick={() => void toggleAgent(a.slug, !a.active)}
|
||||
className={`relative h-5 w-9 rounded-full transition ${a.active ? 'bg-primary' : 'bg-muted'}`}
|
||||
>
|
||||
<span className={`absolute top-0.5 size-4 rounded-full bg-background shadow transition-all ${a.active ? 'left-[18px]' : 'left-0.5'}`} />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</section>
|
||||
|
||||
{readme && (
|
||||
<section className="space-y-1.5">
|
||||
<div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">README</div>
|
||||
<pre className="whitespace-pre-wrap rounded-lg border border-border bg-muted/40 p-3 text-xs leading-relaxed">{readme}</pre>
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function InfoRow({ k, v, mono }: { k: string; v: string; mono?: boolean }) {
|
||||
return (
|
||||
<div className="flex items-baseline gap-2">
|
||||
<span className="w-28 shrink-0 text-xs text-muted-foreground">{k}</span>
|
||||
<span className={`min-w-0 truncate ${mono ? 'font-mono text-xs' : ''}`}>{v}</span>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
105
apps/x/apps/renderer/src/components/apps/app-frame.tsx
Normal file
105
apps/x/apps/renderer/src/components/apps/app-frame.tsx
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
import { useEffect, useState } from 'react'
|
||||
import { ArrowLeft, ExternalLink, Info, RotateCw } from 'lucide-react'
|
||||
import type { rowboatApp } from '@x/shared'
|
||||
import { appOpened } from '@/lib/analytics'
|
||||
import { AppDetail } from '@/components/apps/app-detail'
|
||||
|
||||
// Full-height iframe on the app's own origin (spec §6.6). No sandbox attr —
|
||||
// per-app browser origins are the isolation boundary. Toolbar: back, reload,
|
||||
// open-in-browser, detail panel.
|
||||
|
||||
export function AppFrame({ app, onBack }: { app: rowboatApp.AppSummary; onBack: () => void }) {
|
||||
const [reloadNonce, setReloadNonce] = useState(0)
|
||||
const [showDetail, setShowDetail] = useState(false)
|
||||
// Load watchdog: if the iframe hasn't fired `load` within the deadline,
|
||||
// surface a visible retry state instead of a silent blank pane.
|
||||
const [loadState, setLoadState] = useState<'loading' | 'ok' | 'stuck'>('loading')
|
||||
const title = app.manifest?.name ?? app.folder
|
||||
|
||||
useEffect(() => {
|
||||
appOpened(app.folder)
|
||||
}, [app.folder])
|
||||
|
||||
// Reset the watchdog when the target changes (adjust-during-render pattern).
|
||||
const [watchKey, setWatchKey] = useState(`${app.folder}:${reloadNonce}`)
|
||||
if (watchKey !== `${app.folder}:${reloadNonce}`) {
|
||||
setWatchKey(`${app.folder}:${reloadNonce}`)
|
||||
setLoadState('loading')
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
const timer = window.setTimeout(() => {
|
||||
setLoadState((s) => (s === 'loading' ? 'stuck' : s))
|
||||
}, 6000)
|
||||
return () => window.clearTimeout(timer)
|
||||
}, [watchKey])
|
||||
|
||||
return (
|
||||
<div className="flex h-full flex-col">
|
||||
<div className="flex items-center gap-2 border-b border-border px-3 py-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onBack}
|
||||
className="flex items-center gap-1.5 rounded-md px-2 py-1 text-sm text-muted-foreground hover:bg-accent hover:text-foreground"
|
||||
>
|
||||
<ArrowLeft className="size-4" />
|
||||
Apps
|
||||
</button>
|
||||
<span className="flex-1 truncate text-sm font-medium">{title}</span>
|
||||
<button
|
||||
type="button"
|
||||
title="Reload"
|
||||
onClick={() => setReloadNonce((n) => n + 1)}
|
||||
className="rounded-md p-1.5 text-muted-foreground hover:bg-accent hover:text-foreground"
|
||||
>
|
||||
<RotateCw className="size-4" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
title="Open in browser"
|
||||
onClick={() => window.open(app.origin, '_blank')}
|
||||
className="rounded-md p-1.5 text-muted-foreground hover:bg-accent hover:text-foreground"
|
||||
>
|
||||
<ExternalLink className="size-4" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
title="App details"
|
||||
onClick={() => setShowDetail((v) => !v)}
|
||||
className={`rounded-md p-1.5 hover:bg-accent hover:text-foreground ${showDetail ? 'text-foreground' : 'text-muted-foreground'}`}
|
||||
>
|
||||
<Info className="size-4" />
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex min-h-0 flex-1">
|
||||
<div className="relative min-w-0 flex-1">
|
||||
<iframe
|
||||
key={reloadNonce}
|
||||
title={title}
|
||||
src={`${app.origin}/`}
|
||||
onLoad={() => setLoadState('ok')}
|
||||
className="h-full w-full border-0 bg-background"
|
||||
/>
|
||||
{loadState === 'stuck' && (
|
||||
<div className="absolute inset-0 flex flex-col items-center justify-center gap-3 bg-background/95 text-sm">
|
||||
<div className="text-muted-foreground">This app is taking too long to load.</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setReloadNonce((n) => n + 1)}
|
||||
className="rounded-md border border-border px-3 py-1.5 font-medium hover:bg-accent"
|
||||
>
|
||||
Retry
|
||||
</button>
|
||||
<div className="font-mono text-xs text-muted-foreground">{app.origin}</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{showDetail && (
|
||||
<div className="w-80 shrink-0 border-l border-border">
|
||||
<AppDetail folder={app.folder} onClose={() => setShowDetail(false)} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
238
apps/x/apps/renderer/src/components/apps/apps-view.tsx
Normal file
238
apps/x/apps/renderer/src/components/apps/apps-view.tsx
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
import { useEffect, useState } from 'react'
|
||||
import { Plus, RefreshCw } from 'lucide-react'
|
||||
import type { rowboatApp } from '@x/shared'
|
||||
import { AppFrame } from '@/components/apps/app-frame'
|
||||
|
||||
// Apps home (spec §14): "My apps" grid + Catalog placeholder (M3). Cards are
|
||||
// AppSummary-driven; click opens the app full-height on its own origin.
|
||||
|
||||
type Theme = { accent: string; glow: string }
|
||||
|
||||
const THEMES: Theme[] = [
|
||||
{ accent: '#FF4D8D', glow: 'rgba(255,77,141,0.45)' }, // Pink
|
||||
{ accent: '#EF4444', glow: 'rgba(239,68,68,0.45)' }, // Red
|
||||
{ accent: '#22C55E', glow: 'rgba(34,197,94,0.40)' }, // Emerald
|
||||
{ accent: '#F59E0B', glow: 'rgba(245,158,11,0.42)' }, // Amber
|
||||
{ accent: '#14B8A6', glow: 'rgba(20,184,166,0.40)' }, // Teal
|
||||
{ accent: '#EC4899', glow: 'rgba(236,72,153,0.42)' }, // Rose
|
||||
]
|
||||
const PATTERNS = ['dots', 'grid', 'diagonal', 'radial', 'waves', 'mesh', 'cross', 'rings', 'zigzag', 'plus', 'checker', 'beams']
|
||||
|
||||
function hash(s: string): number {
|
||||
let h = 0
|
||||
for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) | 0
|
||||
return Math.abs(h)
|
||||
}
|
||||
const themeForIndex = (i: number): Theme => THEMES[i % THEMES.length]
|
||||
const patternFor = (id: string): string => PATTERNS[hash(id + '·pat') % PATTERNS.length]
|
||||
|
||||
const CARD_CSS = `
|
||||
.ma-page {
|
||||
container-type: inline-size;
|
||||
--ma-bg:#f8f8f9;
|
||||
--ma-card-from:#ffffff; --ma-card-mid:#f2f3f6; --ma-card-to:#e6e8ee;
|
||||
--ma-card-hover-from:#ffffff; --ma-card-hover-mid:#f5f6f9; --ma-card-hover-to:#eaecf1;
|
||||
--ma-sheen:rgba(255,255,255,0.55); --ma-top-highlight:rgba(255,255,255,0.9);
|
||||
--ma-border:rgba(0,0,0,0.09); --ma-border-hover:rgba(0,0,0,0.15);
|
||||
--ma-shadow:0 1px 2px rgba(0,0,0,0.08);
|
||||
--ma-title:#0d0e11; --ma-desc:rgba(0,0,0,0.6);
|
||||
--ma-h1:#0d0e11; --ma-sub:rgba(0,0,0,0.5); --ma-lastrun:rgba(0,0,0,0.42);
|
||||
--ma-off-bg:rgba(0,0,0,0.05); --ma-off-fg:rgba(0,0,0,0.5);
|
||||
--ma-new-border:rgba(0,0,0,0.14); --ma-new-title:rgba(0,0,0,0.6); --ma-new-hint:rgba(0,0,0,0.4);
|
||||
--ma-pat-opacity:0.10; --ma-glow-opacity:0.16; --ma-glow-hover-opacity:0.24;
|
||||
--ma-badge-mix:20%; --ma-pill-mix:16%; --ma-tint:16%; --ma-tint-hover:22%;
|
||||
height:100%; overflow:auto; background:var(--ma-bg);
|
||||
}
|
||||
.dark .ma-page {
|
||||
--ma-bg:#0b0b0d;
|
||||
--ma-card-from:#262930; --ma-card-mid:#191b21; --ma-card-to:#101116;
|
||||
--ma-card-hover-from:#2b2e36; --ma-card-hover-mid:#1c1e25; --ma-card-hover-to:#131419;
|
||||
--ma-sheen:rgba(255,255,255,0.07); --ma-top-highlight:rgba(255,255,255,0.09);
|
||||
--ma-border:rgba(255,255,255,0.07); --ma-border-hover:rgba(255,255,255,0.12);
|
||||
--ma-shadow:0 1px 2px rgba(0,0,0,0.35);
|
||||
--ma-title:#f4f5f7; --ma-desc:rgba(255,255,255,0.66);
|
||||
--ma-h1:#f4f5f7; --ma-sub:rgba(255,255,255,0.52); --ma-lastrun:rgba(255,255,255,0.38);
|
||||
--ma-off-bg:rgba(255,255,255,0.06); --ma-off-fg:rgba(255,255,255,0.5);
|
||||
--ma-new-border:rgba(255,255,255,0.12); --ma-new-title:rgba(255,255,255,0.6); --ma-new-hint:rgba(255,255,255,0.38);
|
||||
--ma-pat-opacity:0.05; --ma-glow-opacity:0.10; --ma-glow-hover-opacity:0.16;
|
||||
--ma-badge-mix:15%; --ma-pill-mix:13%; --ma-tint:20%; --ma-tint-hover:26%;
|
||||
}
|
||||
.ma-inner { max-width:1120px; margin:0 auto; padding:clamp(20px,3.5cqw,34px) clamp(16px,3cqw,30px) 48px; }
|
||||
.ma-h1 { font-size:clamp(19px,2.6cqw,24px); font-weight:650; letter-spacing:-0.02em; color:var(--ma-h1); margin:0 0 4px; }
|
||||
.ma-sub { font-size:clamp(13px,1.5cqw,14px); color:var(--ma-sub); margin:0 0 clamp(14px,2cqw,20px); }
|
||||
.ma-tabs { display:flex; gap:6px; margin-bottom:clamp(14px,2cqw,22px); }
|
||||
.ma-tab { border:1px solid var(--ma-border); background:transparent; color:var(--ma-sub); border-radius:999px; padding:5px 14px; font-size:13px; font-weight:600; cursor:pointer; }
|
||||
.ma-tab.on { color:var(--ma-title); border-color:var(--ma-border-hover); background:color-mix(in srgb, var(--ma-title) 6%, transparent); }
|
||||
.ma-banner { border:1px solid rgba(239,68,68,.4); background:rgba(239,68,68,.1); color:var(--ma-title); border-radius:12px; padding:10px 14px; font-size:13px; margin-bottom:16px; }
|
||||
.ma-grid { display:grid; grid-template-columns:repeat(auto-fill, minmax(min(100%,248px),1fr)); gap:clamp(14px,2cqw,24px); }
|
||||
.ma-card {
|
||||
position:relative; min-height:clamp(190px,24cqw,244px); border-radius:18px;
|
||||
border:1px solid var(--ma-border);
|
||||
background:
|
||||
linear-gradient(135deg, var(--ma-sheen) 0%, transparent 34%),
|
||||
linear-gradient(158deg, color-mix(in srgb, var(--accent) var(--ma-tint), transparent) 0%, transparent 62%),
|
||||
linear-gradient(158deg, var(--ma-card-from) 0%, var(--ma-card-mid) 52%, var(--ma-card-to) 100%);
|
||||
padding:clamp(15px,2cqw,22px); text-align:left; cursor:pointer; overflow:hidden;
|
||||
display:flex; flex-direction:column; isolation:isolate;
|
||||
box-shadow: var(--ma-shadow), inset 0 1px 0 var(--ma-top-highlight), 0 8px 22px -20px var(--glow);
|
||||
transition: box-shadow .22s ease, border-color .22s ease, background .22s ease;
|
||||
}
|
||||
.ma-card:hover {
|
||||
border-color: var(--ma-border-hover);
|
||||
background:
|
||||
linear-gradient(135deg, var(--ma-sheen) 0%, transparent 36%),
|
||||
linear-gradient(158deg, color-mix(in srgb, var(--accent) var(--ma-tint-hover), transparent) 0%, transparent 64%),
|
||||
linear-gradient(158deg, var(--ma-card-hover-from) 0%, var(--ma-card-hover-mid) 52%, var(--ma-card-hover-to) 100%);
|
||||
}
|
||||
.ma-card::before { content:''; position:absolute; inset:0; z-index:-1; opacity:var(--ma-pat-opacity); pointer-events:none; }
|
||||
.ma-card::after {
|
||||
content:''; position:absolute; top:-45%; right:-25%; width:75%; height:75%; z-index:-1;
|
||||
background: radial-gradient(circle, var(--accent) 0%, transparent 70%);
|
||||
opacity:var(--ma-glow-opacity); filter: blur(18px); pointer-events:none; transition: opacity .22s ease;
|
||||
}
|
||||
.ma-card:hover::after { opacity:var(--ma-glow-hover-opacity); }
|
||||
.ma-pat-dots::before { background-image: radial-gradient(var(--accent) 1px, transparent 1.4px); background-size:16px 16px; }
|
||||
.ma-pat-grid::before { background-image: linear-gradient(var(--accent) 1px, transparent 1px), linear-gradient(90deg, var(--accent) 1px, transparent 1px); background-size:26px 26px; }
|
||||
.ma-pat-diagonal::before { background-image: repeating-linear-gradient(45deg, var(--accent) 0 1px, transparent 1px 14px); }
|
||||
.ma-pat-radial::before { background-image: radial-gradient(circle at 78% 18%, var(--accent) 0%, transparent 55%); opacity:calc(var(--ma-pat-opacity) + 0.05); }
|
||||
.ma-pat-waves::before { background-image: repeating-radial-gradient(circle at 50% -30%, transparent 0 20px, var(--accent) 20px 21px); }
|
||||
.ma-pat-mesh::before { background-image: radial-gradient(circle at 12% 18%, var(--accent) 0%, transparent 42%), radial-gradient(circle at 88% 82%, var(--accent) 0%, transparent 42%); opacity:calc(var(--ma-pat-opacity) + 0.03); }
|
||||
.ma-pat-cross::before { background-image: repeating-linear-gradient(45deg, var(--accent) 0 1px, transparent 1px 18px), repeating-linear-gradient(-45deg, var(--accent) 0 1px, transparent 1px 18px); }
|
||||
.ma-pat-rings::before { background-image: repeating-radial-gradient(circle at 82% 20%, transparent 0 14px, var(--accent) 14px 15px); }
|
||||
.ma-pat-zigzag::before { background-image: linear-gradient(135deg, var(--accent) 25%, transparent 25%), linear-gradient(225deg, var(--accent) 25%, transparent 25%); background-size: 22px 12px; background-position: 0 0, 11px 0; opacity:calc(var(--ma-pat-opacity) - 0.03); }
|
||||
.ma-pat-plus::before { background-image: radial-gradient(var(--accent) 0.8px, transparent 1px), linear-gradient(var(--accent) 1px, transparent 1px), linear-gradient(90deg, var(--accent) 1px, transparent 1px); background-size: 24px 24px, 24px 24px, 24px 24px; background-position: 12px 12px, 0 11.5px, 11.5px 0; opacity:calc(var(--ma-pat-opacity) - 0.02); }
|
||||
.ma-pat-checker::before { background-image: repeating-conic-gradient(var(--accent) 0% 25%, transparent 0% 50%); background-size: 26px 26px; opacity:calc(var(--ma-pat-opacity) - 0.04); }
|
||||
.ma-pat-beams::before { background-image: repeating-linear-gradient(100deg, var(--accent) 0 2px, transparent 2px 34px); }
|
||||
.ma-top { display:flex; justify-content:flex-end; gap:6px; }
|
||||
.ma-badge {
|
||||
display:inline-flex; align-items:center; height:22px; padding:0 10px; border-radius:999px;
|
||||
font-size:9.5px; font-weight:600; letter-spacing:0.07em;
|
||||
color: var(--accent); background: color-mix(in srgb, var(--accent) var(--ma-badge-mix), transparent);
|
||||
}
|
||||
.ma-badge.off { color: var(--ma-off-fg); background: var(--ma-off-bg); }
|
||||
.ma-badge.err { color:#ef4444; background:rgba(239,68,68,.14); }
|
||||
.ma-title { font-size:clamp(17px,2.3cqw,21px); font-weight:600; letter-spacing:-0.02em; color:var(--ma-title); margin:clamp(12px,2cqw,18px) 0 8px; }
|
||||
.ma-desc {
|
||||
font-size:clamp(13px,1.5cqw,14.5px); font-weight:400; line-height:1.45; color:var(--ma-desc); margin:0;
|
||||
display:-webkit-box; -webkit-line-clamp:2; -webkit-box-orient:vertical; overflow:hidden;
|
||||
}
|
||||
.ma-footer { margin-top:auto; padding-top:clamp(14px,2cqw,22px); display:flex; align-items:center; justify-content:space-between; gap:10px; }
|
||||
.ma-source { font-size:11.5px; font-weight:600; color:var(--accent); background: color-mix(in srgb, var(--accent) var(--ma-pill-mix), transparent); padding:5px 10px; border-radius:999px; white-space:nowrap; }
|
||||
.ma-lastrun { font-size:11.5px; color:var(--ma-lastrun); white-space:nowrap; overflow:hidden; text-overflow:ellipsis; }
|
||||
.ma-new {
|
||||
width:100%; font:inherit; min-height:clamp(190px,24cqw,244px); border-radius:18px; border:1px dashed var(--ma-new-border);
|
||||
background:transparent; display:flex; flex-direction:column; align-items:center; justify-content:center;
|
||||
gap:8px; color:var(--ma-new-title); cursor:pointer; transition: border-color .2s ease, background .2s ease;
|
||||
}
|
||||
.ma-new:hover { border-color:var(--ma-border-hover); background:color-mix(in srgb, var(--accent, #888) 6%, transparent); }
|
||||
.ma-new-title { font-size:14.5px; font-weight:600; color:var(--ma-new-title); }
|
||||
.ma-new-hint { font-size:12px; color:var(--ma-new-hint); text-align:center; padding:0 12px; }
|
||||
.ma-empty { padding:36px 8px; text-align:center; color:var(--ma-sub); font-size:14px; grid-column:1/-1; }
|
||||
@container (max-width: 380px) {
|
||||
.ma-footer { flex-direction:column; align-items:flex-start; gap:6px; }
|
||||
}
|
||||
`
|
||||
|
||||
function Card({ app, index, onOpen }: { app: rowboatApp.AppSummary; index: number; onOpen: () => void }) {
|
||||
const theme = themeForIndex(index)
|
||||
const pattern = patternFor(app.folder)
|
||||
const invalid = app.status === 'invalid'
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onOpen}
|
||||
title={invalid ? app.manifestError : undefined}
|
||||
className={`ma-card ma-pat-${pattern}`}
|
||||
style={{ '--accent': theme.accent, '--glow': theme.glow } as React.CSSProperties}
|
||||
>
|
||||
<div className="ma-top">
|
||||
{invalid && <span className="ma-badge err">INVALID</span>}
|
||||
<span className={`ma-badge${app.kind === 'installed' ? '' : ' off'}`}>
|
||||
{app.kind === 'installed' ? 'INSTALLED' : 'LOCAL'}
|
||||
</span>
|
||||
</div>
|
||||
<div className="ma-title">{app.manifest?.name ?? app.folder}</div>
|
||||
<div className="ma-desc">{invalid ? (app.manifestError ?? 'Invalid manifest') : (app.manifest?.description || 'No description yet.')}</div>
|
||||
<div className="ma-footer">
|
||||
<span className="ma-source">v{app.manifest?.version ?? '?'}</span>
|
||||
<span className="ma-lastrun">{app.folder}</span>
|
||||
</div>
|
||||
</button>
|
||||
)
|
||||
}
|
||||
|
||||
export function AppsView({ initialAppFolder, initialVersion, onNewApp }: {
|
||||
initialAppFolder?: string | null
|
||||
initialVersion?: number
|
||||
onNewApp?: () => void
|
||||
} = {}) {
|
||||
const [tab, setTab] = useState<'mine' | 'catalog'>('mine')
|
||||
const [selectedFolder, setSelectedFolder] = useState<string | null>(initialAppFolder ?? null)
|
||||
const [apps, setApps] = useState<rowboatApp.AppSummary[]>([])
|
||||
const [serverError, setServerError] = useState<string | null>(null)
|
||||
|
||||
// Open a specific app when asked from outside (app-navigation open-app).
|
||||
const [appliedVersion, setAppliedVersion] = useState(initialVersion)
|
||||
if (initialVersion !== appliedVersion) {
|
||||
setAppliedVersion(initialVersion)
|
||||
if (initialAppFolder) setSelectedFolder(initialAppFolder)
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
const load = async () => {
|
||||
try {
|
||||
const r = await window.ipc.invoke('apps:list', {})
|
||||
if (cancelled) return
|
||||
setApps(r.apps)
|
||||
setServerError(r.serverRunning ? null : (r.serverError ?? 'Apps server is not running.'))
|
||||
} catch (e) {
|
||||
if (!cancelled) setServerError(e instanceof Error ? e.message : String(e))
|
||||
}
|
||||
}
|
||||
void load()
|
||||
const interval = setInterval(load, 4000) // keep the grid fresh (copilot installs)
|
||||
return () => { cancelled = true; clearInterval(interval) }
|
||||
}, [initialVersion])
|
||||
|
||||
const selected = selectedFolder ? apps.find((a) => a.folder === selectedFolder) : undefined
|
||||
if (selected) {
|
||||
return <AppFrame app={selected} onBack={() => setSelectedFolder(null)} />
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="ma-page">
|
||||
<style>{CARD_CSS}</style>
|
||||
<div className="ma-inner">
|
||||
<h1 className="ma-h1">Apps</h1>
|
||||
<p className="ma-sub">Apps that live inside Rowboat, powered by your agents and integrations.</p>
|
||||
|
||||
<div className="ma-tabs">
|
||||
<button type="button" className={`ma-tab${tab === 'mine' ? ' on' : ''}`} onClick={() => setTab('mine')}>My apps</button>
|
||||
<button type="button" className={`ma-tab${tab === 'catalog' ? ' on' : ''}`} onClick={() => setTab('catalog')}>Catalog</button>
|
||||
</div>
|
||||
|
||||
{serverError && (
|
||||
<div className="ma-banner">
|
||||
<RefreshCw className="mr-1.5 inline size-3.5" /> Apps server unavailable: {serverError}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{tab === 'catalog' ? (
|
||||
<div className="ma-empty">The community catalog is coming soon.</div>
|
||||
) : (
|
||||
<div className="ma-grid">
|
||||
{apps.map((app, i) => (
|
||||
<Card key={app.folder} app={app} index={i} onOpen={() => setSelectedFolder(app.folder)} />
|
||||
))}
|
||||
<button type="button" className="ma-new" onClick={onNewApp}>
|
||||
<Plus className="size-5" />
|
||||
<div className="ma-new-title">New app</div>
|
||||
<div className="ma-new-hint">Describe one to the copilot</div>
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -1,7 +1,19 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { ArrowLeft, ArrowRight, Loader2, Plus, RotateCw, X } from 'lucide-react'
|
||||
|
||||
import type { HttpAuthRequest } from '@x/shared/dist/browser-control.js'
|
||||
|
||||
import { TabBar } from '@/components/tab-bar'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { cn } from '@/lib/utils'
|
||||
|
||||
/**
|
||||
|
|
@ -86,9 +98,80 @@ const getBrowserTabTitle = (tab: BrowserTabState) => {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Credential prompt for HTTP basic/proxy auth challenges raised by pages in
|
||||
* the embedded browser. Rendered as a regular app dialog: BrowserPane already
|
||||
* hides the native WebContentsView whenever a dialog overlay is open, so the
|
||||
* prompt is never obscured by the page that triggered it.
|
||||
*/
|
||||
function BrowserHttpAuthDialog({
|
||||
request,
|
||||
onSubmit,
|
||||
onCancel,
|
||||
}: {
|
||||
request: HttpAuthRequest
|
||||
onSubmit: (username: string, password: string) => void
|
||||
onCancel: () => void
|
||||
}) {
|
||||
const [username, setUsername] = useState('')
|
||||
const [password, setPassword] = useState('')
|
||||
|
||||
// Basic auth allows an empty username (token-style `curl -u :TOKEN`), so the
|
||||
// only invalid submission is fully empty. The server decides the rest.
|
||||
const canSubmit = username.length > 0 || password.length > 0
|
||||
|
||||
const handleSubmit = (e: React.FormEvent) => {
|
||||
e.preventDefault()
|
||||
if (!canSubmit) return
|
||||
onSubmit(username, password)
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open onOpenChange={(open) => { if (!open) onCancel() }}>
|
||||
<DialogContent className="w-[min(24rem,calc(100%-2rem))] max-w-sm">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Sign in</DialogTitle>
|
||||
<DialogDescription>
|
||||
{request.isProxy
|
||||
? `The proxy ${request.host} requires a username and password.`
|
||||
: `${request.host} requires a username and password.`}
|
||||
{request.realm ? ` (${request.realm})` : ''}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<form onSubmit={handleSubmit} className="flex flex-col gap-3">
|
||||
<Input
|
||||
autoFocus
|
||||
value={username}
|
||||
onChange={(e) => setUsername(e.target.value)}
|
||||
placeholder="Username"
|
||||
autoCapitalize="off"
|
||||
autoCorrect="off"
|
||||
spellCheck={false}
|
||||
/>
|
||||
<Input
|
||||
type="password"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
placeholder="Password"
|
||||
/>
|
||||
<DialogFooter>
|
||||
<Button type="button" variant="outline" onClick={onCancel}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button type="submit" disabled={!canSubmit}>
|
||||
Sign in
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</form>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
|
||||
export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps) {
|
||||
const [state, setState] = useState<BrowserState>(EMPTY_STATE)
|
||||
const [addressValue, setAddressValue] = useState('')
|
||||
const [authQueue, setAuthQueue] = useState<HttpAuthRequest[]>([])
|
||||
|
||||
const activeTabIdRef = useRef<string | null>(null)
|
||||
const addressFocusedRef = useRef(false)
|
||||
|
|
@ -121,6 +204,51 @@ export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps)
|
|||
return cleanup
|
||||
}, [applyState])
|
||||
|
||||
// Mirror of authQueue for the unmount handler, which must read the latest
|
||||
// queue without re-subscribing on every change.
|
||||
const authQueueRef = useRef<HttpAuthRequest[]>([])
|
||||
useEffect(() => {
|
||||
authQueueRef.current = authQueue
|
||||
}, [authQueue])
|
||||
|
||||
useEffect(() => {
|
||||
const offRequest = window.ipc.on('browser:httpAuthRequest', (incoming) => {
|
||||
setAuthQueue((queue) => [...queue, incoming as HttpAuthRequest])
|
||||
})
|
||||
// Main resolved a challenge on its own (timeout, or its tab/window was
|
||||
// destroyed) — drop the corresponding dialog so it can't linger over an
|
||||
// unrelated page with a submit that would no-op.
|
||||
const offResolved = window.ipc.on('browser:httpAuthResolved', (incoming) => {
|
||||
const { requestId } = incoming as { requestId: string }
|
||||
setAuthQueue((queue) => queue.filter((request) => request.requestId !== requestId))
|
||||
})
|
||||
return () => {
|
||||
offRequest()
|
||||
offResolved()
|
||||
// Cancel anything still pending so the main-process login callbacks and
|
||||
// timers are freed immediately instead of waiting out the timeout.
|
||||
for (const request of authQueueRef.current) {
|
||||
void window.ipc.invoke('browser:httpAuthResponse', { requestId: request.requestId })
|
||||
}
|
||||
}
|
||||
}, [])
|
||||
|
||||
const respondToAuth = useCallback(
|
||||
(requestId: string, credentials: { username: string; password: string } | null) => {
|
||||
setAuthQueue((queue) => queue.filter((request) => request.requestId !== requestId))
|
||||
// Omit username to cancel; include it (even empty) to submit.
|
||||
void window.ipc.invoke(
|
||||
'browser:httpAuthResponse',
|
||||
credentials
|
||||
? { requestId, username: credentials.username, password: credentials.password }
|
||||
: { requestId },
|
||||
)
|
||||
},
|
||||
[],
|
||||
)
|
||||
|
||||
const activeAuthRequest = authQueue[0] ?? null
|
||||
|
||||
const setViewVisible = useCallback((visible: boolean) => {
|
||||
if (viewVisibleRef.current === visible) return
|
||||
viewVisibleRef.current = visible
|
||||
|
|
@ -420,6 +548,17 @@ export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps)
|
|||
className="relative min-h-0 min-w-0 flex-1"
|
||||
data-browser-viewport
|
||||
/>
|
||||
|
||||
{activeAuthRequest && (
|
||||
<BrowserHttpAuthDialog
|
||||
key={activeAuthRequest.requestId}
|
||||
request={activeAuthRequest}
|
||||
onSubmit={(username, password) =>
|
||||
respondToAuth(activeAuthRequest.requestId, { username, password })
|
||||
}
|
||||
onCancel={() => respondToAuth(activeAuthRequest.requestId, null)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
|
||||
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'
|
||||
import {
|
||||
ArrowUp,
|
||||
|
|
@ -15,16 +15,19 @@ import {
|
|||
FolderCog,
|
||||
FolderOpen,
|
||||
Globe,
|
||||
Headphones,
|
||||
ImagePlus,
|
||||
LoaderIcon,
|
||||
Lock,
|
||||
Mic,
|
||||
MoreHorizontal,
|
||||
Phone,
|
||||
PhoneOff,
|
||||
Plus,
|
||||
Presentation,
|
||||
ShieldCheck,
|
||||
Square,
|
||||
Terminal,
|
||||
Video,
|
||||
X,
|
||||
} from 'lucide-react'
|
||||
|
||||
|
|
@ -210,6 +213,18 @@ function compactWorkDirPath(path: string) {
|
|||
return path.replace(/^\/Users\/[^/]+/, '~')
|
||||
}
|
||||
|
||||
// Call presets: front doors into the same call engine, differing only in
|
||||
// starting devices. 'share' is the call button's main click — the "work
|
||||
// together" default (screen shared, camera off, floating pill). The chevron
|
||||
// menu holds the deviations.
|
||||
export type CallPreset = 'voice' | 'video' | 'share' | 'practice'
|
||||
|
||||
const CALL_PRESET_MENU: Array<{ preset: CallPreset; label: string; description: string; Icon: typeof Phone }> = [
|
||||
{ preset: 'voice', label: 'Voice call', description: 'Just talk — nothing is shared, the mascot hovers while you work', Icon: AudioLines },
|
||||
{ preset: 'video', label: 'Video call', description: 'Camera on, face to face — it sees your expressions', Icon: Video },
|
||||
{ preset: 'practice', label: 'Practice session', description: 'Rehearse a pitch or interview with live coaching', Icon: Presentation },
|
||||
]
|
||||
|
||||
interface ChatInputInnerProps {
|
||||
onSubmit: (message: PromptInputMessage, mentions?: FileMention[], attachments?: StagedAttachment[], searchEnabled?: boolean, codeMode?: 'claude' | 'codex', permissionMode?: PermissionMode) => void
|
||||
onStop?: () => void
|
||||
|
|
@ -230,11 +245,13 @@ interface ChatInputInnerProps {
|
|||
onSubmitRecording?: () => void | Promise<void>
|
||||
onCancelRecording?: () => void
|
||||
voiceAvailable?: boolean
|
||||
ttsAvailable?: boolean
|
||||
ttsEnabled?: boolean
|
||||
ttsMode?: 'summary' | 'full'
|
||||
onToggleTts?: () => void
|
||||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
/** A call is live (hands-free voice loop + spoken responses). */
|
||||
inCall?: boolean
|
||||
/** Start a call with the given preset's device defaults. */
|
||||
onStartCall?: (preset: CallPreset) => void
|
||||
onEndCall?: () => void
|
||||
/** Calls need both voice input (STT) and voice output (TTS) configured. */
|
||||
callAvailable?: boolean
|
||||
/** Fired when the user picks a different model in the dropdown (only when no run exists yet). */
|
||||
onSelectedModelChange?: (model: SelectedModel | null) => void
|
||||
/** Work directory for this chat (per-chat). Null when none is set. */
|
||||
|
|
@ -268,11 +285,10 @@ function ChatInputInner({
|
|||
onSubmitRecording,
|
||||
onCancelRecording,
|
||||
voiceAvailable,
|
||||
ttsAvailable,
|
||||
ttsEnabled,
|
||||
ttsMode,
|
||||
onToggleTts,
|
||||
onTtsModeChange,
|
||||
inCall,
|
||||
onStartCall,
|
||||
onEndCall,
|
||||
callAvailable,
|
||||
onSelectedModelChange,
|
||||
workDir = null,
|
||||
onWorkDirChange,
|
||||
|
|
@ -287,6 +303,11 @@ function ChatInputInner({
|
|||
|
||||
const [configuredModels, setConfiguredModels] = useState<ConfiguredModel[]>([])
|
||||
const [activeModelKey, setActiveModelKey] = useState('')
|
||||
// The effective runtime default (what a run actually uses when the user
|
||||
// hasn't picked a model) — shown in the picker instead of guessing from
|
||||
// list order, which can disagree with the real default.
|
||||
const [defaultModel, setDefaultModel] = useState<ConfiguredModel | null>(null)
|
||||
const loadModelConfigEpoch = useRef(0)
|
||||
const [lockedModel, setLockedModel] = useState<SelectedModel | null>(null)
|
||||
const [searchEnabled, setSearchEnabled] = useState(false)
|
||||
const [searchAvailable, setSearchAvailable] = useState(false)
|
||||
|
|
@ -379,47 +400,54 @@ function ChatInputInner({
|
|||
return cleanup
|
||||
}, [])
|
||||
|
||||
// Load the list of models the user can choose from.
|
||||
// Signed-in: gateway model list. Signed-out: providers configured in models.json.
|
||||
// Load the list of models the user can choose from. Hybrid mode: signed-in
|
||||
// users get the gateway list AND every BYOK provider configured in
|
||||
// models.json (selecting a BYOK model routes that message through the
|
||||
// user's own key / local server). Signed-out users get BYOK only.
|
||||
const loadModelConfig = useCallback(async () => {
|
||||
// Concurrent runs race (mount fires one before the sign-in state resolves,
|
||||
// which fires another) — only the newest run may write state, else a slow
|
||||
// stale run can clobber the fresh list with an empty one.
|
||||
const epoch = ++loadModelConfigEpoch.current
|
||||
try {
|
||||
if (isRowboatConnected) {
|
||||
const def = await window.ipc.invoke('llm:getDefaultModel', null)
|
||||
if (loadModelConfigEpoch.current !== epoch) return
|
||||
setDefaultModel({ provider: def.provider as ProviderName, model: def.model })
|
||||
} catch {
|
||||
if (loadModelConfigEpoch.current === epoch) setDefaultModel(null)
|
||||
}
|
||||
try {
|
||||
const models: ConfiguredModel[] = []
|
||||
const seen = new Set<string>()
|
||||
const push = (provider: string, model: string) => {
|
||||
if (!model) return
|
||||
const key = `${provider}/${model}`
|
||||
if (seen.has(key)) return
|
||||
seen.add(key)
|
||||
models.push({ provider: provider as ProviderName, model })
|
||||
}
|
||||
|
||||
// Full catalog per provider (gateway + cloud). Providers with no
|
||||
// catalog (Ollama, OpenAI-compatible) fall back to the models saved in
|
||||
// config below.
|
||||
const catalog: Record<string, string[]> = {}
|
||||
try {
|
||||
const listResult = await window.ipc.invoke('models:list', null)
|
||||
const rowboatProvider = listResult.providers?.find(
|
||||
(p: { id: string }) => p.id === 'rowboat'
|
||||
)
|
||||
const models: ConfiguredModel[] = (rowboatProvider?.models || []).map(
|
||||
(m: { id: string }) => ({ provider: 'rowboat', model: m.id })
|
||||
)
|
||||
setConfiguredModels(models)
|
||||
} else {
|
||||
for (const p of listResult.providers || []) {
|
||||
catalog[p.id] = (p.models || []).map((m: { id: string }) => m.id)
|
||||
}
|
||||
} catch { /* offline / no catalog — fall back to saved config below */ }
|
||||
|
||||
if (isRowboatConnected) {
|
||||
for (const m of catalog['rowboat'] || []) push('rowboat', m)
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await window.ipc.invoke('workspace:readFile', { path: 'config/models.json' })
|
||||
const parsed = JSON.parse(result.data)
|
||||
|
||||
// Offer every model the configured key supports — the same full catalog
|
||||
// Settings uses for its dropdowns — so BYOK chat matches the signed-in
|
||||
// gateway picker. The picker is no longer limited to a hand-curated
|
||||
// config.models list. Providers with no catalog (Ollama, OpenAI-compatible)
|
||||
// fall back to the model saved in config.
|
||||
const catalog: Record<string, string[]> = {}
|
||||
try {
|
||||
const listResult = await window.ipc.invoke('models:list', null)
|
||||
for (const p of listResult.providers || []) {
|
||||
catalog[p.id] = (p.models || []).map((m: { id: string }) => m.id)
|
||||
}
|
||||
} catch { /* offline / no catalog — fall back to saved config below */ }
|
||||
|
||||
const models: ConfiguredModel[] = []
|
||||
const seen = new Set<string>()
|
||||
const push = (provider: string, model: string) => {
|
||||
if (!model) return
|
||||
const key = `${provider}/${model}`
|
||||
if (seen.has(key)) return
|
||||
seen.add(key)
|
||||
models.push({ provider: provider as ProviderName, model })
|
||||
}
|
||||
|
||||
// List the default provider first so its default model leads the picker.
|
||||
// List the default provider first so its default model leads the
|
||||
// BYOK section of the picker.
|
||||
const defaultFlavor = typeof parsed?.provider?.flavor === 'string' ? parsed.provider.flavor : ''
|
||||
const flavors = Object.keys(parsed?.providers || {})
|
||||
.sort((a, b) => (a === defaultFlavor ? -1 : b === defaultFlavor ? 1 : 0))
|
||||
|
|
@ -441,10 +469,24 @@ function ChatInputInner({
|
|||
for (const m of saved) push(flavor, m)
|
||||
}
|
||||
}
|
||||
setConfiguredModels(models)
|
||||
}
|
||||
} catch {
|
||||
// No config yet
|
||||
|
||||
// The user's explicit default selection leads the whole picker.
|
||||
const sel = parsed?.defaultSelection
|
||||
if (sel && typeof sel.provider === 'string' && typeof sel.model === 'string') {
|
||||
const selKey = `${sel.provider}/${sel.model}`
|
||||
const index = models.findIndex((m) => `${m.provider}/${m.model}` === selKey)
|
||||
if (index > 0) {
|
||||
const [entry] = models.splice(index, 1)
|
||||
models.unshift(entry)
|
||||
}
|
||||
}
|
||||
} catch { /* no BYOK config yet */ }
|
||||
|
||||
if (loadModelConfigEpoch.current !== epoch) return
|
||||
setConfiguredModels(models)
|
||||
} catch (err) {
|
||||
// No config yet — but surface unexpected failures for diagnosis.
|
||||
console.error('[chat-input] failed to load model list', err)
|
||||
}
|
||||
}, [isRowboatConnected])
|
||||
|
||||
|
|
@ -631,15 +673,25 @@ function ChatInputInner({
|
|||
checkSearch()
|
||||
}, [isActive, isRowboatConnected])
|
||||
|
||||
// The dropdown's items: always include the effective default so the picker
|
||||
// is never empty (and never missing the model that actually runs) even
|
||||
// while the full list is still loading.
|
||||
const pickerModels = useMemo<ConfiguredModel[]>(() => {
|
||||
if (!defaultModel) return configuredModels
|
||||
const defaultKey = `${defaultModel.provider}/${defaultModel.model}`
|
||||
if (configuredModels.some((m) => `${m.provider}/${m.model}` === defaultKey)) return configuredModels
|
||||
return [defaultModel, ...configuredModels]
|
||||
}, [configuredModels, defaultModel])
|
||||
|
||||
// Selecting a model affects only the *next* run created from this tab.
|
||||
// Once a run exists, model is frozen on the run and the dropdown is read-only.
|
||||
const handleModelChange = useCallback((key: string) => {
|
||||
if (lockedModel) return
|
||||
const entry = configuredModels.find((m) => `${m.provider}/${m.model}` === key)
|
||||
const entry = pickerModels.find((m) => `${m.provider}/${m.model}` === key)
|
||||
if (!entry) return
|
||||
setActiveModelKey(key)
|
||||
onSelectedModelChange?.({ provider: entry.provider, model: entry.model })
|
||||
}, [configuredModels, lockedModel, onSelectedModelChange])
|
||||
}, [pickerModels, lockedModel, onSelectedModelChange])
|
||||
|
||||
// Restore the tab draft when this input mounts.
|
||||
useEffect(() => {
|
||||
|
|
@ -752,7 +804,7 @@ function ChatInputInner({
|
|||
const currentWorkDirPath = effectiveWorkDir ? compactWorkDirPath(effectiveWorkDir) : ''
|
||||
|
||||
return (
|
||||
<div className="rowboat-chat-input rounded-lg border border-border bg-background shadow-none">
|
||||
<div data-tour-id="chat-composer" className="rowboat-chat-input rounded-lg border border-border bg-background shadow-none">
|
||||
{attachments.length > 0 && (
|
||||
<div className="flex flex-wrap gap-2 px-4 pb-1 pt-3">
|
||||
{attachments.map((attachment) => {
|
||||
|
|
@ -1243,7 +1295,7 @@ function ChatInputInner({
|
|||
{providerDisplayNames[lockedModel.provider] || lockedModel.provider} — fixed for this chat
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
) : configuredModels.length > 0 ? (
|
||||
) : pickerModels.length > 0 ? (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<button
|
||||
|
|
@ -1251,14 +1303,22 @@ function ChatInputInner({
|
|||
className="flex h-7 min-w-0 items-center gap-1 rounded-full px-2 text-xs text-muted-foreground transition-colors hover:bg-muted hover:text-foreground"
|
||||
>
|
||||
<span className="min-w-0 truncate">
|
||||
{getSelectedModelDisplayName(configuredModels.find((m) => `${m.provider}/${m.model}` === activeModelKey)?.model || configuredModels[0]?.model || 'Model')}
|
||||
{getSelectedModelDisplayName(
|
||||
pickerModels.find((m) => `${m.provider}/${m.model}` === activeModelKey)?.model
|
||||
|| defaultModel?.model
|
||||
|| pickerModels[0]?.model
|
||||
|| 'Model'
|
||||
)}
|
||||
</span>
|
||||
<ChevronDown className="h-3 w-3 shrink-0" />
|
||||
</button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuRadioGroup value={activeModelKey} onValueChange={handleModelChange}>
|
||||
{configuredModels.map((m) => {
|
||||
<DropdownMenuRadioGroup
|
||||
value={activeModelKey || (defaultModel ? `${defaultModel.provider}/${defaultModel.model}` : '')}
|
||||
onValueChange={handleModelChange}
|
||||
>
|
||||
{pickerModels.map((m) => {
|
||||
const key = `${m.provider}/${m.model}`
|
||||
return (
|
||||
<DropdownMenuRadioItem key={key} value={key}>
|
||||
|
|
@ -1271,48 +1331,66 @@ function ChatInputInner({
|
|||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
) : null}
|
||||
{onToggleTts && ttsAvailable && (
|
||||
{onStartCall && (
|
||||
<div className="flex shrink-0 items-center">
|
||||
<Tooltip delayDuration={CHAT_INPUT_TOOLTIP_DELAY_MS}>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleTts}
|
||||
onClick={() => {
|
||||
if (inCall) {
|
||||
onEndCall?.()
|
||||
} else if (callAvailable) {
|
||||
onStartCall('share')
|
||||
}
|
||||
}}
|
||||
className={cn(
|
||||
'relative flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
|
||||
ttsEnabled
|
||||
? 'text-foreground hover:bg-muted'
|
||||
: 'text-muted-foreground hover:bg-muted hover:text-foreground'
|
||||
'flex h-7 w-7 shrink-0 items-center justify-center rounded-full transition-colors',
|
||||
inCall
|
||||
? 'bg-red-600 text-white hover:bg-red-500'
|
||||
: callAvailable
|
||||
? 'text-muted-foreground hover:bg-muted hover:text-foreground'
|
||||
: 'cursor-default text-muted-foreground/40'
|
||||
)}
|
||||
aria-label={ttsEnabled ? 'Disable voice output' : 'Enable voice output'}
|
||||
aria-label={inCall ? 'End call' : 'Start a call'}
|
||||
>
|
||||
<Headphones className="h-4 w-4" />
|
||||
{!ttsEnabled && (
|
||||
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
|
||||
<span className="block h-[1.5px] w-5 -rotate-45 rounded-full bg-muted-foreground" />
|
||||
</span>
|
||||
)}
|
||||
{inCall ? <PhoneOff className="h-4 w-4" /> : <Phone className="h-4 w-4" />}
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top">
|
||||
{ttsEnabled ? 'Voice output on' : 'Voice output off'}
|
||||
{inCall
|
||||
? 'End call'
|
||||
: callAvailable
|
||||
? 'Start a call — it sees your screen while you talk it through'
|
||||
: 'Calls need voice input and output configured'}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
{ttsEnabled && onTtsModeChange && (
|
||||
{!inCall && (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
className="flex h-7 w-4 shrink-0 items-center justify-center text-muted-foreground transition-colors hover:text-foreground"
|
||||
aria-label="Call options"
|
||||
>
|
||||
<ChevronDown className="h-3 w-3" />
|
||||
</button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuRadioGroup value={ttsMode ?? 'summary'} onValueChange={(v) => onTtsModeChange(v as 'summary' | 'full')}>
|
||||
<DropdownMenuRadioItem value="summary">Speak summary</DropdownMenuRadioItem>
|
||||
<DropdownMenuRadioItem value="full">Speak full response</DropdownMenuRadioItem>
|
||||
</DropdownMenuRadioGroup>
|
||||
<DropdownMenuContent align="end" className="w-72">
|
||||
{CALL_PRESET_MENU.map(({ preset, label, description, Icon }) => (
|
||||
<DropdownMenuItem
|
||||
key={preset}
|
||||
disabled={!callAvailable}
|
||||
onSelect={() => onStartCall(preset)}
|
||||
className="items-start gap-3 py-2"
|
||||
>
|
||||
<Icon className="mt-0.5 h-4 w-4 shrink-0 text-muted-foreground" />
|
||||
<span className="min-w-0">
|
||||
<span className="block text-sm font-medium leading-tight">{label}</span>
|
||||
<span className="block pt-0.5 text-xs leading-tight text-muted-foreground">{description}</span>
|
||||
</span>
|
||||
</DropdownMenuItem>
|
||||
))}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
)}
|
||||
|
|
@ -1487,11 +1565,10 @@ export interface ChatInputWithMentionsProps {
|
|||
onSubmitRecording?: () => void | Promise<void>
|
||||
onCancelRecording?: () => void
|
||||
voiceAvailable?: boolean
|
||||
ttsAvailable?: boolean
|
||||
ttsEnabled?: boolean
|
||||
ttsMode?: 'summary' | 'full'
|
||||
onToggleTts?: () => void
|
||||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
inCall?: boolean
|
||||
onStartCall?: (preset: CallPreset) => void
|
||||
onEndCall?: () => void
|
||||
callAvailable?: boolean
|
||||
onSelectedModelChange?: (model: SelectedModel | null) => void
|
||||
workDir?: string | null
|
||||
onWorkDirChange?: (value: string | null) => void
|
||||
|
|
@ -1521,11 +1598,10 @@ export function ChatInputWithMentions({
|
|||
onSubmitRecording,
|
||||
onCancelRecording,
|
||||
voiceAvailable,
|
||||
ttsAvailable,
|
||||
ttsEnabled,
|
||||
ttsMode,
|
||||
onToggleTts,
|
||||
onTtsModeChange,
|
||||
inCall,
|
||||
onStartCall,
|
||||
onEndCall,
|
||||
callAvailable,
|
||||
onSelectedModelChange,
|
||||
workDir,
|
||||
onWorkDirChange,
|
||||
|
|
@ -1552,11 +1628,10 @@ export function ChatInputWithMentions({
|
|||
onSubmitRecording={onSubmitRecording}
|
||||
onCancelRecording={onCancelRecording}
|
||||
voiceAvailable={voiceAvailable}
|
||||
ttsAvailable={ttsAvailable}
|
||||
ttsEnabled={ttsEnabled}
|
||||
ttsMode={ttsMode}
|
||||
onToggleTts={onToggleTts}
|
||||
onTtsModeChange={onTtsModeChange}
|
||||
inCall={inCall}
|
||||
onStartCall={onStartCall}
|
||||
onEndCall={onEndCall}
|
||||
callAvailable={callAvailable}
|
||||
onSelectedModelChange={onSelectedModelChange}
|
||||
workDir={workDir}
|
||||
onWorkDirChange={onWorkDirChange}
|
||||
|
|
|
|||
|
|
@ -91,11 +91,28 @@ interface ChatMessageAttachmentsProps {
|
|||
export function ChatMessageAttachments({ attachments, className }: ChatMessageAttachmentsProps) {
|
||||
if (attachments.length === 0) return null
|
||||
|
||||
const imageAttachments = attachments.filter((attachment) => isImageMime(attachment.mimeType))
|
||||
const fileAttachments = attachments.filter((attachment) => !isImageMime(attachment.mimeType))
|
||||
const videoFrames = attachments.filter((attachment) => attachment.isVideoFrame)
|
||||
const imageAttachments = attachments.filter(
|
||||
(attachment) => !attachment.isVideoFrame && isImageMime(attachment.mimeType)
|
||||
)
|
||||
const fileAttachments = attachments.filter(
|
||||
(attachment) => !attachment.isVideoFrame && !isImageMime(attachment.mimeType)
|
||||
)
|
||||
|
||||
return (
|
||||
<div className={cn('flex flex-col items-end gap-2', className)}>
|
||||
{videoFrames.length > 0 && (
|
||||
<div className="flex max-w-[340px] flex-wrap justify-end gap-1">
|
||||
{videoFrames.map((frame, index) => (
|
||||
<img
|
||||
key={`frame-${index}`}
|
||||
src={frame.thumbnailUrl}
|
||||
alt={`Camera frame ${index + 1}`}
|
||||
className="h-12 w-auto rounded-md border border-border/60 bg-muted object-cover"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{imageAttachments.length > 0 && (
|
||||
<div className="flex flex-wrap justify-end gap-2">
|
||||
{imageAttachments.map((attachment, index) => (
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ import { MarkdownPreOverride } from '@/components/ai-elements/markdown-code-over
|
|||
import { defaultRemarkPlugins } from 'streamdown'
|
||||
import remarkBreaks from 'remark-breaks'
|
||||
import { type ChatTab } from '@/components/tab-bar'
|
||||
import { ChatInputWithMentions, type PermissionMode, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions'
|
||||
import { ChatInputWithMentions, type CallPreset, type PermissionMode, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions'
|
||||
import { ChatMessageAttachments } from '@/components/chat-message-attachments'
|
||||
import { useSidebar } from '@/components/ui/sidebar'
|
||||
import { wikiLabel } from '@/lib/wiki-links'
|
||||
|
|
@ -51,6 +51,7 @@ import {
|
|||
getWebSearchCardData,
|
||||
getComposioConnectCardData,
|
||||
getToolDisplayName,
|
||||
getToolErrorText,
|
||||
groupConversationItems,
|
||||
isChatMessage,
|
||||
isErrorMessage,
|
||||
|
|
@ -187,11 +188,10 @@ interface ChatSidebarProps {
|
|||
onSubmitRecording?: () => void | Promise<void>
|
||||
onCancelRecording?: () => void
|
||||
voiceAvailable?: boolean
|
||||
ttsAvailable?: boolean
|
||||
ttsEnabled?: boolean
|
||||
ttsMode?: 'summary' | 'full'
|
||||
onToggleTts?: () => void
|
||||
onTtsModeChange?: (mode: 'summary' | 'full') => void
|
||||
inCall?: boolean
|
||||
onStartCall?: (preset: CallPreset) => void
|
||||
onEndCall?: () => void
|
||||
callAvailable?: boolean
|
||||
onComposioConnected?: (toolkitSlug: string) => void
|
||||
}
|
||||
|
||||
|
|
@ -251,11 +251,10 @@ export function ChatSidebar({
|
|||
onSubmitRecording,
|
||||
onCancelRecording,
|
||||
voiceAvailable,
|
||||
ttsAvailable,
|
||||
ttsEnabled,
|
||||
ttsMode,
|
||||
onToggleTts,
|
||||
onTtsModeChange,
|
||||
inCall,
|
||||
onStartCall,
|
||||
onEndCall,
|
||||
callAvailable,
|
||||
onComposioConnected,
|
||||
}: ChatSidebarProps) {
|
||||
const { state: sidebarState } = useSidebar()
|
||||
|
|
@ -486,7 +485,7 @@ export function ChatSidebar({
|
|||
)
|
||||
}
|
||||
const toolTitle = getToolDisplayName(item)
|
||||
const errorText = item.status === 'error' ? 'Tool error' : ''
|
||||
const errorText = getToolErrorText(item)
|
||||
const output = normalizeToolOutput(item.result, item.status)
|
||||
const input = normalizeToolInput(item.input)
|
||||
return (
|
||||
|
|
@ -825,11 +824,10 @@ export function ChatSidebar({
|
|||
onSubmitRecording={isActive ? onSubmitRecording : undefined}
|
||||
onCancelRecording={isActive ? onCancelRecording : undefined}
|
||||
voiceAvailable={isActive && voiceAvailable}
|
||||
ttsAvailable={isActive && ttsAvailable}
|
||||
ttsEnabled={ttsEnabled}
|
||||
ttsMode={ttsMode}
|
||||
onToggleTts={isActive ? onToggleTts : undefined}
|
||||
onTtsModeChange={isActive ? onTtsModeChange : undefined}
|
||||
inCall={inCall}
|
||||
onStartCall={isActive ? onStartCall : undefined}
|
||||
onEndCall={isActive ? onEndCall : undefined}
|
||||
callAvailable={callAvailable}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { Conversation, ConversationContent, ConversationScrollButton } from '@/c
|
|||
import { MessageResponse } from '@/components/ai-elements/message'
|
||||
import { Shimmer } from '@/components/ai-elements/shimmer'
|
||||
import { Tool, ToolContent, ToolHeader } from '@/components/ai-elements/tool'
|
||||
import { toToolState, getToolDisplayName, getWebSearchCardData, type ToolCall } from '@/lib/chat-conversation'
|
||||
import { toToolState, getToolDisplayName, getToolErrorText, getWebSearchCardData, type ToolCall } from '@/lib/chat-conversation'
|
||||
import { CodeRunPermissionRequest, CodingRunTimeline } from '@/components/coding-run'
|
||||
import { PermissionRequest } from '@/components/ai-elements/permission-request'
|
||||
import { AskHumanRequest } from '@/components/ai-elements/ask-human-request'
|
||||
|
|
@ -84,7 +84,7 @@ function RowboatToolCall({ item, onOpenDiff }: { item: ToolCall; onOpenDiff: (pa
|
|||
<Tool open={open || item.status === 'running'} onOpenChange={setOpen}>
|
||||
<ToolHeader title={AGENT_LABEL[agent ?? ''] ?? 'Coding agent'} type="tool-code_agent_run" state={toToolState(item.status)} />
|
||||
<ToolContent>
|
||||
<CodingRunTimeline events={item.codeRunEvents ?? []} onOpenDiff={onOpenDiff} />
|
||||
<CodingRunTimeline events={item.codeRunEvents ?? []} error={getToolErrorText(item)} onOpenDiff={onOpenDiff} />
|
||||
</ToolContent>
|
||||
</Tool>
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { useMemo, useState } from 'react'
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import {
|
||||
AlertCircle,
|
||||
CheckCircle2,
|
||||
Circle,
|
||||
CircleDot,
|
||||
|
|
@ -17,7 +18,8 @@ import {
|
|||
import type { CodeRunEvent, PermissionAsk, PermissionDecision } from '@x/shared/src/code-mode.js'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { Tool, ToolContent, ToolHeader } from '@/components/ai-elements/tool'
|
||||
import { toToolState, type ToolCall } from '@/lib/chat-conversation'
|
||||
import { getToolErrorText, toToolState, type ToolCall } from '@/lib/chat-conversation'
|
||||
import { clearCodeRunBuffer, useCodeRunFeed } from '@/lib/code-run-feed'
|
||||
|
||||
// ── Timeline reduction ──────────────────────────────────────────────
|
||||
// The raw ACP stream is a flat list of events; collapse it into ordered rows,
|
||||
|
|
@ -111,14 +113,26 @@ const basename = (p: string) => p.split(/[\\/]/).pop() || p
|
|||
|
||||
export function CodingRunTimeline({
|
||||
events,
|
||||
error,
|
||||
onOpenDiff,
|
||||
}: {
|
||||
events: CodeRunEvent[]
|
||||
error?: string
|
||||
// When set, changed-file names become clickable (the Code section opens the diff).
|
||||
onOpenDiff?: (path: string) => void
|
||||
}) {
|
||||
const rows = useMemo(() => reduceEvents(events), [events])
|
||||
if (rows.length === 0) {
|
||||
if (error) {
|
||||
return (
|
||||
<div className="px-4 py-3">
|
||||
<div className="flex items-start gap-2 rounded-md border border-destructive/30 bg-destructive/5 px-3 py-2 text-sm text-destructive">
|
||||
<AlertCircle className="mt-0.5 size-3.5 shrink-0" />
|
||||
<span className="min-w-0 whitespace-pre-wrap break-words">{error}</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
return <div className="px-4 py-3 text-xs text-muted-foreground">Starting the agent…</div>
|
||||
}
|
||||
return (
|
||||
|
|
@ -133,12 +147,17 @@ export function CodingRunTimeline({
|
|||
}
|
||||
if (row.kind === 'tool') {
|
||||
const running = row.status !== 'completed' && row.status !== 'failed'
|
||||
const failed = row.status === 'failed'
|
||||
return (
|
||||
<div key={row.id} className="flex flex-col gap-1">
|
||||
<div className="flex items-center gap-2 text-sm">
|
||||
{running
|
||||
? <Loader className="size-3.5 shrink-0 animate-spin text-muted-foreground" />
|
||||
: <CheckCircle2 className="size-3.5 shrink-0 text-green-600" />}
|
||||
{running ? (
|
||||
<Loader className="size-3.5 shrink-0 animate-spin text-muted-foreground" />
|
||||
) : failed ? (
|
||||
<AlertCircle className="size-3.5 shrink-0 text-destructive" />
|
||||
) : (
|
||||
<CheckCircle2 className="size-3.5 shrink-0 text-green-600" />
|
||||
)}
|
||||
{toolKindIcon(row.toolKind, row.title)}
|
||||
<span className="truncate text-foreground/90">{row.title ?? row.toolKind ?? 'Tool call'}</span>
|
||||
</div>
|
||||
|
|
@ -189,6 +208,12 @@ export function CodingRunTimeline({
|
|||
</div>
|
||||
)
|
||||
})}
|
||||
{error && (
|
||||
<div className="flex items-start gap-2 rounded-md border border-destructive/30 bg-destructive/5 px-3 py-2 text-sm text-destructive">
|
||||
<AlertCircle className="mt-0.5 size-3.5 shrink-0" />
|
||||
<span className="min-w-0 whitespace-pre-wrap break-words">{error}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -258,12 +283,23 @@ export function CodingRunBlock({
|
|||
(item.result as { agent?: string } | undefined)?.agent ??
|
||||
(item.input as { agent?: string } | undefined)?.agent
|
||||
const title = AGENT_LABEL[agent ?? ''] ?? 'Coding agent'
|
||||
const error = getToolErrorText(item)
|
||||
// Timeline source: the durable record (item.codeRunEvents — the settle-time
|
||||
// batch, or the legacy path's inline accumulation) wins; while it's absent
|
||||
// the live CodeRunFeed buffer streams the run in real time.
|
||||
const liveEvents = useCodeRunFeed(item.id)
|
||||
const durableEvents = item.codeRunEvents
|
||||
const events = durableEvents?.length ? durableEvents : liveEvents
|
||||
// Once the durable batch has landed the buffer is redundant — drop it.
|
||||
useEffect(() => {
|
||||
if (durableEvents?.length) clearCodeRunBuffer(item.id)
|
||||
}, [durableEvents?.length, item.id])
|
||||
return (
|
||||
<>
|
||||
<Tool open={open} onOpenChange={onOpenChange}>
|
||||
<ToolHeader title={title} type="tool-code_agent_run" state={toToolState(item.status)} />
|
||||
<ToolContent>
|
||||
<CodingRunTimeline events={item.codeRunEvents ?? []} />
|
||||
<CodingRunTimeline events={events} error={error} />
|
||||
</ToolContent>
|
||||
</Tool>
|
||||
{item.pendingCodePermission && (
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
isErrorMessage,
|
||||
isToolCall,
|
||||
getToolDisplayName,
|
||||
getToolErrorText,
|
||||
toToolState,
|
||||
normalizeToolOutput,
|
||||
} from '@/lib/chat-conversation'
|
||||
|
|
@ -62,7 +63,7 @@ function CompactToolRow({ tool }: { tool: ToolCall }) {
|
|||
const [open, setOpen] = useState(false)
|
||||
const title = getToolDisplayName(tool)
|
||||
const state = toToolState(tool.status)
|
||||
const errorText = tool.status === 'error' && typeof tool.result === 'string' ? tool.result : undefined
|
||||
const errorText = getToolErrorText(tool)
|
||||
return (
|
||||
<Tool open={open} onOpenChange={setOpen} className="mb-0 text-xs">
|
||||
<ToolHeader title={title} type={`tool-${tool.name}` as `tool-${string}`} state={state} />
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -183,8 +183,8 @@ export function OnboardingModal({ open, onComplete }: OnboardingModalProps) {
|
|||
|
||||
// Preferred default models for each provider
|
||||
const preferredDefaults: Partial<Record<LlmProviderFlavor, string>> = {
|
||||
openai: "gpt-5.2",
|
||||
anthropic: "claude-opus-4-6-20260202",
|
||||
openai: "gpt-5.4",
|
||||
anthropic: "claude-opus-4-8",
|
||||
}
|
||||
|
||||
// Initialize default models from catalog
|
||||
|
|
|
|||
|
|
@ -155,8 +155,8 @@ export function useOnboardingState(open: boolean, onComplete: () => void) {
|
|||
|
||||
// Preferred default models for each provider
|
||||
const preferredDefaults: Partial<Record<LlmProviderFlavor, string>> = {
|
||||
openai: "gpt-5.2",
|
||||
anthropic: "claude-opus-4-6-20260202",
|
||||
openai: "gpt-5.4",
|
||||
anthropic: "claude-opus-4-8",
|
||||
}
|
||||
|
||||
// Initialize default models from catalog
|
||||
|
|
@ -434,10 +434,29 @@ export function useOnboardingState(open: boolean, onComplete: () => void) {
|
|||
}
|
||||
|
||||
const catalog: string[] = result.models ?? []
|
||||
const typed = activeConfig.model.trim()
|
||||
// Hosted providers hide the model field (it holds an auto-seeded
|
||||
// default), so only treat it as user intent where the field is shown —
|
||||
// mirrors showModelInput in llm-setup-step.
|
||||
const hostedProviders: LlmProviderFlavor[] = ["openai", "anthropic", "google"]
|
||||
const modelInputShown = !hostedProviders.includes(llmProvider)
|
||||
|
||||
if (modelInputShown && typed && llmProvider === "ollama" && catalog.length > 0 && !catalog.includes(typed)) {
|
||||
// Ollama's tag list is authoritative: an unlisted model isn't pulled,
|
||||
// so saving it would break chat at runtime with no obvious cause.
|
||||
const error = `Model '${typed}' is not available on this Ollama server. Pull it first (ollama pull ${typed}) or pick one of: ${catalog.slice(0, 5).join(", ")}${catalog.length > 5 ? ", …" : ""}`
|
||||
setTestState({ status: "error", error })
|
||||
toast.error(error)
|
||||
return false
|
||||
}
|
||||
|
||||
const preferred = preferredDefaults[llmProvider]
|
||||
const model =
|
||||
(preferred && catalog.includes(preferred) && preferred) ||
|
||||
catalog[0] || activeConfig.model.trim() || ""
|
||||
// A model the user explicitly entered always wins — this used to prefer
|
||||
// catalog[0], which silently replaced the user's Ollama model with
|
||||
// whatever model the local server happened to list first.
|
||||
const model = modelInputShown
|
||||
? (typed || catalog[0] || "")
|
||||
: ((preferred && catalog.includes(preferred) && preferred) || catalog[0] || typed || "")
|
||||
|
||||
// `models` is the user's curated assistant-model list (shown in Settings),
|
||||
// NOT the full provider catalog. Onboarding seeds it with just the selected
|
||||
|
|
|
|||
878
apps/x/apps/renderer/src/components/product-tour.tsx
Normal file
878
apps/x/apps/renderer/src/components/product-tour.tsx
Normal file
|
|
@ -0,0 +1,878 @@
|
|||
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
|
||||
import { createPortal } from 'react-dom'
|
||||
import { X } from 'lucide-react'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { TalkingHead, type MascotHat } from '@/components/talking-head'
|
||||
import { AgentsFleet, MascotVignette, type TourVignetteKind } from '@/components/tour-vignettes'
|
||||
import { TourSounds } from '@/lib/tour-sounds'
|
||||
import type { TTSState } from '@/hooks/useVoiceTTS'
|
||||
import { cn } from '@/lib/utils'
|
||||
import tourClipWelcome from '@/assets/tour/welcome.mp3'
|
||||
import tourClipHome from '@/assets/tour/home.mp3'
|
||||
import tourClipEmail from '@/assets/tour/email.mp3'
|
||||
import tourClipMeetings from '@/assets/tour/meetings.mp3'
|
||||
import tourClipCode from '@/assets/tour/code.mp3'
|
||||
import tourClipKnowledge from '@/assets/tour/knowledge.mp3'
|
||||
import tourClipAgents from '@/assets/tour/agents.mp3'
|
||||
import tourClipWorkspaces from '@/assets/tour/workspaces.mp3'
|
||||
import tourClipChats from '@/assets/tour/chats.mp3'
|
||||
import tourClipComposer from '@/assets/tour/composer.mp3'
|
||||
import tourClipDone from '@/assets/tour/done.mp3'
|
||||
|
||||
export type TourNavTarget =
|
||||
| 'home'
|
||||
| 'email'
|
||||
| 'meetings'
|
||||
| 'code'
|
||||
| 'knowledge'
|
||||
| 'agents'
|
||||
| 'workspaces'
|
||||
|
||||
type TourStep = {
|
||||
id: string
|
||||
/** Matches a [data-tour-id] element. Steps whose target is absent are skipped. */
|
||||
targetId?: string
|
||||
/** View to open when the step starts, via App's navigation handlers. */
|
||||
navigate?: TourNavTarget
|
||||
/** Costume the mascot wears at this stop. */
|
||||
hat?: MascotHat
|
||||
/** Looping animation staged around the mascot while it presents this stop. */
|
||||
vignette?: TourVignetteKind
|
||||
title: string
|
||||
text: string
|
||||
/** Spoken narration, when it should differ from the bubble text. */
|
||||
voiceText?: string
|
||||
}
|
||||
|
||||
const TOUR_STEPS: TourStep[] = [
|
||||
{
|
||||
id: 'welcome',
|
||||
title: 'All aboard! ⚓',
|
||||
text: "I'm your captain for the next minute. The lights are down and the water's in — let me row you across Rowboat, one stop at a time. Use Next or your arrow keys.",
|
||||
voiceText: "I'm your captain for the next minute. The lights are down and the water's in — let me row you across Rowboat, one stop at a time.",
|
||||
},
|
||||
{
|
||||
id: 'home',
|
||||
targetId: 'nav-home',
|
||||
navigate: 'home',
|
||||
title: 'First stop: Home',
|
||||
text: 'Home is your landing spot — a quick overview of what needs your attention to get you back into the flow.',
|
||||
},
|
||||
{
|
||||
id: 'email',
|
||||
targetId: 'nav-email',
|
||||
navigate: 'email',
|
||||
hat: 'mailcap',
|
||||
vignette: 'email',
|
||||
title: 'Email',
|
||||
text: 'Read and triage your inbox right here. Rowboat can summarize threads, label messages, and help you draft replies.',
|
||||
},
|
||||
{
|
||||
id: 'meetings',
|
||||
targetId: 'nav-meetings',
|
||||
navigate: 'meetings',
|
||||
hat: 'headphones',
|
||||
vignette: 'meetings',
|
||||
title: 'Meetings',
|
||||
text: 'Record or join meetings, and get transcripts and notes automatically — prep briefs show up before your calls, too.',
|
||||
},
|
||||
{
|
||||
id: 'code',
|
||||
targetId: 'nav-code',
|
||||
navigate: 'code',
|
||||
hat: 'hardhat',
|
||||
title: 'Code',
|
||||
text: 'The Code section runs coding agents on your projects — point one at a folder and drive it from a chat.',
|
||||
},
|
||||
{
|
||||
id: 'knowledge',
|
||||
targetId: 'nav-knowledge',
|
||||
navigate: 'knowledge',
|
||||
hat: 'gradcap',
|
||||
vignette: 'brain',
|
||||
title: 'Brain',
|
||||
text: "Brain is your knowledge base — notes, files, and everything Rowboat learns for you, all connected and searchable.",
|
||||
},
|
||||
{
|
||||
id: 'agents',
|
||||
targetId: 'nav-agents',
|
||||
navigate: 'agents',
|
||||
hat: 'captain',
|
||||
vignette: 'agents',
|
||||
title: 'Background agents',
|
||||
text: 'Background agents work on schedules — they keep your Brain fresh and take care of recurring tasks while you row elsewhere.',
|
||||
},
|
||||
{
|
||||
id: 'workspaces',
|
||||
targetId: 'nav-workspaces',
|
||||
navigate: 'workspaces',
|
||||
hat: 'explorer',
|
||||
title: 'Workspaces',
|
||||
text: 'Workspaces hold your project folders and files, so related work stays docked together.',
|
||||
},
|
||||
{
|
||||
id: 'chats',
|
||||
targetId: 'nav-chats',
|
||||
title: 'Chats',
|
||||
text: 'Your recent conversations live here — pick any of them back up right where you left off.',
|
||||
},
|
||||
{
|
||||
id: 'composer',
|
||||
targetId: 'chat-composer',
|
||||
title: 'Talk to Rowboat',
|
||||
text: 'And this is where we talk! Type, dictate with the mic, or turn on voice output — tap my face button and I’ll read replies out loud myself.',
|
||||
},
|
||||
{
|
||||
id: 'done',
|
||||
hat: 'party',
|
||||
title: "Land ho! 🎉",
|
||||
text: "That's the whole bay — and there's my wake to prove it. Take this voyage again anytime from the bottom of the sidebar. Happy rowing!",
|
||||
},
|
||||
]
|
||||
|
||||
// Pre-recorded narration bundled with the app (no TTS API call, works
|
||||
// offline/signed-out). Regenerate with scripts/generate-tour-audio.mjs after
|
||||
// editing any step's text. Steps without a clip fall back to live TTS.
|
||||
const TOUR_CLIPS: Record<string, string> = {
|
||||
welcome: tourClipWelcome,
|
||||
home: tourClipHome,
|
||||
email: tourClipEmail,
|
||||
meetings: tourClipMeetings,
|
||||
code: tourClipCode,
|
||||
knowledge: tourClipKnowledge,
|
||||
agents: tourClipAgents,
|
||||
workspaces: tourClipWorkspaces,
|
||||
chats: tourClipChats,
|
||||
composer: tourClipComposer,
|
||||
done: tourClipDone,
|
||||
}
|
||||
|
||||
const MASCOT_SIZE = 120
|
||||
const VIEWPORT_MARGIN = 16
|
||||
const BUBBLE_WIDTH = 288
|
||||
const TARGET_RESOLVE_TIMEOUT_MS = 1500
|
||||
const ZOOM_SCALE = 1.05
|
||||
const GLIDE_EASING = 'cubic-bezier(0.45, 0, 0.2, 1)'
|
||||
|
||||
type Pt = { x: number; y: number }
|
||||
type Rect = { left: number; top: number; width: number; height: number }
|
||||
type Spot = Rect & { round: boolean }
|
||||
|
||||
function clamp(v: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, v))
|
||||
}
|
||||
|
||||
function easeInOutCubic(t: number): number {
|
||||
return t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2
|
||||
}
|
||||
|
||||
function quadPoint(p0: Pt, c: Pt, p1: Pt, t: number): Pt {
|
||||
const u = 1 - t
|
||||
return {
|
||||
x: u * u * p0.x + 2 * u * t * c.x + t * t * p1.x,
|
||||
y: u * u * p0.y + 2 * u * t * c.y + t * t * p1.y,
|
||||
}
|
||||
}
|
||||
|
||||
function quadPathLength(d: string): number {
|
||||
const p = document.createElementNS('http://www.w3.org/2000/svg', 'path')
|
||||
p.setAttribute('d', d)
|
||||
return p.getTotalLength()
|
||||
}
|
||||
|
||||
// A data-tour-id can legitimately appear on several elements (e.g. the chat
|
||||
// composer renders in both the full-screen chat and the side pane) — pick the
|
||||
// one that is actually laid out.
|
||||
function findTourTarget(targetId: string): HTMLElement | null {
|
||||
const nodes = document.querySelectorAll<HTMLElement>(`[data-tour-id="${targetId}"]`)
|
||||
for (const el of nodes) {
|
||||
const rect = el.getBoundingClientRect()
|
||||
if (rect.width > 0 && rect.height > 0) return el
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function mascotDestForCenter(): Pt {
|
||||
return {
|
||||
x: window.innerWidth / 2 - MASCOT_SIZE / 2 - BUBBLE_WIDTH / 2,
|
||||
y: window.innerHeight / 2 - MASCOT_SIZE / 2,
|
||||
}
|
||||
}
|
||||
|
||||
function mascotDestForRect(rect: Rect): Pt {
|
||||
let x: number
|
||||
let y: number
|
||||
const fitsRight = rect.left + rect.width + MASCOT_SIZE + BUBBLE_WIDTH + VIEWPORT_MARGIN * 3 < window.innerWidth
|
||||
if (fitsRight) {
|
||||
x = rect.left + rect.width + 20
|
||||
y = rect.top + rect.height / 2 - MASCOT_SIZE / 2
|
||||
} else if (rect.top > MASCOT_SIZE + VIEWPORT_MARGIN * 2) {
|
||||
x = rect.left + rect.width / 2 - MASCOT_SIZE / 2
|
||||
y = rect.top - MASCOT_SIZE - 20
|
||||
} else {
|
||||
x = rect.left - MASCOT_SIZE - 20
|
||||
y = rect.top + rect.height / 2 - MASCOT_SIZE / 2
|
||||
}
|
||||
return {
|
||||
x: clamp(x, VIEWPORT_MARGIN, window.innerWidth - MASCOT_SIZE - VIEWPORT_MARGIN),
|
||||
y: clamp(y, VIEWPORT_MARGIN, window.innerHeight - MASCOT_SIZE - VIEWPORT_MARGIN),
|
||||
}
|
||||
}
|
||||
|
||||
function spotForRect(rect: Rect): Spot {
|
||||
return {
|
||||
left: rect.left - 6,
|
||||
top: rect.top - 6,
|
||||
width: rect.width + 12,
|
||||
height: rect.height + 12,
|
||||
round: false,
|
||||
}
|
||||
}
|
||||
|
||||
function spotForMascot(dest: Pt): Spot {
|
||||
return {
|
||||
left: dest.x - 26,
|
||||
top: dest.y - 18,
|
||||
width: MASCOT_SIZE + 52,
|
||||
height: MASCOT_SIZE + 44,
|
||||
round: true,
|
||||
}
|
||||
}
|
||||
|
||||
type ProductTourProps = {
|
||||
onClose: () => void
|
||||
onNavigate: (target: TourNavTarget) => void
|
||||
ttsAvailable: boolean
|
||||
ttsState: TTSState
|
||||
speak: (text: string) => void
|
||||
speakUrl: (url: string) => void
|
||||
cancelSpeech: () => void
|
||||
getLevel: () => number
|
||||
}
|
||||
|
||||
/**
|
||||
* The Grand Voyage: a mascot-guided walkthrough where the app dims to a
|
||||
* night-time bay, the boat rows curved routes between [data-tour-id] anchors
|
||||
* (leaving a dotted wake behind it), a spotlight and gentle camera zoom reveal
|
||||
* each section, and a parchment mini-map charts progress. Narrated with TTS
|
||||
* lip sync when available; ends in confetti.
|
||||
*
|
||||
* Rendered through a portal to <body> so the camera zoom applied to the app
|
||||
* shell never transforms the tour's own fixed-position layers.
|
||||
*/
|
||||
export function ProductTour({
|
||||
onClose,
|
||||
onNavigate,
|
||||
ttsAvailable,
|
||||
ttsState,
|
||||
speak,
|
||||
speakUrl,
|
||||
cancelSpeech,
|
||||
getLevel,
|
||||
}: ProductTourProps) {
|
||||
const [stepIndex, setStepIndex] = useState(0)
|
||||
const [arrived, setArrived] = useState(false)
|
||||
const [rowing, setRowing] = useState(false)
|
||||
const [flipped, setFlipped] = useState(false)
|
||||
const [bubbleSide, setBubbleSide] = useState<'left' | 'right'>('right')
|
||||
const [spot, setSpot] = useState<Spot | null>(null)
|
||||
const [wakes, setWakes] = useState<{ id: number; d: string }[]>([])
|
||||
const [activeWake, setActiveWake] = useState<{ d: string; len: number } | null>(null)
|
||||
const [confettiOn, setConfettiOn] = useState(false)
|
||||
const [resizeNonce, setResizeNonce] = useState(0)
|
||||
|
||||
const reducedMotion = useMemo(
|
||||
() => window.matchMedia('(prefers-reduced-motion: reduce)').matches,
|
||||
[]
|
||||
)
|
||||
|
||||
// Mascot position is animated by mutating the container's transform directly
|
||||
// (60fps travel without re-rendering React); posRef is the source of truth.
|
||||
const mascotElRef = useRef<HTMLDivElement>(null)
|
||||
const posRef = useRef<Pt>({
|
||||
x: window.innerWidth / 2 - MASCOT_SIZE / 2,
|
||||
y: window.innerHeight + 60,
|
||||
})
|
||||
const travelRafRef = useRef(0)
|
||||
const wakePathElRef = useRef<SVGPathElement>(null)
|
||||
const wakeIdRef = useRef(0)
|
||||
const curveSideRef = useRef(1)
|
||||
const lastSplashRef = useRef(0)
|
||||
|
||||
const directionRef = useRef(1)
|
||||
const enteredStepRef = useRef(-1)
|
||||
const stepIndexRef = useRef(stepIndex)
|
||||
|
||||
// Camera zoom state applied to the app shell (outside the portal)
|
||||
const shellRef = useRef<HTMLElement | null>(null)
|
||||
const zoomRef = useRef<{ ox: number; oy: number; s: number } | null>(null)
|
||||
|
||||
const soundsRef = useRef<TourSounds | null>(null)
|
||||
|
||||
const onCloseRef = useRef(onClose)
|
||||
const onNavigateRef = useRef(onNavigate)
|
||||
const speakRef = useRef(speak)
|
||||
const speakUrlRef = useRef(speakUrl)
|
||||
const cancelSpeechRef = useRef(cancelSpeech)
|
||||
const ttsAvailableRef = useRef(ttsAvailable)
|
||||
|
||||
// Keep latest callbacks/state in refs so the step effect and key handlers
|
||||
// stay stable. Runs before the step effect below (effect order = call order).
|
||||
useEffect(() => {
|
||||
stepIndexRef.current = stepIndex
|
||||
onCloseRef.current = onClose
|
||||
onNavigateRef.current = onNavigate
|
||||
speakRef.current = speak
|
||||
speakUrlRef.current = speakUrl
|
||||
cancelSpeechRef.current = cancelSpeech
|
||||
ttsAvailableRef.current = ttsAvailable
|
||||
})
|
||||
|
||||
useLayoutEffect(() => {
|
||||
if (mascotElRef.current) {
|
||||
mascotElRef.current.style.transform = `translate(${posRef.current.x}px, ${posRef.current.y}px)`
|
||||
}
|
||||
soundsRef.current = new TourSounds()
|
||||
return () => {
|
||||
soundsRef.current?.dispose()
|
||||
soundsRef.current = null
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Grab the app shell for the camera zoom; restore it when the tour ends
|
||||
useEffect(() => {
|
||||
const shell = document.querySelector<HTMLElement>('.rowboat-shell')
|
||||
shellRef.current = shell
|
||||
return () => {
|
||||
if (shell) {
|
||||
shell.style.transform = ''
|
||||
shell.style.transformOrigin = ''
|
||||
shell.style.transition = ''
|
||||
}
|
||||
shellRef.current = null
|
||||
zoomRef.current = null
|
||||
}
|
||||
}, [])
|
||||
|
||||
const applyZoom = useCallback((origin: { ox: number; oy: number } | null) => {
|
||||
const shell = shellRef.current
|
||||
if (!shell || reducedMotion) return
|
||||
if (origin) {
|
||||
// transform-origin transitions too, so moving between targets pans
|
||||
// smoothly instead of jumping when the origin changes
|
||||
shell.style.transition = `transform 0.9s ${GLIDE_EASING}, transform-origin 0.9s ${GLIDE_EASING}`
|
||||
shell.style.transformOrigin = `${origin.ox}px ${origin.oy}px`
|
||||
shell.style.transform = `scale(${ZOOM_SCALE})`
|
||||
zoomRef.current = { ox: origin.ox, oy: origin.oy, s: ZOOM_SCALE }
|
||||
} else {
|
||||
shell.style.transition = `transform 0.9s ${GLIDE_EASING}, transform-origin 0.9s ${GLIDE_EASING}`
|
||||
shell.style.transform = 'scale(1)'
|
||||
zoomRef.current = null
|
||||
}
|
||||
}, [reducedMotion])
|
||||
|
||||
// Where the element will sit on screen once this step's zoom settles:
|
||||
// undo the current zoom mathematically, then apply the upcoming one (whose
|
||||
// origin is the target's own center, so the center never moves).
|
||||
const displayedRect = useCallback((el: HTMLElement, willZoom: boolean): { rect: Rect; origin: { ox: number; oy: number } } => {
|
||||
const m = el.getBoundingClientRect()
|
||||
const z = zoomRef.current
|
||||
let cx = m.left + m.width / 2
|
||||
let cy = m.top + m.height / 2
|
||||
let w = m.width
|
||||
let h = m.height
|
||||
if (z) {
|
||||
cx = z.ox + (cx - z.ox) / z.s
|
||||
cy = z.oy + (cy - z.oy) / z.s
|
||||
w /= z.s
|
||||
h /= z.s
|
||||
}
|
||||
const s = willZoom && !reducedMotion ? ZOOM_SCALE : 1
|
||||
return {
|
||||
rect: { left: cx - (w * s) / 2, top: cy - (h * s) / 2, width: w * s, height: h * s },
|
||||
origin: { ox: cx, oy: cy },
|
||||
}
|
||||
}, [reducedMotion])
|
||||
|
||||
const cancelTravel = useCallback(() => {
|
||||
cancelAnimationFrame(travelRafRef.current)
|
||||
}, [])
|
||||
|
||||
const moveMascot = useCallback((p: Pt) => {
|
||||
posRef.current = p
|
||||
if (mascotElRef.current) {
|
||||
mascotElRef.current.style.transform = `translate(${p.x}px, ${p.y}px)`
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Row along a curved path from the current position, drawing the wake as we
|
||||
// go and splashing the oar; commits the wake as a dotted trail on arrival.
|
||||
const startTravel = useCallback((dest: Pt, onArrive: () => void) => {
|
||||
cancelTravel()
|
||||
const from = { ...posRef.current }
|
||||
const dist = Math.hypot(dest.x - from.x, dest.y - from.y)
|
||||
if (dist < 6 || reducedMotion) {
|
||||
moveMascot(dest)
|
||||
setRowing(false)
|
||||
onArrive()
|
||||
return
|
||||
}
|
||||
const dur = clamp(dist * 1.1, 550, 1500)
|
||||
curveSideRef.current = -curveSideRef.current
|
||||
const mx = (from.x + dest.x) / 2
|
||||
const my = (from.y + dest.y) / 2
|
||||
const nx = -(dest.y - from.y) / dist
|
||||
const ny = (dest.x - from.x) / dist
|
||||
const mag = Math.min(140, dist * 0.3) * curveSideRef.current
|
||||
const c = { x: mx + nx * mag, y: my + ny * mag }
|
||||
|
||||
// Wake follows the stern (bottom-center of the mascot box)
|
||||
const sternX = MASCOT_SIZE / 2
|
||||
const sternY = MASCOT_SIZE * 0.82
|
||||
const d = `M ${from.x + sternX} ${from.y + sternY} Q ${c.x + sternX} ${c.y + sternY} ${dest.x + sternX} ${dest.y + sternY}`
|
||||
const len = quadPathLength(d)
|
||||
setActiveWake({ d, len })
|
||||
setFlipped(dest.x < from.x - 4)
|
||||
setRowing(true)
|
||||
lastSplashRef.current = 0
|
||||
|
||||
const t0 = performance.now()
|
||||
const frame = (now: number) => {
|
||||
const raw = Math.min(1, (now - t0) / dur)
|
||||
const t = easeInOutCubic(raw)
|
||||
moveMascot(quadPoint(from, c, dest, t))
|
||||
if (wakePathElRef.current) {
|
||||
wakePathElRef.current.style.strokeDashoffset = String(len * (1 - t))
|
||||
}
|
||||
if (now - lastSplashRef.current > 420) {
|
||||
lastSplashRef.current = now
|
||||
soundsRef.current?.splash()
|
||||
}
|
||||
if (raw < 1) {
|
||||
travelRafRef.current = requestAnimationFrame(frame)
|
||||
} else {
|
||||
setRowing(false)
|
||||
setActiveWake(null)
|
||||
setWakes((ws) => [...ws, { id: wakeIdRef.current++, d }])
|
||||
onArrive()
|
||||
}
|
||||
}
|
||||
travelRafRef.current = requestAnimationFrame(frame)
|
||||
}, [cancelTravel, moveMascot, reducedMotion])
|
||||
|
||||
const playDing = useCallback(() => soundsRef.current?.ding(), [])
|
||||
|
||||
const finish = useCallback(() => {
|
||||
cancelSpeechRef.current()
|
||||
onCloseRef.current()
|
||||
}, [])
|
||||
|
||||
const goTo = useCallback((index: number, direction: 1 | -1) => {
|
||||
directionRef.current = direction
|
||||
if (index < 0) return
|
||||
if (index >= TOUR_STEPS.length) {
|
||||
finish()
|
||||
return
|
||||
}
|
||||
// Silence the current step's narration right away — not on arrival —
|
||||
// so it can't talk over (or into) the next step's speech.
|
||||
cancelSpeechRef.current()
|
||||
setStepIndex(index)
|
||||
}, [finish])
|
||||
|
||||
// Stop any in-flight narration when the tour unmounts
|
||||
useEffect(() => () => cancelSpeechRef.current(), [])
|
||||
|
||||
useEffect(() => {
|
||||
const handleResize = () => setResizeNonce((n) => n + 1)
|
||||
window.addEventListener('resize', handleResize)
|
||||
return () => window.removeEventListener('resize', handleResize)
|
||||
}, [])
|
||||
|
||||
// Enter the current step: navigate, wait for its anchor, aim the spotlight
|
||||
// and camera, row over, then narrate. Re-runs on resize to re-anchor.
|
||||
useEffect(() => {
|
||||
const step = TOUR_STEPS[stepIndex]
|
||||
const entering = enteredStepRef.current !== stepIndex
|
||||
let cancelled = false
|
||||
|
||||
if (entering && step.navigate) {
|
||||
onNavigateRef.current(step.navigate)
|
||||
}
|
||||
|
||||
const settle = (dest: Pt, side: 'left' | 'right', spotlight: Spot, origin: { ox: number; oy: number } | null) => {
|
||||
applyZoom(origin)
|
||||
setSpot(spotlight)
|
||||
setBubbleSide(side)
|
||||
if (!entering) {
|
||||
// Resize while already at this step: jump, keep the bubble up
|
||||
cancelTravel()
|
||||
moveMascot(dest)
|
||||
return
|
||||
}
|
||||
enteredStepRef.current = stepIndex
|
||||
setArrived(false)
|
||||
startTravel(dest, () => {
|
||||
if (cancelled) return
|
||||
setArrived(true)
|
||||
soundsRef.current?.bump()
|
||||
cancelSpeechRef.current()
|
||||
const clip = TOUR_CLIPS[step.id]
|
||||
if (clip) {
|
||||
speakUrlRef.current(clip)
|
||||
} else if (ttsAvailableRef.current) {
|
||||
speakRef.current(step.voiceText ?? step.text)
|
||||
}
|
||||
if (stepIndex === TOUR_STEPS.length - 1) {
|
||||
setConfettiOn(true)
|
||||
soundsRef.current?.fanfare()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if (!step.targetId) {
|
||||
const dest = mascotDestForCenter()
|
||||
applyZoom(null)
|
||||
settle(dest, 'right', spotForMascot(dest), null)
|
||||
return () => {
|
||||
cancelled = true
|
||||
cancelTravel()
|
||||
}
|
||||
}
|
||||
|
||||
const startedAt = performance.now()
|
||||
let pollRaf = 0
|
||||
const attempt = () => {
|
||||
if (cancelled) return
|
||||
const el = findTourTarget(step.targetId!)
|
||||
if (el) {
|
||||
const { rect, origin } = displayedRect(el, true)
|
||||
const dest = mascotDestForRect(rect)
|
||||
const side: 'left' | 'right' = dest.x + MASCOT_SIZE / 2 < window.innerWidth / 2 ? 'right' : 'left'
|
||||
settle(dest, side, spotForRect(rect), origin)
|
||||
return
|
||||
}
|
||||
if (performance.now() - startedAt < TARGET_RESOLVE_TIMEOUT_MS) {
|
||||
pollRaf = requestAnimationFrame(attempt)
|
||||
} else if (entering) {
|
||||
// Anchor never appeared (feature disabled / pane closed) — skip past it
|
||||
goTo(stepIndex + directionRef.current, directionRef.current as 1 | -1)
|
||||
}
|
||||
}
|
||||
attempt()
|
||||
return () => {
|
||||
cancelled = true
|
||||
cancelAnimationFrame(pollRaf)
|
||||
cancelTravel()
|
||||
}
|
||||
}, [stepIndex, resizeNonce, goTo, applyZoom, displayedRect, startTravel, cancelTravel, moveMascot])
|
||||
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') {
|
||||
e.preventDefault()
|
||||
finish()
|
||||
} else if (e.key === 'ArrowRight' || e.key === 'Enter') {
|
||||
e.preventDefault()
|
||||
goTo(stepIndexRef.current + 1, 1)
|
||||
} else if (e.key === 'ArrowLeft') {
|
||||
e.preventDefault()
|
||||
goTo(stepIndexRef.current - 1, -1)
|
||||
}
|
||||
}
|
||||
window.addEventListener('keydown', handleKeyDown)
|
||||
return () => window.removeEventListener('keydown', handleKeyDown)
|
||||
}, [finish, goTo])
|
||||
|
||||
const step = TOUR_STEPS[stepIndex]
|
||||
const isFirst = stepIndex === 0
|
||||
const isLast = stepIndex === TOUR_STEPS.length - 1
|
||||
|
||||
return createPortal(
|
||||
<>
|
||||
<style>{`
|
||||
@keyframes tour-bubble-in {
|
||||
0% { opacity: 0; transform: translateY(6px) scale(0.97); }
|
||||
100% { opacity: 1; transform: translateY(0) scale(1); }
|
||||
}
|
||||
@keyframes tour-wave-drift {
|
||||
from { transform: translateX(0); }
|
||||
to { transform: translateX(-50%); }
|
||||
}
|
||||
@keyframes tour-stamp-in {
|
||||
0% { opacity: 0; transform: scale(2); }
|
||||
100% { opacity: 1; transform: scale(1); }
|
||||
}
|
||||
`}</style>
|
||||
|
||||
{/* night falls: dim everything except the spotlight cutout */}
|
||||
{spot && (
|
||||
<div
|
||||
className="pointer-events-none fixed z-[64]"
|
||||
style={{
|
||||
left: spot.left,
|
||||
top: spot.top,
|
||||
width: spot.width,
|
||||
height: spot.height,
|
||||
borderRadius: spot.round ? 9999 : 14,
|
||||
boxShadow: '0 0 0 200vmax rgba(7, 14, 26, 0.52)',
|
||||
transition: `all 0.9s ${GLIDE_EASING}`,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
<TourWater />
|
||||
|
||||
{/* wake trails: the committed dotted route + the wake being drawn now */}
|
||||
<svg className="pointer-events-none fixed inset-0 z-[66] h-full w-full">
|
||||
{wakes.map((w) => (
|
||||
<path
|
||||
key={w.id}
|
||||
d={w.d}
|
||||
fill="none"
|
||||
stroke="#9CCBEA"
|
||||
strokeWidth={4}
|
||||
strokeLinecap="round"
|
||||
strokeDasharray="1 11"
|
||||
opacity={0.75}
|
||||
/>
|
||||
))}
|
||||
{activeWake && (
|
||||
<path
|
||||
ref={wakePathElRef}
|
||||
d={activeWake.d}
|
||||
fill="none"
|
||||
stroke="#BFE0F5"
|
||||
strokeWidth={5}
|
||||
strokeLinecap="round"
|
||||
strokeDasharray={activeWake.len}
|
||||
strokeDashoffset={activeWake.len}
|
||||
opacity={0.8}
|
||||
/>
|
||||
)}
|
||||
</svg>
|
||||
|
||||
<TourMiniMap total={TOUR_STEPS.length} current={stepIndex} arrived={arrived} />
|
||||
|
||||
{/* the boat (position driven imperatively during travel) */}
|
||||
<div
|
||||
ref={mascotElRef}
|
||||
className="fixed left-0 top-0 z-[70]"
|
||||
style={{ width: MASCOT_SIZE, pointerEvents: 'none' }}
|
||||
>
|
||||
{arrived && !reducedMotion && step.vignette && step.vignette !== 'agents' && (
|
||||
<MascotVignette kind={step.vignette} playDing={playDing} />
|
||||
)}
|
||||
<div style={{ transform: flipped ? 'scaleX(-1)' : undefined, transition: 'transform 0.35s ease-in-out' }}>
|
||||
<TalkingHead ttsState={ttsState} getLevel={getLevel} size={MASCOT_SIZE} hat={step.hat} rowing={rowing} />
|
||||
</div>
|
||||
{arrived && (
|
||||
<div
|
||||
key={step.id}
|
||||
className={cn(
|
||||
'pointer-events-auto absolute top-0 z-10 rounded-xl border border-border bg-popover p-4 text-popover-foreground shadow-lg',
|
||||
bubbleSide === 'right' ? 'left-full ml-3' : 'right-full mr-3'
|
||||
)}
|
||||
style={{
|
||||
width: BUBBLE_WIDTH,
|
||||
animation: 'tour-bubble-in 0.25s ease-out',
|
||||
}}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
onClick={finish}
|
||||
className="absolute right-2 top-2 flex h-6 w-6 items-center justify-center rounded-md text-muted-foreground transition-colors hover:bg-muted hover:text-foreground"
|
||||
aria-label="End tour"
|
||||
>
|
||||
<X className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
<p className="pr-6 text-sm font-semibold">{step.title}</p>
|
||||
<p className="mt-1.5 text-sm text-muted-foreground">{step.text}</p>
|
||||
<div className="mt-3 flex items-center justify-between">
|
||||
<span className="text-xs tabular-nums text-muted-foreground">
|
||||
{stepIndex + 1} / {TOUR_STEPS.length}
|
||||
</span>
|
||||
<div className="flex items-center gap-1.5">
|
||||
{!isFirst && (
|
||||
<Button variant="ghost" size="sm" className="h-7 px-2.5" onClick={() => goTo(stepIndex - 1, -1)}>
|
||||
Back
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
size="sm"
|
||||
className="h-7 px-3"
|
||||
onClick={() => (isLast ? finish() : goTo(stepIndex + 1, 1))}
|
||||
>
|
||||
{isLast ? 'Done' : 'Next'}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{arrived && !reducedMotion && step.vignette === 'agents' && <AgentsFleet />}
|
||||
|
||||
{confettiOn && !reducedMotion && <ConfettiBurst />}
|
||||
</>,
|
||||
document.body
|
||||
)
|
||||
}
|
||||
|
||||
/** Animated translucent waves lapping at the bottom of the screen. */
|
||||
function TourWater() {
|
||||
const back = useMemo(() => wavePath(2400, 96, 8, 22), [])
|
||||
const front = useMemo(() => wavePath(2400, 96, 12, 30), [])
|
||||
return (
|
||||
<div className="pointer-events-none fixed inset-x-0 bottom-0 z-[65] h-24 overflow-hidden" aria-hidden="true">
|
||||
<svg
|
||||
className="absolute bottom-0 left-0 h-full"
|
||||
style={{ width: '200%', animation: 'tour-wave-drift 11s linear infinite' }}
|
||||
viewBox="0 0 2400 96"
|
||||
preserveAspectRatio="none"
|
||||
>
|
||||
<path d={back} fill="#5F9BC9" opacity={0.3} />
|
||||
</svg>
|
||||
<svg
|
||||
className="absolute bottom-0 left-0 h-full"
|
||||
style={{ width: '200%', animation: 'tour-wave-drift 7s linear infinite reverse' }}
|
||||
viewBox="0 0 2400 96"
|
||||
preserveAspectRatio="none"
|
||||
>
|
||||
<path d={front} fill="#8FB6D9" opacity={0.35} />
|
||||
</svg>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Periodic wave: alternating up/down humps so a -50% translate loops seamlessly
|
||||
// (both hump counts are even, so half the width is a whole number of periods).
|
||||
function wavePath(width: number, height: number, humps: number, amp: number): string {
|
||||
const yTop = 34
|
||||
const seg = width / humps
|
||||
let d = `M 0 ${yTop}`
|
||||
for (let i = 0; i < humps; i++) {
|
||||
d += ` q ${seg / 2} ${i % 2 === 0 ? -amp : amp} ${seg} 0`
|
||||
}
|
||||
d += ` L ${width} ${height} L 0 ${height} Z`
|
||||
return d
|
||||
}
|
||||
|
||||
/** Parchment chart in the corner: islands per stop, dotted route, boat marker. */
|
||||
function TourMiniMap({ total, current, arrived }: { total: number; current: number; arrived: boolean }) {
|
||||
const MW = 184
|
||||
const MH = 96
|
||||
const points = useMemo(
|
||||
() =>
|
||||
Array.from({ length: total }, (_, i) => {
|
||||
const t = total === 1 ? 0 : i / (total - 1)
|
||||
return {
|
||||
x: 14 + (MW - 28) * t,
|
||||
y: MH / 2 + 4 + Math.sin(t * Math.PI * 1.5 + 0.6) * (MH * 0.26),
|
||||
}
|
||||
}),
|
||||
[total]
|
||||
)
|
||||
const route = points.map((p, i) => (i === 0 ? `M ${p.x} ${p.y}` : `L ${p.x} ${p.y}`)).join(' ')
|
||||
const boat = points[clamp(current, 0, total - 1)]
|
||||
|
||||
return (
|
||||
<div
|
||||
className="fixed bottom-5 left-5 z-[71] rounded-lg border-2 border-[#8A6B3D]/60 bg-[#F4E9CE] px-2 pb-1.5 pt-2 shadow-xl"
|
||||
style={{ transform: 'rotate(-1.2deg)' }}
|
||||
aria-hidden="true"
|
||||
>
|
||||
<p className="mb-0.5 px-1 text-[9px] font-semibold uppercase tracking-[0.22em] text-[#6B5138]">
|
||||
Voyage chart
|
||||
</p>
|
||||
<svg width={MW} height={MH} viewBox={`0 0 ${MW} ${MH}`}>
|
||||
<path d={route} fill="none" stroke="#8A6B3D" strokeWidth={1.5} strokeDasharray="3 4" opacity={0.65} />
|
||||
{points.map((p, i) => {
|
||||
const visited = i < current || (i === current && arrived)
|
||||
return (
|
||||
<g key={i}>
|
||||
<ellipse cx={p.x} cy={p.y} rx={7} ry={5} fill="#DFC896" stroke="#8A6B3D" strokeWidth={1.5} />
|
||||
{visited && (
|
||||
<g style={{ animation: 'tour-stamp-in 0.3s ease-out backwards' }}>
|
||||
<line x1={p.x - 1} y1={p.y - 13} x2={p.x - 1} y2={p.y - 3} stroke="#7A4A21" strokeWidth={1.5} />
|
||||
<path d={`M ${p.x - 1} ${p.y - 13} L ${p.x + 6} ${p.y - 10.5} L ${p.x - 1} ${p.y - 8} Z`} fill="#D9534F" />
|
||||
</g>
|
||||
)}
|
||||
</g>
|
||||
)
|
||||
})}
|
||||
{/* boat marker glides between islands in step with the real mascot */}
|
||||
<g style={{ transform: `translate(${boat.x}px, ${boat.y - 7}px)`, transition: `transform 0.9s ${GLIDE_EASING}` }}>
|
||||
<path d="M -7 0 Q 0 4 7 0 Q 4 6 0 6 Q -4 6 -7 0 Z" fill="#54402F" stroke="#3E2E24" strokeWidth={1} />
|
||||
<circle cx={0} cy={-3} r={3} fill="#E8E9F5" stroke="#17171B" strokeWidth={1} />
|
||||
</g>
|
||||
</svg>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** Two confetti cannons firing from the bottom corners, canvas-driven. */
|
||||
function ConfettiBurst() {
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null)
|
||||
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current
|
||||
const ctx = canvas?.getContext('2d')
|
||||
if (!canvas || !ctx) return
|
||||
const w = window.innerWidth
|
||||
const h = window.innerHeight
|
||||
const dpr = window.devicePixelRatio || 1
|
||||
canvas.width = w * dpr
|
||||
canvas.height = h * dpr
|
||||
ctx.scale(dpr, dpr)
|
||||
|
||||
const colors = ['#5B8DEF', '#F2B8BE', '#FFD166', '#7AC74F', '#8FB6D9', '#F2699C']
|
||||
const parts = Array.from({ length: 150 }, (_, i) => {
|
||||
const fromLeft = i % 2 === 0
|
||||
return {
|
||||
x: fromLeft ? 24 : w - 24,
|
||||
y: h - 40,
|
||||
vx: (fromLeft ? 1 : -1) * (2.5 + Math.random() * 6.5),
|
||||
vy: -(9 + Math.random() * 8),
|
||||
size: 5 + Math.random() * 5,
|
||||
color: colors[i % colors.length],
|
||||
rot: Math.random() * Math.PI,
|
||||
vr: (Math.random() - 0.5) * 0.3,
|
||||
}
|
||||
})
|
||||
|
||||
let raf = 0
|
||||
const t0 = performance.now()
|
||||
const frame = (now: number) => {
|
||||
ctx.clearRect(0, 0, w, h)
|
||||
for (const p of parts) {
|
||||
p.vy += 0.18
|
||||
p.vx *= 0.99
|
||||
p.x += p.vx
|
||||
p.y += p.vy
|
||||
p.rot += p.vr
|
||||
ctx.save()
|
||||
ctx.translate(p.x, p.y)
|
||||
ctx.rotate(p.rot)
|
||||
ctx.fillStyle = p.color
|
||||
ctx.fillRect(-p.size / 2, -p.size / 3, p.size, p.size * 0.66)
|
||||
ctx.restore()
|
||||
}
|
||||
if (now - t0 < 3200) {
|
||||
raf = requestAnimationFrame(frame)
|
||||
} else {
|
||||
ctx.clearRect(0, 0, w, h)
|
||||
}
|
||||
}
|
||||
raf = requestAnimationFrame(frame)
|
||||
return () => cancelAnimationFrame(raf)
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
className="pointer-events-none fixed inset-0 z-[72]"
|
||||
style={{ width: '100vw', height: '100vh' }}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import * as React from "react"
|
||||
import { useState, useEffect, useCallback, useMemo } from "react"
|
||||
import { Server, Key, Shield, Palette, Monitor, Sun, Moon, Loader2, CheckCircle2, Plus, X, Wrench, Search, ChevronRight, Link2, Tags, Mail, BookOpen, User, Plug, HelpCircle, MessageCircle, Bug, Terminal, AlertTriangle, RefreshCw, PanelRight, Bell } from "lucide-react"
|
||||
import { Server, Key, Shield, Palette, Monitor, Sun, Moon, Loader2, CheckCircle2, Plus, X, Wrench, Search, ChevronRight, Link2, Tags, Mail, BookOpen, User, Plug, HelpCircle, MessageCircle, Bug, Terminal, AlertTriangle, RefreshCw, PanelRight, Bell, Smartphone } from "lucide-react"
|
||||
|
||||
import {
|
||||
Dialog,
|
||||
|
|
@ -25,10 +25,11 @@ import { useTheme } from "@/contexts/theme-context"
|
|||
import { toast } from "sonner"
|
||||
import { AccountSettings } from "@/components/settings/account-settings"
|
||||
import { ConnectedAccountsSettings } from "@/components/settings/connected-accounts-settings"
|
||||
import { MobileChannelsSettings } from "@/components/settings/mobile-channels-settings"
|
||||
import type { ApprovalPolicy } from "@x/shared/src/code-mode.js"
|
||||
import { startProvisioning, useProvisioning, enabledOptimistic, type AgentStatus, type CodeModeAgentStatus } from "@/lib/code-mode-provisioning"
|
||||
|
||||
type ConfigTab = "account" | "connections" | "models" | "mcp" | "security" | "code-mode" | "appearance" | "notifications" | "note-tagging" | "help"
|
||||
type ConfigTab = "account" | "connections" | "mobile" | "models" | "mcp" | "security" | "code-mode" | "appearance" | "notifications" | "note-tagging" | "help"
|
||||
|
||||
interface TabConfig {
|
||||
id: ConfigTab
|
||||
|
|
@ -51,6 +52,12 @@ const tabs: TabConfig[] = [
|
|||
icon: Plug,
|
||||
description: "Manage accounts and tools",
|
||||
},
|
||||
{
|
||||
id: "mobile",
|
||||
label: "Mobile",
|
||||
icon: Smartphone,
|
||||
description: "Chat with Rowboat from WhatsApp or Telegram",
|
||||
},
|
||||
{
|
||||
id: "models",
|
||||
label: "Models",
|
||||
|
|
@ -320,8 +327,8 @@ const moreProviders: Array<{ id: LlmProviderFlavor; name: string; description: s
|
|||
]
|
||||
|
||||
const preferredDefaults: Partial<Record<LlmProviderFlavor, string>> = {
|
||||
openai: "gpt-5.2",
|
||||
anthropic: "claude-opus-4-6-20260202",
|
||||
openai: "gpt-5.4",
|
||||
anthropic: "claude-opus-4-8",
|
||||
}
|
||||
|
||||
const defaultBaseURLs: Partial<Record<LlmProviderFlavor, string>> = {
|
||||
|
|
@ -339,7 +346,7 @@ type ProviderModelConfig = {
|
|||
autoPermissionDecisionModel: string
|
||||
}
|
||||
|
||||
function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
||||
function ModelSettings({ dialogOpen, rowboatConnected = false }: { dialogOpen: boolean; rowboatConnected?: boolean }) {
|
||||
const [provider, setProvider] = useState<LlmProviderFlavor>("openai")
|
||||
const [defaultProvider, setDefaultProvider] = useState<LlmProviderFlavor | null>(null)
|
||||
const [providerConfigs, setProviderConfigs] = useState<Record<LlmProviderFlavor, ProviderModelConfig>>({
|
||||
|
|
@ -357,6 +364,11 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
const [testState, setTestState] = useState<{ status: "idle" | "testing" | "success" | "error"; error?: string }>({ status: "idle" })
|
||||
const [configLoading, setConfigLoading] = useState(true)
|
||||
const [showMoreProviders, setShowMoreProviders] = useState(false)
|
||||
// "Defer background tasks while a chat is running" — a top-level
|
||||
// models.json flag. deferExplicit tracks whether the user (or the Ollama
|
||||
// auto-enable) has ever set it, so we only auto-enable once.
|
||||
const [deferBackgroundTasks, setDeferBackgroundTasks] = useState(false)
|
||||
const [deferExplicit, setDeferExplicit] = useState(false)
|
||||
|
||||
const activeConfig = providerConfigs[provider]
|
||||
const showApiKey = provider === "openai" || provider === "anthropic" || provider === "google" || provider === "openrouter" || provider === "aigateway" || provider === "openai-compatible"
|
||||
|
|
@ -390,6 +402,8 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
useEffect(() => {
|
||||
if (!dialogOpen) return
|
||||
|
||||
const asString = (v: unknown): string => (typeof v === "string" ? v : "")
|
||||
|
||||
async function loadCurrentConfig() {
|
||||
try {
|
||||
setConfigLoading(true)
|
||||
|
|
@ -397,6 +411,8 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
path: "config/models.json",
|
||||
})
|
||||
const parsed = JSON.parse(result.data)
|
||||
setDeferBackgroundTasks(parsed?.deferBackgroundTasks === true)
|
||||
setDeferExplicit(typeof parsed?.deferBackgroundTasks === "boolean")
|
||||
if (parsed?.provider?.flavor && parsed?.model) {
|
||||
const flavor = parsed.provider.flavor as LlmProviderFlavor
|
||||
setProvider(flavor)
|
||||
|
|
@ -415,10 +431,10 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
apiKey: e.apiKey || "",
|
||||
baseURL: e.baseURL || (defaultBaseURLs[key as LlmProviderFlavor] || ""),
|
||||
models: savedModels,
|
||||
knowledgeGraphModel: e.knowledgeGraphModel || "",
|
||||
meetingNotesModel: e.meetingNotesModel || "",
|
||||
liveNoteAgentModel: e.liveNoteAgentModel || "",
|
||||
autoPermissionDecisionModel: e.autoPermissionDecisionModel || "",
|
||||
knowledgeGraphModel: asString(e.knowledgeGraphModel),
|
||||
meetingNotesModel: asString(e.meetingNotesModel),
|
||||
liveNoteAgentModel: asString(e.liveNoteAgentModel),
|
||||
autoPermissionDecisionModel: asString(e.autoPermissionDecisionModel),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -434,10 +450,10 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
apiKey: parsed.provider.apiKey || "",
|
||||
baseURL: parsed.provider.baseURL || (defaultBaseURLs[flavor] || ""),
|
||||
models: activeModels.length > 0 ? activeModels : [""],
|
||||
knowledgeGraphModel: parsed.knowledgeGraphModel || "",
|
||||
meetingNotesModel: parsed.meetingNotesModel || "",
|
||||
liveNoteAgentModel: parsed.liveNoteAgentModel || "",
|
||||
autoPermissionDecisionModel: parsed.autoPermissionDecisionModel || "",
|
||||
knowledgeGraphModel: asString(parsed.knowledgeGraphModel),
|
||||
meetingNotesModel: asString(parsed.meetingNotesModel),
|
||||
liveNoteAgentModel: asString(parsed.liveNoteAgentModel),
|
||||
autoPermissionDecisionModel: asString(parsed.autoPermissionDecisionModel),
|
||||
};
|
||||
}
|
||||
return next;
|
||||
|
|
@ -453,6 +469,17 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
loadCurrentConfig()
|
||||
}, [dialogOpen])
|
||||
|
||||
const handleDeferToggle = useCallback(async (value: boolean) => {
|
||||
setDeferBackgroundTasks(value)
|
||||
setDeferExplicit(true)
|
||||
try {
|
||||
await window.ipc.invoke("models:updateConfig", { deferBackgroundTasks: value })
|
||||
window.dispatchEvent(new Event("models-config-changed"))
|
||||
} catch {
|
||||
toast.error("Failed to save setting")
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Load models catalog
|
||||
useEffect(() => {
|
||||
if (!dialogOpen) return
|
||||
|
|
@ -510,10 +537,12 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
},
|
||||
model: allModels[0] || "",
|
||||
models: allModels,
|
||||
knowledgeGraphModel: activeConfig.knowledgeGraphModel.trim() || undefined,
|
||||
meetingNotesModel: activeConfig.meetingNotesModel.trim() || undefined,
|
||||
liveNoteAgentModel: activeConfig.liveNoteAgentModel.trim() || undefined,
|
||||
autoPermissionDecisionModel: activeConfig.autoPermissionDecisionModel.trim() || undefined,
|
||||
...(rowboatConnected ? {} : {
|
||||
knowledgeGraphModel: activeConfig.knowledgeGraphModel.trim() || undefined,
|
||||
meetingNotesModel: activeConfig.meetingNotesModel.trim() || undefined,
|
||||
liveNoteAgentModel: activeConfig.liveNoteAgentModel.trim() || undefined,
|
||||
autoPermissionDecisionModel: activeConfig.autoPermissionDecisionModel.trim() || undefined,
|
||||
}),
|
||||
}
|
||||
const result = await window.ipc.invoke("models:test", providerConfig)
|
||||
if (result.success) {
|
||||
|
|
@ -521,7 +550,23 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
setDefaultProvider(provider)
|
||||
setTestState({ status: "success" })
|
||||
window.dispatchEvent(new Event('models-config-changed'))
|
||||
toast.success("Model configuration saved")
|
||||
// Local models compete with background agents for the same hardware:
|
||||
// when the user connects Ollama and has never touched the defer
|
||||
// flag, enable it for them (they can switch it off below).
|
||||
if (provider === "ollama" && !deferExplicit && !deferBackgroundTasks) {
|
||||
void handleDeferToggle(true)
|
||||
}
|
||||
// Capability probe caveats (local models): saved, but the user should
|
||||
// know when the model can't do tools or has a too-small context.
|
||||
const warnings: string[] = result.warnings ?? []
|
||||
if (warnings.length > 0) {
|
||||
for (const warning of warnings) {
|
||||
toast.warning(warning, { duration: 12000 })
|
||||
}
|
||||
toast.success("Model configuration saved (with warnings)")
|
||||
} else {
|
||||
toast.success("Model configuration saved")
|
||||
}
|
||||
} else {
|
||||
setTestState({ status: "error", error: result.error })
|
||||
toast.error(result.error || "Connection test failed")
|
||||
|
|
@ -530,7 +575,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
setTestState({ status: "error", error: "Connection test failed" })
|
||||
toast.error("Connection test failed")
|
||||
}
|
||||
}, [canTest, provider, activeConfig])
|
||||
}, [canTest, provider, activeConfig, rowboatConnected, deferExplicit, deferBackgroundTasks, handleDeferToggle])
|
||||
|
||||
const handleSetDefault = useCallback(async (prov: LlmProviderFlavor) => {
|
||||
const config = providerConfigs[prov]
|
||||
|
|
@ -545,10 +590,12 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
},
|
||||
model: allModels[0],
|
||||
models: allModels,
|
||||
knowledgeGraphModel: config.knowledgeGraphModel.trim() || undefined,
|
||||
meetingNotesModel: config.meetingNotesModel.trim() || undefined,
|
||||
liveNoteAgentModel: config.liveNoteAgentModel.trim() || undefined,
|
||||
autoPermissionDecisionModel: config.autoPermissionDecisionModel.trim() || undefined,
|
||||
...(rowboatConnected ? {} : {
|
||||
knowledgeGraphModel: config.knowledgeGraphModel.trim() || undefined,
|
||||
meetingNotesModel: config.meetingNotesModel.trim() || undefined,
|
||||
liveNoteAgentModel: config.liveNoteAgentModel.trim() || undefined,
|
||||
autoPermissionDecisionModel: config.autoPermissionDecisionModel.trim() || undefined,
|
||||
}),
|
||||
})
|
||||
setDefaultProvider(prov)
|
||||
window.dispatchEvent(new Event('models-config-changed'))
|
||||
|
|
@ -556,7 +603,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
} catch {
|
||||
toast.error("Failed to set default provider")
|
||||
}
|
||||
}, [providerConfigs])
|
||||
}, [providerConfigs, rowboatConnected])
|
||||
|
||||
const handleDeleteProvider = useCallback(async (prov: LlmProviderFlavor) => {
|
||||
try {
|
||||
|
|
@ -577,10 +624,12 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
}
|
||||
parsed.model = defModels[0] || ""
|
||||
parsed.models = defModels
|
||||
parsed.knowledgeGraphModel = defConfig.knowledgeGraphModel.trim() || undefined
|
||||
parsed.meetingNotesModel = defConfig.meetingNotesModel.trim() || undefined
|
||||
parsed.liveNoteAgentModel = defConfig.liveNoteAgentModel.trim() || undefined
|
||||
parsed.autoPermissionDecisionModel = defConfig.autoPermissionDecisionModel.trim() || undefined
|
||||
if (!rowboatConnected) {
|
||||
parsed.knowledgeGraphModel = defConfig.knowledgeGraphModel.trim() || undefined
|
||||
parsed.meetingNotesModel = defConfig.meetingNotesModel.trim() || undefined
|
||||
parsed.liveNoteAgentModel = defConfig.liveNoteAgentModel.trim() || undefined
|
||||
parsed.autoPermissionDecisionModel = defConfig.autoPermissionDecisionModel.trim() || undefined
|
||||
}
|
||||
}
|
||||
await window.ipc.invoke("workspace:writeFile", {
|
||||
path: "config/models.json",
|
||||
|
|
@ -596,7 +645,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
} catch {
|
||||
toast.error("Failed to remove provider")
|
||||
}
|
||||
}, [defaultProvider, providerConfigs])
|
||||
}, [defaultProvider, providerConfigs, rowboatConnected])
|
||||
|
||||
const renderProviderCard = (p: { id: LlmProviderFlavor; name: string; description: string }) => {
|
||||
const isDefault = defaultProvider === p.id
|
||||
|
|
@ -618,7 +667,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="text-sm font-medium">{p.name}</span>
|
||||
{isDefault && (
|
||||
{isDefault && !rowboatConnected && (
|
||||
<span className="rounded-full bg-primary/10 px-1.5 py-0.5 text-[10px] font-medium leading-none text-primary">
|
||||
Default
|
||||
</span>
|
||||
|
|
@ -627,16 +676,18 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
<div className="text-xs text-muted-foreground mt-0.5">{p.description}</div>
|
||||
{!isDefault && hasModel && isSelected && (
|
||||
<div className="mt-1.5 flex items-center gap-3">
|
||||
<span
|
||||
role="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
handleSetDefault(p.id)
|
||||
}}
|
||||
className="inline-flex text-[11px] text-muted-foreground hover:text-primary transition-colors cursor-pointer"
|
||||
>
|
||||
Set as default
|
||||
</span>
|
||||
{!rowboatConnected && (
|
||||
<span
|
||||
role="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
handleSetDefault(p.id)
|
||||
}}
|
||||
className="inline-flex text-[11px] text-muted-foreground hover:text-primary transition-colors cursor-pointer"
|
||||
>
|
||||
Set as default
|
||||
</span>
|
||||
)}
|
||||
<span
|
||||
role="button"
|
||||
onClick={(e) => {
|
||||
|
|
@ -688,7 +739,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
<div className="grid grid-cols-2 gap-3">
|
||||
{/* Assistant models (left column) */}
|
||||
<div className="space-y-2">
|
||||
<span className="text-xs font-medium text-muted-foreground uppercase tracking-wider">Assistant model</span>
|
||||
<span className="text-xs font-medium text-muted-foreground uppercase tracking-wider">{rowboatConnected ? "Model" : "Assistant model"}</span>
|
||||
{modelsLoading ? (
|
||||
<div className="flex items-center gap-2 text-sm text-muted-foreground">
|
||||
<Loader2 className="size-4 animate-spin" />
|
||||
|
|
@ -726,6 +777,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
)}
|
||||
</div>
|
||||
|
||||
{!rowboatConnected && (<>
|
||||
{/* Knowledge graph model (right column) */}
|
||||
<div className="space-y-2">
|
||||
<span className="text-xs font-medium text-muted-foreground uppercase tracking-wider">Knowledge graph model</span>
|
||||
|
|
@ -861,6 +913,7 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
</Select>
|
||||
)}
|
||||
</div>
|
||||
</>)}
|
||||
</div>
|
||||
|
||||
{/* API Key */}
|
||||
|
|
@ -909,6 +962,17 @@ function ModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
</div>
|
||||
)}
|
||||
|
||||
{/* Defer background tasks while chatting */}
|
||||
<div className="flex items-center justify-between gap-4 rounded-md border px-3 py-2.5">
|
||||
<div className="min-w-0">
|
||||
<div className="text-sm font-medium">Defer background tasks while chatting</div>
|
||||
<div className="text-xs text-muted-foreground mt-0.5">
|
||||
Background agents (knowledge sync, live notes, tasks) wait until your chat finishes. Recommended for local models.
|
||||
</div>
|
||||
</div>
|
||||
<Switch checked={deferBackgroundTasks} onCheckedChange={handleDeferToggle} />
|
||||
</div>
|
||||
|
||||
{/* Test & Save button */}
|
||||
<Button
|
||||
onClick={handleTestAndSave}
|
||||
|
|
@ -1245,11 +1309,45 @@ function ToolsLibrarySettings({ dialogOpen, rowboatConnected }: { dialogOpen: bo
|
|||
}
|
||||
|
||||
// --- Rowboat Model Settings (when signed in via Rowboat) ---
|
||||
//
|
||||
// Hybrid mode: every dropdown lists the gateway catalog PLUS any models from
|
||||
// BYOK providers configured below. Values are provider-qualified
|
||||
// ("provider::model") and saved via models:updateConfig as {provider, model}
|
||||
// refs, so a signed-in user can e.g. keep the gateway assistant while
|
||||
// running background agents on a local Ollama model.
|
||||
|
||||
interface HybridModelOption {
|
||||
provider: string
|
||||
model: string
|
||||
label: string
|
||||
}
|
||||
|
||||
const providerDisplayNames: Record<string, string> = {
|
||||
openai: 'OpenAI',
|
||||
anthropic: 'Anthropic',
|
||||
google: 'Gemini',
|
||||
ollama: 'Ollama',
|
||||
openrouter: 'OpenRouter',
|
||||
aigateway: 'AI Gateway',
|
||||
'openai-compatible': 'OpenAI-Compatible',
|
||||
rowboat: 'Rowboat',
|
||||
}
|
||||
|
||||
const HYBRID_SEP = "::"
|
||||
const hybridKey = (provider: string, model: string) => `${provider}${HYBRID_SEP}${model}`
|
||||
|
||||
function parseHybridKey(key: string): { provider: string; model: string } | null {
|
||||
const index = key.indexOf(HYBRID_SEP)
|
||||
if (index <= 0) return null
|
||||
return { provider: key.slice(0, index), model: key.slice(index + HYBRID_SEP.length) }
|
||||
}
|
||||
|
||||
function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
||||
const [gatewayModels, setGatewayModels] = useState<LlmModelOption[]>([])
|
||||
const [selectedModel, setSelectedModel] = useState("")
|
||||
const [selectedKgModel, setSelectedKgModel] = useState("")
|
||||
const [options, setOptions] = useState<HybridModelOption[]>([])
|
||||
const [selectedDefault, setSelectedDefault] = useState("")
|
||||
const [selectedKg, setSelectedKg] = useState("")
|
||||
const [selectedLiveNote, setSelectedLiveNote] = useState("")
|
||||
const [selectedAutoPermission, setSelectedAutoPermission] = useState("")
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [saving, setSaving] = useState(false)
|
||||
|
||||
|
|
@ -1259,22 +1357,63 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
async function load() {
|
||||
setLoading(true)
|
||||
try {
|
||||
// Fetch gateway models
|
||||
const listResult = await window.ipc.invoke("models:list", null)
|
||||
const rowboatProvider = listResult.providers?.find((p: { id: string }) => p.id === "rowboat")
|
||||
const models = rowboatProvider?.models || []
|
||||
setGatewayModels(models)
|
||||
const collected: HybridModelOption[] = []
|
||||
const seen = new Set<string>()
|
||||
const push = (provider: string, model: string, label?: string) => {
|
||||
if (!model) return
|
||||
const key = hybridKey(provider, model)
|
||||
if (seen.has(key)) return
|
||||
seen.add(key)
|
||||
collected.push({ provider, model, label: label || model })
|
||||
}
|
||||
|
||||
// Read current selection from config
|
||||
const catalog: Record<string, LlmModelOption[]> = {}
|
||||
try {
|
||||
const listResult = await window.ipc.invoke("models:list", null)
|
||||
for (const p of listResult.providers || []) {
|
||||
catalog[p.id] = p.models || []
|
||||
}
|
||||
} catch { /* offline — BYOK entries below still load */ }
|
||||
for (const m of catalog["rowboat"] || []) push("rowboat", m.id, m.name || m.id)
|
||||
|
||||
let parsed: Record<string, unknown> = {}
|
||||
try {
|
||||
const configResult = await window.ipc.invoke("workspace:readFile", { path: "config/models.json" })
|
||||
const parsed = JSON.parse(configResult.data)
|
||||
if (parsed?.model) setSelectedModel(parsed.model)
|
||||
if (parsed?.knowledgeGraphModel) setSelectedKgModel(parsed.knowledgeGraphModel)
|
||||
} catch {
|
||||
// No config yet — pick first model as default
|
||||
if (models.length > 0) setSelectedModel(models[0].id)
|
||||
parsed = JSON.parse(configResult.data)
|
||||
} catch { /* no BYOK config yet */ }
|
||||
|
||||
const providersMap = (parsed.providers ?? {}) as Record<string, Record<string, unknown>>
|
||||
for (const [flavor, entry] of Object.entries(providersMap)) {
|
||||
const hasKey = typeof entry.apiKey === "string" && (entry.apiKey as string).trim().length > 0
|
||||
const hasBaseURL = typeof entry.baseURL === "string" && (entry.baseURL as string).trim().length > 0
|
||||
if (!hasKey && !hasBaseURL) continue
|
||||
push(flavor, typeof entry.model === "string" ? entry.model : "")
|
||||
const catalogModels = catalog[flavor] || []
|
||||
if (catalogModels.length > 0) {
|
||||
for (const m of catalogModels) push(flavor, m.id, m.name || m.id)
|
||||
} else {
|
||||
for (const m of Array.isArray(entry.models) ? entry.models as string[] : []) push(flavor, m)
|
||||
}
|
||||
}
|
||||
setOptions(collected)
|
||||
|
||||
// Current selections. Legacy string overrides pair with the BYOK
|
||||
// top-level flavor (mirrors core/models/defaults.ts).
|
||||
const legacyFlavor = (parsed.provider as Record<string, unknown> | undefined)?.flavor
|
||||
const toKey = (value: unknown): string => {
|
||||
if (!value) return ""
|
||||
if (typeof value === "string") {
|
||||
return typeof legacyFlavor === "string" ? hybridKey(legacyFlavor, value) : ""
|
||||
}
|
||||
const ref = value as { provider?: unknown; model?: unknown }
|
||||
return typeof ref.provider === "string" && typeof ref.model === "string"
|
||||
? hybridKey(ref.provider, ref.model)
|
||||
: ""
|
||||
}
|
||||
setSelectedDefault(toKey(parsed.defaultSelection))
|
||||
setSelectedKg(toKey(parsed.knowledgeGraphModel))
|
||||
setSelectedLiveNote(toKey(parsed.liveNoteAgentModel))
|
||||
setSelectedAutoPermission(toKey(parsed.autoPermissionDecisionModel))
|
||||
} catch {
|
||||
toast.error("Failed to load models")
|
||||
} finally {
|
||||
|
|
@ -1286,13 +1425,14 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
}, [dialogOpen])
|
||||
|
||||
const handleSave = useCallback(async () => {
|
||||
if (!selectedModel) return
|
||||
setSaving(true)
|
||||
try {
|
||||
await window.ipc.invoke("models:saveConfig", {
|
||||
provider: { flavor: "openrouter" as const },
|
||||
model: selectedModel,
|
||||
knowledgeGraphModel: selectedKgModel || undefined,
|
||||
const toRef = (key: string) => (key ? parseHybridKey(key) : null)
|
||||
await window.ipc.invoke("models:updateConfig", {
|
||||
defaultSelection: toRef(selectedDefault),
|
||||
knowledgeGraphModel: toRef(selectedKg),
|
||||
liveNoteAgentModel: toRef(selectedLiveNote),
|
||||
autoPermissionDecisionModel: toRef(selectedAutoPermission),
|
||||
})
|
||||
window.dispatchEvent(new Event("models-config-changed"))
|
||||
toast.success("Model configuration saved")
|
||||
|
|
@ -1301,7 +1441,37 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
} finally {
|
||||
setSaving(false)
|
||||
}
|
||||
}, [selectedModel, selectedKgModel])
|
||||
}, [selectedDefault, selectedKg, selectedLiveNote, selectedAutoPermission])
|
||||
|
||||
const renderSelect = (
|
||||
label: string,
|
||||
value: string,
|
||||
onChange: (v: string) => void,
|
||||
defaultLabel: string,
|
||||
) => (
|
||||
<div className="space-y-2">
|
||||
<label className="text-sm font-medium">{label}</label>
|
||||
<Select value={value || "__default__"} onValueChange={(v) => onChange(v === "__default__" ? "" : v)}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue placeholder={defaultLabel} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="__default__">{defaultLabel}</SelectItem>
|
||||
{options.map((o) => {
|
||||
const key = hybridKey(o.provider, o.model)
|
||||
return (
|
||||
<SelectItem key={key} value={key}>
|
||||
{o.label}
|
||||
<span className="ml-2 text-xs text-muted-foreground">
|
||||
{providerDisplayNames[o.provider] || o.provider}
|
||||
</span>
|
||||
</SelectItem>
|
||||
)
|
||||
})}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
)
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
|
|
@ -1314,46 +1484,16 @@ function RowboatModelSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
return (
|
||||
<div className="space-y-6">
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Select the models Rowboat uses. These are provided through your Rowboat account.
|
||||
Select the models Rowboat uses. Rowboat models are provided through your account; models from your own providers route through your keys or local runtimes.
|
||||
</p>
|
||||
|
||||
{/* Assistant model */}
|
||||
<div className="space-y-2">
|
||||
<label className="text-sm font-medium">Assistant model</label>
|
||||
<Select value={selectedModel} onValueChange={setSelectedModel}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue placeholder="Select a model" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{gatewayModels.map((m) => (
|
||||
<SelectItem key={m.id} value={m.id}>
|
||||
{m.name || m.id}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
|
||||
{/* Knowledge graph model */}
|
||||
<div className="space-y-2">
|
||||
<label className="text-sm font-medium">Knowledge graph model</label>
|
||||
<Select value={selectedKgModel || "__same__"} onValueChange={(v) => setSelectedKgModel(v === "__same__" ? "" : v)}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue placeholder="Same as assistant" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="__same__">Same as assistant</SelectItem>
|
||||
{gatewayModels.map((m) => (
|
||||
<SelectItem key={m.id} value={m.id}>
|
||||
{m.name || m.id}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
{renderSelect("Assistant model", selectedDefault, setSelectedDefault, "Rowboat default")}
|
||||
{renderSelect("Knowledge graph model", selectedKg, setSelectedKg, "Rowboat default")}
|
||||
{renderSelect("Background agents model", selectedLiveNote, setSelectedLiveNote, "Rowboat default")}
|
||||
{renderSelect("Permission checks model", selectedAutoPermission, setSelectedAutoPermission, "Rowboat default")}
|
||||
|
||||
{/* Save */}
|
||||
<Button onClick={handleSave} disabled={!selectedModel || saving}>
|
||||
<Button onClick={handleSave} disabled={saving}>
|
||||
{saving ? (
|
||||
<><Loader2 className="size-4 animate-spin mr-2" />Saving...</>
|
||||
) : (
|
||||
|
|
@ -2094,7 +2234,9 @@ export function SettingsDialog({ children, defaultTab = "account", open: control
|
|||
})
|
||||
}, [open])
|
||||
|
||||
const visibleTabs = useMemo(() => rowboatConnected ? tabs.filter(t => t.id !== "models") : tabs, [rowboatConnected])
|
||||
// Hybrid mode: the Models tab is shown in both modes — signed-in users can
|
||||
// pick gateway models AND bring their own providers/models alongside.
|
||||
const visibleTabs = tabs
|
||||
|
||||
const activeTabConfig = visibleTabs.find((t) => t.id === activeTab) ?? visibleTabs[0]
|
||||
const isJsonTab = activeTab === "mcp" || activeTab === "security"
|
||||
|
|
@ -2216,7 +2358,7 @@ export function SettingsDialog({ children, defaultTab = "account", open: control
|
|||
</div>
|
||||
|
||||
{/* Content */}
|
||||
<div className={cn("flex-1 p-4 min-h-0", (activeTab === "models" || activeTab === "connections" || activeTab === "account" || activeTab === "code-mode" || activeTab === "notifications") ? "overflow-y-auto" : activeTab === "note-tagging" ? "overflow-hidden flex flex-col" : "overflow-hidden")}>
|
||||
<div className={cn("flex-1 p-4 min-h-0", (activeTab === "models" || activeTab === "connections" || activeTab === "mobile" || activeTab === "account" || activeTab === "code-mode" || activeTab === "notifications") ? "overflow-y-auto" : activeTab === "note-tagging" ? "overflow-hidden flex flex-col" : "overflow-hidden")}>
|
||||
{activeTab === "account" ? (
|
||||
<AccountSettings dialogOpen={open} />
|
||||
) : activeTab === "connections" ? (
|
||||
|
|
@ -2231,9 +2373,23 @@ export function SettingsDialog({ children, defaultTab = "account", open: control
|
|||
<ToolsLibrarySettings dialogOpen={open} rowboatConnected={rowboatConnected} />
|
||||
</div>
|
||||
</div>
|
||||
) : activeTab === "mobile" ? (
|
||||
<MobileChannelsSettings dialogOpen={open} />
|
||||
) : activeTab === "models" ? (
|
||||
rowboatConnected
|
||||
? <RowboatModelSettings dialogOpen={open} />
|
||||
? (
|
||||
<div className="space-y-8">
|
||||
<RowboatModelSettings dialogOpen={open} />
|
||||
<Separator />
|
||||
<div className="space-y-2">
|
||||
<h4 className="text-sm font-semibold">Your own providers</h4>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Connect your own API keys or local runtimes (Ollama, LM Studio). Their models appear in the model pickers above and alongside your Rowboat models, and are billed to you directly.
|
||||
</p>
|
||||
<ModelSettings dialogOpen={open} rowboatConnected />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
: <ModelSettings dialogOpen={open} />
|
||||
) : activeTab === "note-tagging" ? (
|
||||
<NoteTaggingSettings dialogOpen={open} />
|
||||
|
|
|
|||
|
|
@ -0,0 +1,287 @@
|
|||
"use client"
|
||||
|
||||
import { useCallback, useEffect, useState } from "react"
|
||||
import type { z } from "zod"
|
||||
import { Loader2, MessageCircle, Send, Smartphone } from "lucide-react"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Input } from "@/components/ui/input"
|
||||
import { Separator } from "@/components/ui/separator"
|
||||
import { Switch } from "@/components/ui/switch"
|
||||
import { toast } from "sonner"
|
||||
import type { ChannelsConfig, ChannelsStatus } from "@x/shared/src/channels.js"
|
||||
|
||||
type Config = z.infer<typeof ChannelsConfig>
|
||||
type Status = z.infer<typeof ChannelsStatus>
|
||||
|
||||
// Comma/newline separated entries; each entry keeps digits only, so a
|
||||
// formatted number like "+1 (415) 555-1234" survives as one entry instead of
|
||||
// being shattered at the spaces.
|
||||
function parseIdList(draft: string): string[] {
|
||||
return draft
|
||||
.split(/[,;\n]+/)
|
||||
.map((s) => s.replace(/\D/g, ""))
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
export function MobileChannelsSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
||||
const [config, setConfig] = useState<Config | null>(null)
|
||||
const [status, setStatus] = useState<Status | null>(null)
|
||||
const [tokenDraft, setTokenDraft] = useState("")
|
||||
const [waAllowDraft, setWaAllowDraft] = useState("")
|
||||
const [tgAllowDraft, setTgAllowDraft] = useState("")
|
||||
const [saving, setSaving] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (!dialogOpen) return
|
||||
let cancelled = false
|
||||
void (async () => {
|
||||
try {
|
||||
const [cfg, st] = await Promise.all([
|
||||
window.ipc.invoke("channels:getConfig", null),
|
||||
window.ipc.invoke("channels:getStatus", null),
|
||||
])
|
||||
if (cancelled) return
|
||||
setConfig(cfg)
|
||||
setStatus(st)
|
||||
setTokenDraft(cfg.telegram.botToken)
|
||||
setWaAllowDraft(cfg.whatsapp.allowFrom.join(", "))
|
||||
setTgAllowDraft(cfg.telegram.allowFrom.join(", "))
|
||||
} catch {
|
||||
if (!cancelled) toast.error("Failed to load mobile channel settings")
|
||||
}
|
||||
})()
|
||||
const unsubscribe = window.ipc.on("channels:status", (st) => {
|
||||
setStatus(st)
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
unsubscribe()
|
||||
}
|
||||
}, [dialogOpen])
|
||||
|
||||
const save = useCallback(async (next: Config) => {
|
||||
setConfig(next)
|
||||
setSaving(true)
|
||||
try {
|
||||
await window.ipc.invoke("channels:setConfig", next)
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "Failed to save channel settings")
|
||||
} finally {
|
||||
setSaving(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
if (!config) {
|
||||
return (
|
||||
<div className="flex items-center justify-center py-8 text-muted-foreground">
|
||||
<Loader2 className="size-4 animate-spin" />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const wa = status?.whatsapp
|
||||
const tg = status?.telegram
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div className="flex items-start gap-2.5 rounded-md bg-muted/50 px-3 py-2.5">
|
||||
<Smartphone className="size-4 mt-0.5 shrink-0 text-muted-foreground" />
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Chat with Rowboat from your phone. Send <span className="font-mono">help</span> for
|
||||
commands (<span className="font-mono">list</span>, <span className="font-mono">resume 2</span>,{" "}
|
||||
<span className="font-mono">new</span>, <span className="font-mono">stop</span>) — anything
|
||||
else is a message to the current chat. Your computer must be on with Rowboat running.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* WhatsApp */}
|
||||
<div className="space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2.5">
|
||||
<div className="flex size-8 items-center justify-center rounded-md bg-muted">
|
||||
<MessageCircle className="size-4" />
|
||||
</div>
|
||||
<div className="flex flex-col">
|
||||
<span className="text-sm font-medium">WhatsApp</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{wa?.state === "connected"
|
||||
? `Linked as +${wa.self ?? "?"} — message yourself to use it`
|
||||
: wa?.state === "qr"
|
||||
? "Scan the QR below with your phone"
|
||||
: wa?.state === "starting"
|
||||
? "Connecting…"
|
||||
: wa?.state === "error"
|
||||
? wa.error ?? "Error"
|
||||
: "Links your own WhatsApp as a companion device"}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
{wa?.state === "connected" && (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-7 px-3 text-xs"
|
||||
onClick={() => {
|
||||
void window.ipc.invoke("channels:whatsappLogout", null).catch(() => {
|
||||
toast.error("Failed to unlink WhatsApp")
|
||||
})
|
||||
}}
|
||||
>
|
||||
Unlink
|
||||
</Button>
|
||||
)}
|
||||
<Switch
|
||||
checked={config.whatsapp.enabled}
|
||||
disabled={saving}
|
||||
onCheckedChange={(enabled) =>
|
||||
void save({ ...config, whatsapp: { ...config.whatsapp, enabled } })
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{config.whatsapp.enabled && wa?.state === "qr" && wa.qrDataUrl && (
|
||||
<div className="flex items-center gap-4 rounded-md border p-3">
|
||||
<img src={wa.qrDataUrl} alt="WhatsApp pairing QR" className="size-40 rounded" />
|
||||
<div className="text-xs text-muted-foreground space-y-1">
|
||||
<p className="font-medium text-foreground">Link Rowboat to WhatsApp</p>
|
||||
<p>1. Open WhatsApp on your phone</p>
|
||||
<p>2. Settings → Linked Devices → Link a Device</p>
|
||||
<p>3. Scan this code</p>
|
||||
<p className="pt-1">
|
||||
Then message <span className="font-medium">yourself</span> (your own contact) to talk
|
||||
to Rowboat.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{config.whatsapp.enabled && (
|
||||
<div className="space-y-1.5">
|
||||
<label className="text-xs font-medium">Additional allowed numbers</label>
|
||||
<div className="flex gap-2">
|
||||
<Input
|
||||
value={waAllowDraft}
|
||||
onChange={(e) => setWaAllowDraft(e.target.value)}
|
||||
placeholder="e.g. 14155551234, 919876543210"
|
||||
className="h-8 text-xs"
|
||||
/>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-8 px-3 text-xs"
|
||||
disabled={saving}
|
||||
onClick={() =>
|
||||
void save({
|
||||
...config,
|
||||
whatsapp: { ...config.whatsapp, allowFrom: parseIdList(waAllowDraft) },
|
||||
})
|
||||
}
|
||||
>
|
||||
Save
|
||||
</Button>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Your own number (self-chat) is always allowed. Digits only, with country code.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<Separator />
|
||||
|
||||
{/* Telegram */}
|
||||
<div className="space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2.5">
|
||||
<div className="flex size-8 items-center justify-center rounded-md bg-muted">
|
||||
<Send className="size-4" />
|
||||
</div>
|
||||
<div className="flex flex-col">
|
||||
<span className="text-sm font-medium">Telegram</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{tg?.state === "polling"
|
||||
? `Listening${tg.botUsername ? ` as @${tg.botUsername}` : ""}`
|
||||
: tg?.state === "starting"
|
||||
? "Connecting…"
|
||||
: tg?.state === "error"
|
||||
? tg.error ?? "Error"
|
||||
: "Uses your own bot — create one with @BotFather"}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<Switch
|
||||
checked={config.telegram.enabled}
|
||||
disabled={saving}
|
||||
onCheckedChange={(enabled) =>
|
||||
void save({ ...config, telegram: { ...config.telegram, enabled } })
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{config.telegram.enabled && (
|
||||
<>
|
||||
<div className="space-y-1.5">
|
||||
<label className="text-xs font-medium">Bot token</label>
|
||||
<div className="flex gap-2">
|
||||
<Input
|
||||
type="password"
|
||||
value={tokenDraft}
|
||||
onChange={(e) => setTokenDraft(e.target.value)}
|
||||
placeholder="123456789:AAF…"
|
||||
className="h-8 text-xs font-mono"
|
||||
/>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-8 px-3 text-xs"
|
||||
disabled={saving}
|
||||
onClick={() =>
|
||||
void save({
|
||||
...config,
|
||||
telegram: { ...config.telegram, botToken: tokenDraft.trim() },
|
||||
})
|
||||
}
|
||||
>
|
||||
Save
|
||||
</Button>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Message @BotFather on Telegram → /newbot → paste the token here.
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-1.5">
|
||||
<label className="text-xs font-medium">Allowed chat IDs</label>
|
||||
<div className="flex gap-2">
|
||||
<Input
|
||||
value={tgAllowDraft}
|
||||
onChange={(e) => setTgAllowDraft(e.target.value)}
|
||||
placeholder="e.g. 123456789"
|
||||
className="h-8 text-xs"
|
||||
/>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-8 px-3 text-xs"
|
||||
disabled={saving}
|
||||
onClick={() =>
|
||||
void save({
|
||||
...config,
|
||||
telegram: { ...config.telegram, allowFrom: parseIdList(tgAllowDraft) },
|
||||
})
|
||||
}
|
||||
>
|
||||
Save
|
||||
</Button>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Message your bot once — it replies with your chat ID to add here.
|
||||
</p>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ import {
|
|||
Globe,
|
||||
AlertTriangle,
|
||||
Home,
|
||||
LayoutGrid,
|
||||
Mic,
|
||||
SquarePen,
|
||||
Plug,
|
||||
|
|
@ -22,6 +23,8 @@ import {
|
|||
Settings,
|
||||
Square,
|
||||
Video,
|
||||
CircleAlert,
|
||||
X,
|
||||
} from "lucide-react"
|
||||
import {
|
||||
AlertDialog,
|
||||
|
|
@ -51,6 +54,7 @@ import {
|
|||
Popover,
|
||||
PopoverContent,
|
||||
PopoverTrigger,
|
||||
PopoverArrow,
|
||||
} from "@/components/ui/popover"
|
||||
import {
|
||||
Tooltip,
|
||||
|
|
@ -58,7 +62,9 @@ import {
|
|||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip"
|
||||
import { cn } from "@/lib/utils"
|
||||
import { isOutOfCredits, CREDIT_EXHAUSTED_EVENT, CREDIT_REPLENISHED_EVENT } from "@/lib/credit-status"
|
||||
import { SettingsDialog } from "@/components/settings-dialog"
|
||||
import { MascotFaceIcon } from "@/components/talking-head"
|
||||
import { extractConferenceLink } from "@/lib/calendar-event"
|
||||
import { useBilling } from "@/hooks/useBilling"
|
||||
import { toast } from "@/lib/toast"
|
||||
|
|
@ -161,6 +167,7 @@ type SidebarContentPanelProps = {
|
|||
onOpenMeetings?: () => void
|
||||
onOpenCode?: () => void
|
||||
onOpenBgTasks?: () => void
|
||||
onOpenApps?: () => void
|
||||
onOpenAgent?: (slug: string) => void
|
||||
recentRuns?: { id: string; title?: string; createdAt: string; modifiedAt?: string }[]
|
||||
onOpenRun?: (runId: string) => void
|
||||
|
|
@ -170,8 +177,10 @@ type SidebarContentPanelProps = {
|
|||
onNewChat?: () => void
|
||||
onToggleBrowser?: () => void
|
||||
onVoiceNoteCreated?: (path: string) => void
|
||||
/** Starts the mascot-guided product tour. */
|
||||
onStartTour?: () => void
|
||||
/** Which primary destination is currently active, for nav highlighting. */
|
||||
activeNav?: 'home' | 'email' | 'meetings' | 'code' | 'knowledge' | 'agents' | 'workspaces' | null
|
||||
activeNav?: 'home' | 'email' | 'meetings' | 'code' | 'knowledge' | 'agents' | 'apps' | 'workspaces' | null
|
||||
/** Live meeting recording state, so the recording row can show its indicator/stop. */
|
||||
meetingRecordingState?: 'idle' | 'connecting' | 'recording' | 'stopping'
|
||||
recordingMeetingSource?: string | null
|
||||
|
|
@ -410,6 +419,7 @@ export function SidebarContentPanel({
|
|||
onOpenMeetings,
|
||||
onOpenCode,
|
||||
onOpenBgTasks,
|
||||
onOpenApps,
|
||||
recentRuns = [],
|
||||
onOpenRun,
|
||||
onOpenChatHistory,
|
||||
|
|
@ -418,6 +428,7 @@ export function SidebarContentPanel({
|
|||
onNewChat,
|
||||
onToggleBrowser,
|
||||
onVoiceNoteCreated,
|
||||
onStartTour,
|
||||
activeNav,
|
||||
meetingRecordingState = 'idle',
|
||||
recordingMeetingSource = null,
|
||||
|
|
@ -430,9 +441,13 @@ export function SidebarContentPanel({
|
|||
const [openConnectionsAfterClose, setOpenConnectionsAfterClose] = useState(false)
|
||||
const connectorsButtonRef = useRef<HTMLButtonElement | null>(null)
|
||||
const [isRowboatConnected, setIsRowboatConnected] = useState(false)
|
||||
const [creditPopoverOpen, setCreditPopoverOpen] = useState(false)
|
||||
const [outOfCredits, setOutOfCredits] = useState(false)
|
||||
const outOfCreditsRef = useRef(false)
|
||||
const creditPopoverAutoShownRef = useRef(false)
|
||||
const [loggingIn, setLoggingIn] = useState(false)
|
||||
const [appUrl, setAppUrl] = useState<string | null>(null)
|
||||
const { billing } = useBilling(isRowboatConnected)
|
||||
const { billing, refresh: refreshBilling } = useBilling(isRowboatConnected)
|
||||
const currentBillingPlan = billing ? getBillingPlanData(billing.catalog, billing.subscriptionPlanId) : null
|
||||
|
||||
// Nav previews: unread important emails + next upcoming meetings (top 2 each).
|
||||
|
|
@ -654,6 +669,50 @@ export function SidebarContentPanel({
|
|||
}
|
||||
}, [])
|
||||
|
||||
// Re-anchor the warning whenever billing (re)loads — billing is authoritative.
|
||||
useEffect(() => {
|
||||
if (billing) {
|
||||
const next = isOutOfCredits(billing)
|
||||
outOfCreditsRef.current = next
|
||||
setOutOfCredits(next)
|
||||
}
|
||||
}, [billing])
|
||||
|
||||
// Live signals: a usage API error flips it on; a successful cost-incurring
|
||||
// call flips it off and triggers a single billing refresh to reconcile.
|
||||
useEffect(() => {
|
||||
const onExhausted = () => {
|
||||
outOfCreditsRef.current = true
|
||||
setOutOfCredits(true)
|
||||
}
|
||||
const onReplenished = () => {
|
||||
const wasOut = outOfCreditsRef.current
|
||||
outOfCreditsRef.current = false
|
||||
setOutOfCredits(false)
|
||||
if (wasOut) void refreshBilling()
|
||||
}
|
||||
window.addEventListener(CREDIT_EXHAUSTED_EVENT, onExhausted)
|
||||
window.addEventListener(CREDIT_REPLENISHED_EVENT, onReplenished)
|
||||
return () => {
|
||||
window.removeEventListener(CREDIT_EXHAUSTED_EVENT, onExhausted)
|
||||
window.removeEventListener(CREDIT_REPLENISHED_EVENT, onReplenished)
|
||||
}
|
||||
}, [refreshBilling])
|
||||
|
||||
// Auto-open the popover the first time we go out of credits; reset when
|
||||
// credits return so it can auto-open again on a future episode.
|
||||
useEffect(() => {
|
||||
if (outOfCredits) {
|
||||
if (!creditPopoverAutoShownRef.current) {
|
||||
creditPopoverAutoShownRef.current = true
|
||||
setCreditPopoverOpen(true)
|
||||
}
|
||||
} else {
|
||||
creditPopoverAutoShownRef.current = false
|
||||
setCreditPopoverOpen(false)
|
||||
}
|
||||
}, [outOfCredits])
|
||||
|
||||
// Single preview shown as a sublabel on the Email / Meetings nav buttons.
|
||||
const previewEmail = emailThreads[0]
|
||||
const previewMeeting = meetings[0]
|
||||
|
|
@ -695,13 +754,14 @@ export function SidebarContentPanel({
|
|||
<SidebarGroupContent>
|
||||
<SidebarMenu>
|
||||
<SidebarMenuItem>
|
||||
<SidebarMenuButton isActive={activeNav === 'home'} onClick={onOpenHome}>
|
||||
<SidebarMenuButton data-tour-id="nav-home" isActive={activeNav === 'home'} onClick={onOpenHome}>
|
||||
<Home className="size-4 shrink-0" />
|
||||
<span className="flex-1 truncate">Home</span>
|
||||
</SidebarMenuButton>
|
||||
</SidebarMenuItem>
|
||||
<SidebarMenuItem>
|
||||
<SidebarMenuButton
|
||||
data-tour-id="nav-email"
|
||||
isActive={activeNav === 'email'}
|
||||
onClick={() => onOpenEmail?.()}
|
||||
className={previewEmail ? 'h-auto py-1.5' : undefined}
|
||||
|
|
@ -724,6 +784,7 @@ export function SidebarContentPanel({
|
|||
</SidebarMenuItem>
|
||||
<SidebarMenuItem>
|
||||
<SidebarMenuButton
|
||||
data-tour-id="nav-meetings"
|
||||
isActive={activeNav === 'meetings'}
|
||||
onClick={onOpenMeetings}
|
||||
className={meetingSublabel ? 'h-auto py-1.5' : undefined}
|
||||
|
|
@ -802,7 +863,7 @@ export function SidebarContentPanel({
|
|||
</SidebarMenuItem>
|
||||
{codeModeEnabled && (
|
||||
<SidebarMenuItem>
|
||||
<SidebarMenuButton isActive={activeNav === 'code'} onClick={onOpenCode}>
|
||||
<SidebarMenuButton data-tour-id="nav-code" isActive={activeNav === 'code'} onClick={onOpenCode}>
|
||||
<Code2 className="size-4 shrink-0" />
|
||||
<span className="flex-1 truncate">Code</span>
|
||||
</SidebarMenuButton>
|
||||
|
|
@ -810,6 +871,7 @@ export function SidebarContentPanel({
|
|||
)}
|
||||
<SidebarMenuItem>
|
||||
<SidebarMenuButton
|
||||
data-tour-id="nav-knowledge"
|
||||
isActive={activeNav === 'knowledge'}
|
||||
onClick={() => knowledgeActions.openKnowledgeView()}
|
||||
className={knowledgeUpdatedLabel ? 'h-auto py-1.5' : undefined}
|
||||
|
|
@ -830,6 +892,7 @@ export function SidebarContentPanel({
|
|||
<SidebarMenu>
|
||||
<SidebarMenuItem>
|
||||
<SidebarMenuButton
|
||||
data-tour-id="nav-agents"
|
||||
isActive={activeNav === 'agents'}
|
||||
onClick={onOpenBgTasks}
|
||||
className={bgAgentsLabel ? 'h-auto py-1.5' : undefined}
|
||||
|
|
@ -850,6 +913,16 @@ export function SidebarContentPanel({
|
|||
</SidebarMenuItem>
|
||||
<SidebarMenuItem>
|
||||
<SidebarMenuButton
|
||||
isActive={activeNav === 'apps'}
|
||||
onClick={onOpenApps}
|
||||
>
|
||||
<LayoutGrid className="size-4 shrink-0 text-muted-foreground" />
|
||||
<span className="flex-1 truncate text-muted-foreground">Apps</span>
|
||||
</SidebarMenuButton>
|
||||
</SidebarMenuItem>
|
||||
<SidebarMenuItem>
|
||||
<SidebarMenuButton
|
||||
data-tour-id="nav-workspaces"
|
||||
isActive={activeNav === 'workspaces'}
|
||||
onClick={() => knowledgeActions.openWorkspaceAt()}
|
||||
className="h-auto py-1.5"
|
||||
|
|
@ -874,6 +947,7 @@ export function SidebarContentPanel({
|
|||
<SidebarGroupContent>
|
||||
<button
|
||||
type="button"
|
||||
data-tour-id="nav-chats"
|
||||
onClick={() => setChatsExpanded((v) => !v)}
|
||||
className="flex w-full items-center gap-1.5 px-3 py-1 text-[10.5px] font-semibold uppercase tracking-wider text-muted-foreground"
|
||||
>
|
||||
|
|
@ -913,31 +987,89 @@ export function SidebarContentPanel({
|
|||
</SidebarGroup>
|
||||
</SidebarContent>
|
||||
{/* Billing / upgrade CTA or Log in CTA */}
|
||||
{isRowboatConnected && billing ? (
|
||||
<div className="px-3 py-2">
|
||||
<div className="flex items-center justify-between rounded-lg border border-sidebar-border bg-sidebar-accent/20 px-3 py-2">
|
||||
<div className="min-w-0">
|
||||
<span className="text-xs font-medium capitalize text-sidebar-foreground">
|
||||
{currentBillingPlan?.displayName ?? (billing.subscriptionPlanId ? 'Unknown' : 'No plan')}
|
||||
</span>
|
||||
{billing.subscriptionStatus === 'trialing' && billing.trialExpiresAt && (() => {
|
||||
const days = Math.max(0, Math.ceil((new Date(billing.trialExpiresAt).getTime() - Date.now()) / (1000 * 60 * 60 * 24)))
|
||||
return (
|
||||
<p className="text-[10px] text-sidebar-foreground/60">
|
||||
{days === 0 ? 'Trial expires today' : days === 1 ? '1 day left' : `${days} days left`}
|
||||
{isRowboatConnected && billing ? (() => {
|
||||
const upgradeLabel = !billing.subscriptionPlanId || currentBillingPlan?.category === 'free' || currentBillingPlan?.category === 'starter' ? 'Upgrade' : 'Manage'
|
||||
if (outOfCredits) {
|
||||
return (
|
||||
<div className="px-3 py-2">
|
||||
<Popover open={creditPopoverOpen} onOpenChange={setCreditPopoverOpen}>
|
||||
<div className="flex items-center justify-between rounded-lg border border-red-500/50 bg-red-500/10 px-3 py-2">
|
||||
<PopoverTrigger asChild>
|
||||
<button type="button" className="flex min-w-0 flex-1 items-center gap-2 text-left">
|
||||
<AlertTriangle className="size-4 shrink-0 text-red-500" />
|
||||
<div className="min-w-0">
|
||||
<span className="text-xs font-medium capitalize text-sidebar-foreground">
|
||||
{currentBillingPlan?.displayName ?? (billing.subscriptionPlanId ? 'Unknown' : 'No plan')}
|
||||
</span>
|
||||
<p className="text-[10px] text-red-500">Out of credits</p>
|
||||
</div>
|
||||
</button>
|
||||
</PopoverTrigger>
|
||||
<button
|
||||
onClick={() => appUrl && window.open(`${appUrl}?intent=upgrade`)}
|
||||
className="shrink-0 rounded-md bg-sidebar-foreground/10 px-2.5 py-1 text-[11px] font-medium text-sidebar-foreground transition-colors hover:bg-sidebar-foreground/20"
|
||||
>
|
||||
{upgradeLabel}
|
||||
</button>
|
||||
</div>
|
||||
<PopoverContent side="top" align="start" sideOffset={10} className="w-72">
|
||||
<PopoverArrow className="fill-popover" />
|
||||
<div className="flex items-start justify-between gap-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="flex size-8 shrink-0 items-center justify-center rounded-md bg-red-500/15 text-red-500">
|
||||
<CircleAlert className="size-4" />
|
||||
</span>
|
||||
<h4 className="text-sm font-bold text-foreground">You've run out of credits</h4>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Close"
|
||||
onClick={() => setCreditPopoverOpen(false)}
|
||||
className="rounded-md p-0.5 text-muted-foreground hover:bg-accent hover:text-foreground"
|
||||
>
|
||||
<X className="size-4" />
|
||||
</button>
|
||||
</div>
|
||||
<p className="mt-2 text-xs text-muted-foreground">
|
||||
Upgrade your plan to continue using all features.
|
||||
</p>
|
||||
)
|
||||
})()}
|
||||
<button
|
||||
onClick={() => { appUrl && window.open(`${appUrl}?intent=upgrade`); setCreditPopoverOpen(false) }}
|
||||
className="mt-3 w-full rounded-md bg-red-500 px-3 py-1.5 text-xs font-medium text-white transition-colors hover:bg-red-600"
|
||||
>
|
||||
Upgrade now
|
||||
</button>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
return (
|
||||
<div className="px-3 py-2">
|
||||
<div className="flex items-center justify-between rounded-lg border border-sidebar-border bg-sidebar-accent/20 px-3 py-2">
|
||||
<div className="min-w-0">
|
||||
<span className="text-xs font-medium capitalize text-sidebar-foreground">
|
||||
{currentBillingPlan?.displayName ?? (billing.subscriptionPlanId ? 'Unknown' : 'No plan')}
|
||||
</span>
|
||||
{billing.subscriptionStatus === 'trialing' && billing.trialExpiresAt && (() => {
|
||||
const days = Math.max(0, Math.ceil((new Date(billing.trialExpiresAt).getTime() - Date.now()) / (1000 * 60 * 60 * 24)))
|
||||
return (
|
||||
<p className="text-[10px] text-sidebar-foreground/60">
|
||||
{days === 0 ? 'Trial expires today' : days === 1 ? '1 day left' : `${days} days left`}
|
||||
</p>
|
||||
)
|
||||
})()}
|
||||
</div>
|
||||
<button
|
||||
onClick={() => appUrl && window.open(`${appUrl}?intent=upgrade`)}
|
||||
className="shrink-0 rounded-md bg-sidebar-foreground/10 px-2.5 py-1 text-[11px] font-medium text-sidebar-foreground transition-colors hover:bg-sidebar-foreground/20"
|
||||
>
|
||||
{upgradeLabel}
|
||||
</button>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => appUrl && window.open(`${appUrl}?intent=upgrade`)}
|
||||
className="shrink-0 rounded-md bg-sidebar-foreground/10 px-2.5 py-1 text-[11px] font-medium text-sidebar-foreground transition-colors hover:bg-sidebar-foreground/20"
|
||||
>
|
||||
{!billing.subscriptionPlanId || currentBillingPlan?.category === 'free' || currentBillingPlan?.category === 'starter' ? 'Upgrade' : 'Manage'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
)
|
||||
})() : null}
|
||||
{/* Sign in CTA */}
|
||||
{!isRowboatConnected && (
|
||||
<div className="px-3 py-2">
|
||||
|
|
@ -1015,6 +1147,15 @@ export function SidebarContentPanel({
|
|||
</AlertDialog>
|
||||
)}
|
||||
</div>
|
||||
{onStartTour && (
|
||||
<button
|
||||
onClick={onStartTour}
|
||||
className="flex w-full items-center gap-2 rounded-md px-2 py-1 text-xs text-sidebar-foreground/70 hover:bg-sidebar-accent hover:text-sidebar-accent-foreground transition-colors"
|
||||
>
|
||||
<MascotFaceIcon className="size-4" />
|
||||
<span>Take a tour</span>
|
||||
</button>
|
||||
)}
|
||||
<SettingsDialog>
|
||||
<button className="flex w-full items-center gap-2 rounded-md px-2 py-1 text-xs text-sidebar-foreground/70 hover:bg-sidebar-accent hover:text-sidebar-accent-foreground transition-colors">
|
||||
<Settings className="size-4" />
|
||||
|
|
|
|||
487
apps/x/apps/renderer/src/components/talking-head.tsx
Normal file
487
apps/x/apps/renderer/src/components/talking-head.tsx
Normal file
|
|
@ -0,0 +1,487 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { X } from 'lucide-react'
|
||||
import type { TTSState } from '@/hooks/useVoiceTTS'
|
||||
import { cn } from '@/lib/utils'
|
||||
|
||||
const POSITION_STORAGE_KEY = 'talking-head-position'
|
||||
|
||||
// Must match the overlay's `bottom-28 right-8` anchor classes.
|
||||
const ANCHOR_RIGHT_PX = 32
|
||||
const ANCHOR_BOTTOM_PX = 112
|
||||
const VIEWPORT_MARGIN_PX = 8
|
||||
|
||||
// Palette pulled from the mascot artwork: pale lavender body, dark walnut boat.
|
||||
const BODY_FILL = '#E8E9F5'
|
||||
const BODY_STROKE = '#17171B'
|
||||
const CHEEK_FILL = '#F2B8BE'
|
||||
const BOAT_DARK = '#3E2E24'
|
||||
const BOAT_MID = '#54402F'
|
||||
const BOAT_LIGHT = '#6B5138'
|
||||
const MOUTH_FILL = '#2A1E19'
|
||||
|
||||
export type MascotHat =
|
||||
| 'mailcap'
|
||||
| 'headphones'
|
||||
| 'hardhat'
|
||||
| 'gradcap'
|
||||
| 'captain'
|
||||
| 'explorer'
|
||||
| 'party'
|
||||
|
||||
type TalkingHeadProps = {
|
||||
ttsState: TTSState
|
||||
getLevel: () => number
|
||||
size?: number
|
||||
/** Costume piece drawn on the head (used by the product tour). */
|
||||
hat?: MascotHat
|
||||
/** Paddle hard: fast bobbing + oar strokes, e.g. while traveling in the tour. */
|
||||
rowing?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* The Rowboat mascot as an animated inline SVG: a round pale character sitting
|
||||
* in a wooden rowboat holding an oar. The mouth is driven every animation
|
||||
* frame from the live TTS audio level; eyes blink on a randomized timer.
|
||||
*/
|
||||
export function TalkingHead({ ttsState, getLevel, size = 160, hat, rowing = false }: TalkingHeadProps) {
|
||||
const mouthOpenRef = useRef<SVGEllipseElement>(null)
|
||||
const mouthSmileRef = useRef<SVGPathElement>(null)
|
||||
const oarRef = useRef<SVGGElement>(null)
|
||||
const smoothedRef = useRef(0)
|
||||
const [blinking, setBlinking] = useState(false)
|
||||
|
||||
const speaking = ttsState === 'speaking'
|
||||
const thinking = ttsState === 'synthesizing'
|
||||
|
||||
// Lip sync + oar paddle loop. Writes SVG attributes directly to avoid
|
||||
// re-rendering React at 60fps. Stops itself once speech has ended and the
|
||||
// mouth has settled closed; restarts when `speaking` flips this effect.
|
||||
useEffect(() => {
|
||||
let raf = 0
|
||||
let t = 0
|
||||
const tick = () => {
|
||||
const target = speaking ? getLevel() : 0
|
||||
const prev = smoothedRef.current
|
||||
// Fast attack, slower decay reads as natural mouth movement
|
||||
const smoothed = target > prev ? prev + (target - prev) * 0.5 : prev + (target - prev) * 0.2
|
||||
const settled = !speaking && !rowing && smoothed < 0.005
|
||||
smoothedRef.current = settled ? 0 : smoothed
|
||||
const open = settled ? 0 : Math.min(1, smoothed * 1.6)
|
||||
|
||||
const mouthOpen = mouthOpenRef.current
|
||||
const mouthSmile = mouthSmileRef.current
|
||||
if (mouthOpen && mouthSmile) {
|
||||
if (open > 0.06) {
|
||||
mouthOpen.setAttribute('rx', String(6.5 + open * 4))
|
||||
mouthOpen.setAttribute('ry', String(1.5 + open * 9))
|
||||
mouthOpen.style.opacity = '1'
|
||||
mouthSmile.style.opacity = '0'
|
||||
} else {
|
||||
mouthOpen.style.opacity = '0'
|
||||
mouthSmile.style.opacity = '1'
|
||||
}
|
||||
}
|
||||
|
||||
const oar = oarRef.current
|
||||
if (oar) {
|
||||
if (rowing) {
|
||||
t += 0.14
|
||||
const angle = Math.sin(t) * 13
|
||||
oar.setAttribute('transform', `rotate(${angle.toFixed(2)} 128 118)`)
|
||||
} else if (speaking) {
|
||||
t += 0.045
|
||||
const angle = Math.sin(t) * 7
|
||||
oar.setAttribute('transform', `rotate(${angle.toFixed(2)} 128 118)`)
|
||||
} else {
|
||||
oar.setAttribute('transform', 'rotate(0 128 118)')
|
||||
}
|
||||
}
|
||||
|
||||
if (!settled) {
|
||||
raf = requestAnimationFrame(tick)
|
||||
}
|
||||
}
|
||||
raf = requestAnimationFrame(tick)
|
||||
return () => cancelAnimationFrame(raf)
|
||||
}, [speaking, rowing, getLevel])
|
||||
|
||||
// Randomized blinking
|
||||
useEffect(() => {
|
||||
let timeout: ReturnType<typeof setTimeout>
|
||||
let cancelled = false
|
||||
const scheduleBlink = () => {
|
||||
timeout = setTimeout(() => {
|
||||
if (cancelled) return
|
||||
setBlinking(true)
|
||||
setTimeout(() => {
|
||||
if (cancelled) return
|
||||
setBlinking(false)
|
||||
scheduleBlink()
|
||||
}, 140)
|
||||
}, 2400 + Math.random() * 2600)
|
||||
}
|
||||
scheduleBlink()
|
||||
return () => {
|
||||
cancelled = true
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<div
|
||||
className="talking-head-bob relative select-none"
|
||||
style={{
|
||||
width: size,
|
||||
height: size,
|
||||
animationDuration: rowing ? '0.8s' : speaking ? '1.6s' : '3.2s',
|
||||
}}
|
||||
>
|
||||
<style>{`
|
||||
@keyframes talking-head-bob {
|
||||
0%, 100% { transform: translateY(0) rotate(-1.6deg); }
|
||||
50% { transform: translateY(-4px) rotate(1.6deg); }
|
||||
}
|
||||
.talking-head-bob {
|
||||
animation-name: talking-head-bob;
|
||||
animation-timing-function: ease-in-out;
|
||||
animation-iteration-count: infinite;
|
||||
}
|
||||
@keyframes talking-head-ripple {
|
||||
0% { transform: scale(0.6); opacity: 0.5; }
|
||||
100% { transform: scale(1.25); opacity: 0; }
|
||||
}
|
||||
.talking-head-ripple {
|
||||
transform-origin: center;
|
||||
transform-box: fill-box;
|
||||
animation: talking-head-ripple 2.6s ease-out infinite;
|
||||
}
|
||||
@keyframes talking-head-bubble {
|
||||
0%, 100% { opacity: 0.25; transform: translateY(0); }
|
||||
50% { opacity: 1; transform: translateY(-2px); }
|
||||
}
|
||||
.talking-head-bubble {
|
||||
animation: talking-head-bubble 1.2s ease-in-out infinite;
|
||||
}
|
||||
`}</style>
|
||||
<svg viewBox="0 0 200 190" width={size} height={size} aria-hidden="true">
|
||||
{/* water ripples under the boat */}
|
||||
<g>
|
||||
<ellipse className="talking-head-ripple" cx="100" cy="168" rx="62" ry="9" fill="none" stroke="#8FB6D9" strokeWidth="2" style={{ animationDelay: '0s' }} />
|
||||
<ellipse className="talking-head-ripple" cx="100" cy="168" rx="62" ry="9" fill="none" stroke="#8FB6D9" strokeWidth="2" style={{ animationDelay: '1.3s' }} />
|
||||
<ellipse cx="100" cy="168" rx="52" ry="7" fill="#8FB6D9" opacity="0.18" />
|
||||
</g>
|
||||
|
||||
{/* thinking bubbles while synthesizing */}
|
||||
{thinking && (
|
||||
<g fill={BODY_STROKE} opacity="0.75">
|
||||
<circle className="talking-head-bubble" cx="146" cy="34" r="3" style={{ animationDelay: '0s' }} />
|
||||
<circle className="talking-head-bubble" cx="157" cy="26" r="4.2" style={{ animationDelay: '0.2s' }} />
|
||||
<circle className="talking-head-bubble" cx="170" cy="16" r="5.4" style={{ animationDelay: '0.4s' }} />
|
||||
</g>
|
||||
)}
|
||||
|
||||
{/* character: head + body blob */}
|
||||
<g>
|
||||
<path
|
||||
d="M 100 22
|
||||
C 129 22 148 43 148 68
|
||||
C 148 82 141 93 131 100
|
||||
C 141 107 147 117 148 128
|
||||
L 52 128
|
||||
C 53 115 60 105 69 99
|
||||
C 59 92 52 81 52 68
|
||||
C 52 43 71 22 100 22 Z"
|
||||
fill={BODY_FILL}
|
||||
stroke={BODY_STROKE}
|
||||
strokeWidth="5"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
{/* eyes */}
|
||||
<g style={{ transform: thinking ? 'translateY(-2.5px)' : undefined, transition: 'transform 0.3s' }}>
|
||||
<ellipse
|
||||
cx="84" cy="64" rx="5" ry={blinking ? 0.8 : 7}
|
||||
fill={BODY_STROKE}
|
||||
style={{ transition: 'ry 0.06s' }}
|
||||
/>
|
||||
<ellipse
|
||||
cx="116" cy="64" rx="5" ry={blinking ? 0.8 : 7}
|
||||
fill={BODY_STROKE}
|
||||
style={{ transition: 'ry 0.06s' }}
|
||||
/>
|
||||
<circle cx="86" cy="61" r="1.6" fill="#FFFFFF" opacity={blinking ? 0 : 0.9} />
|
||||
<circle cx="118" cy="61" r="1.6" fill="#FFFFFF" opacity={blinking ? 0 : 0.9} />
|
||||
</g>
|
||||
{/* cheeks */}
|
||||
<ellipse cx="72" cy="76" rx="6.5" ry="4" fill={CHEEK_FILL} opacity="0.85" />
|
||||
<ellipse cx="128" cy="76" rx="6.5" ry="4" fill={CHEEK_FILL} opacity="0.85" />
|
||||
{/* mouth: smile when quiet, open ellipse driven by audio level */}
|
||||
<path
|
||||
ref={mouthSmileRef}
|
||||
d="M 91 80 Q 100 88 109 80"
|
||||
fill="none"
|
||||
stroke={BODY_STROKE}
|
||||
strokeWidth="4"
|
||||
strokeLinecap="round"
|
||||
/>
|
||||
<ellipse
|
||||
ref={mouthOpenRef}
|
||||
cx="100" cy="84" rx="7" ry="2"
|
||||
fill={MOUTH_FILL}
|
||||
stroke={BODY_STROKE}
|
||||
strokeWidth="3"
|
||||
style={{ opacity: 0 }}
|
||||
/>
|
||||
{hat && <MascotHatArt hat={hat} />}
|
||||
</g>
|
||||
|
||||
{/* oar (rotates while speaking) */}
|
||||
<g ref={oarRef}>
|
||||
<line x1="158" y1="88" x2="88" y2="152" stroke={BODY_STROKE} strokeWidth="12" strokeLinecap="round" />
|
||||
<line x1="158" y1="88" x2="88" y2="152" stroke={BOAT_MID} strokeWidth="7" strokeLinecap="round" />
|
||||
<path
|
||||
d="M 84 148 L 56 170 C 52 173 52 178 57 178 L 90 165 Z"
|
||||
fill={BOAT_DARK}
|
||||
stroke={BODY_STROKE}
|
||||
strokeWidth="4"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
</g>
|
||||
|
||||
{/* hand resting over the oar */}
|
||||
<ellipse cx="121" cy="120" rx="10" ry="8" fill={BODY_FILL} stroke={BODY_STROKE} strokeWidth="4" />
|
||||
|
||||
{/* boat hull (drawn last so it overlaps the body) */}
|
||||
<g>
|
||||
<path
|
||||
d="M 30 120
|
||||
C 50 132 150 132 170 120
|
||||
C 168 142 152 160 100 160
|
||||
C 48 160 32 142 30 120 Z"
|
||||
fill={BOAT_MID}
|
||||
stroke={BODY_STROKE}
|
||||
strokeWidth="5"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
{/* plank lines */}
|
||||
<path d="M 36 133 C 60 143 140 143 164 133" fill="none" stroke={BOAT_DARK} strokeWidth="3" strokeLinecap="round" />
|
||||
<path d="M 44 145 C 66 153 134 153 156 145" fill="none" stroke={BOAT_DARK} strokeWidth="3" strokeLinecap="round" />
|
||||
{/* gunwale highlight */}
|
||||
<path d="M 33 121 C 52 131 148 131 167 121" fill="none" stroke={BOAT_LIGHT} strokeWidth="4" strokeLinecap="round" />
|
||||
</g>
|
||||
</svg>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
type TalkingHeadOverlayProps = {
|
||||
ttsState: TTSState
|
||||
getLevel: () => number
|
||||
onDismiss?: () => void
|
||||
}
|
||||
|
||||
// Keep the widget fully on-screen relative to its bottom-right CSS anchor.
|
||||
// Falls back to the default render size when the element isn't mounted yet.
|
||||
function clampPositionToViewport(pos: { x: number; y: number }, el: HTMLDivElement | null): { x: number; y: number } {
|
||||
const width = el?.offsetWidth ?? 160
|
||||
const height = el?.offsetHeight ?? 160
|
||||
const baseLeft = window.innerWidth - ANCHOR_RIGHT_PX - width
|
||||
const baseTop = window.innerHeight - ANCHOR_BOTTOM_PX - height
|
||||
const clamp = (v: number, min: number, max: number) => Math.max(min, Math.min(max, v))
|
||||
return {
|
||||
x: clamp(pos.x, VIEWPORT_MARGIN_PX - baseLeft, window.innerWidth - VIEWPORT_MARGIN_PX - width - baseLeft),
|
||||
y: clamp(pos.y, VIEWPORT_MARGIN_PX - baseTop, window.innerHeight - VIEWPORT_MARGIN_PX - height - baseTop),
|
||||
}
|
||||
}
|
||||
|
||||
function loadStoredPosition(): { x: number; y: number } {
|
||||
try {
|
||||
const raw = localStorage.getItem(POSITION_STORAGE_KEY)
|
||||
if (raw) {
|
||||
const parsed = JSON.parse(raw)
|
||||
if (typeof parsed?.x === 'number' && typeof parsed?.y === 'number') {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore corrupt stored position
|
||||
}
|
||||
return { x: 0, y: 0 }
|
||||
}
|
||||
|
||||
/**
|
||||
* Floating, draggable widget that hosts the talking head. Anchored to the
|
||||
* bottom-right of the window (above the composer) and offset by a persisted
|
||||
* drag position, so it hovers over whatever view is active.
|
||||
*/
|
||||
export function TalkingHeadOverlay({ ttsState, getLevel, onDismiss }: TalkingHeadOverlayProps) {
|
||||
// Clamp the stored offset at init so a stale position (e.g. saved on a
|
||||
// bigger window) can't leave the widget stranded off-screen.
|
||||
const [offset, setOffset] = useState(() => clampPositionToViewport(loadStoredPosition(), null))
|
||||
const [dragging, setDragging] = useState(false)
|
||||
const dragStartRef = useRef<{ pointerX: number; pointerY: number; x: number; y: number } | null>(null)
|
||||
const containerRef = useRef<HTMLDivElement>(null)
|
||||
|
||||
const clampToViewport = useCallback(
|
||||
(pos: { x: number; y: number }) => clampPositionToViewport(pos, containerRef.current),
|
||||
[]
|
||||
)
|
||||
|
||||
// Re-clamp when the window shrinks
|
||||
useEffect(() => {
|
||||
const handleResize = () => setOffset(prev => clampToViewport(prev))
|
||||
window.addEventListener('resize', handleResize)
|
||||
return () => window.removeEventListener('resize', handleResize)
|
||||
}, [clampToViewport])
|
||||
|
||||
const handlePointerDown = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
|
||||
if (e.button !== 0) return
|
||||
dragStartRef.current = { pointerX: e.clientX, pointerY: e.clientY, x: offset.x, y: offset.y }
|
||||
setDragging(true)
|
||||
e.currentTarget.setPointerCapture(e.pointerId)
|
||||
}, [offset])
|
||||
|
||||
const handlePointerMove = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
|
||||
const start = dragStartRef.current
|
||||
if (!start) return
|
||||
setOffset({
|
||||
x: start.x + (e.clientX - start.pointerX),
|
||||
y: start.y + (e.clientY - start.pointerY),
|
||||
})
|
||||
}, [])
|
||||
|
||||
const handlePointerUp = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
|
||||
if (!dragStartRef.current) return
|
||||
dragStartRef.current = null
|
||||
setDragging(false)
|
||||
setOffset(prev => clampToViewport(prev))
|
||||
e.currentTarget.releasePointerCapture(e.pointerId)
|
||||
}, [clampToViewport])
|
||||
|
||||
useEffect(() => {
|
||||
if (dragging) return
|
||||
try {
|
||||
localStorage.setItem(POSITION_STORAGE_KEY, JSON.stringify(offset))
|
||||
} catch {
|
||||
// best-effort persistence
|
||||
}
|
||||
}, [offset, dragging])
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={containerRef}
|
||||
className={cn(
|
||||
'group fixed bottom-28 right-8 z-50 touch-none',
|
||||
dragging ? 'cursor-grabbing' : 'cursor-grab'
|
||||
)}
|
||||
style={{
|
||||
transform: `translate(${offset.x}px, ${offset.y}px)`,
|
||||
// Constant value so the entrance animation runs once on mount and
|
||||
// never restarts (re-applying it after a drag would replay the pop).
|
||||
animation: 'talking-head-pop 0.35s cubic-bezier(0.34, 1.56, 0.64, 1)',
|
||||
}}
|
||||
onPointerDown={handlePointerDown}
|
||||
onPointerMove={handlePointerMove}
|
||||
onPointerUp={handlePointerUp}
|
||||
onPointerCancel={handlePointerUp}
|
||||
role="img"
|
||||
aria-label="Rowboat talking head"
|
||||
>
|
||||
<style>{`
|
||||
@keyframes talking-head-pop {
|
||||
0% { opacity: 0; scale: 0.4; }
|
||||
100% { opacity: 1; scale: 1; }
|
||||
}
|
||||
`}</style>
|
||||
{onDismiss && (
|
||||
<button
|
||||
type="button"
|
||||
onPointerDown={(e) => e.stopPropagation()}
|
||||
onClick={onDismiss}
|
||||
className="absolute -right-1 -top-1 z-10 flex h-5 w-5 items-center justify-center rounded-full border border-border bg-background text-muted-foreground opacity-0 shadow-sm transition-opacity hover:text-foreground group-hover:opacity-100"
|
||||
aria-label="Hide talking head"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
<TalkingHead ttsState={ttsState} getLevel={getLevel} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** Costume pieces for the product tour, drawn on the mascot's head. */
|
||||
function MascotHatArt({ hat }: { hat: MascotHat }) {
|
||||
const outline = { stroke: BODY_STROKE, strokeWidth: 4, strokeLinejoin: 'round' as const }
|
||||
switch (hat) {
|
||||
case 'mailcap':
|
||||
return (
|
||||
<g>
|
||||
<path d="M 68 36 Q 100 4 132 36 Q 100 26 68 36 Z" fill="#4A7DDB" {...outline} />
|
||||
<path d="M 126 33 Q 148 31 150 39 Q 132 42 124 39 Z" fill="#3D68B8" {...outline} />
|
||||
</g>
|
||||
)
|
||||
case 'headphones':
|
||||
return (
|
||||
<g>
|
||||
<path d="M 64 58 Q 100 2 136 58" fill="none" stroke={BODY_STROKE} strokeWidth="11" strokeLinecap="round" />
|
||||
<path d="M 64 58 Q 100 2 136 58" fill="none" stroke="#4B5563" strokeWidth="6" strokeLinecap="round" />
|
||||
<ellipse cx="64" cy="61" rx="8" ry="11" fill="#4B5563" {...outline} />
|
||||
<ellipse cx="136" cy="61" rx="8" ry="11" fill="#4B5563" {...outline} />
|
||||
</g>
|
||||
)
|
||||
case 'hardhat':
|
||||
return (
|
||||
<g>
|
||||
<path d="M 66 38 Q 100 0 134 38 Z" fill="#F6C445" {...outline} />
|
||||
<path d="M 96 10 Q 94 22 96 36 L 106 36 Q 107 22 105 9 Z" fill="#E0AC2C" stroke="none" />
|
||||
<path d="M 56 38 L 144 38" fill="none" stroke={BODY_STROKE} strokeWidth="9" strokeLinecap="round" />
|
||||
<path d="M 58 38 L 142 38" fill="none" stroke="#F6C445" strokeWidth="5" strokeLinecap="round" />
|
||||
</g>
|
||||
)
|
||||
case 'gradcap':
|
||||
return (
|
||||
<g>
|
||||
<path d="M 80 28 Q 100 42 120 28 L 120 38 Q 100 50 80 38 Z" fill="#232838" {...outline} />
|
||||
<path d="M 100 2 L 144 20 L 100 38 L 56 20 Z" fill="#2E3450" {...outline} />
|
||||
<path d="M 100 20 Q 124 26 136 34" fill="none" stroke="#E8B94A" strokeWidth="3" strokeLinecap="round" />
|
||||
<circle cx="137" cy="38" r="4" fill="#E8B94A" stroke={BODY_STROKE} strokeWidth="3" />
|
||||
</g>
|
||||
)
|
||||
case 'captain':
|
||||
return (
|
||||
<g>
|
||||
<path d="M 68 36 Q 100 -2 132 36 Q 100 28 68 36 Z" fill="#F4F5F9" {...outline} />
|
||||
<path d="M 66 36 Q 100 46 134 36 L 134 42 Q 100 52 66 42 Z" fill="#22262E" {...outline} />
|
||||
<path d="M 122 42 Q 146 42 148 48 Q 130 52 120 48 Z" fill="#22262E" {...outline} />
|
||||
<circle cx="100" cy="38" r="4.5" fill="#E8B94A" stroke={BODY_STROKE} strokeWidth="3" />
|
||||
</g>
|
||||
)
|
||||
case 'explorer':
|
||||
return (
|
||||
<g>
|
||||
<ellipse cx="100" cy="35" rx="46" ry="9" fill="#C9A46B" {...outline} />
|
||||
<path d="M 72 34 Q 100 4 128 34 Z" fill="#C9A46B" {...outline} />
|
||||
<path d="M 76 30 Q 100 38 124 30" fill="none" stroke="#8A6B3D" strokeWidth="4" strokeLinecap="round" />
|
||||
</g>
|
||||
)
|
||||
case 'party':
|
||||
return (
|
||||
<g>
|
||||
<path d="M 100 -2 L 84 34 L 116 34 Z" fill="#F2699C" {...outline} />
|
||||
<path d="M 92 16 L 111 22 M 88 26 L 114 31" fill="none" stroke="#FFD166" strokeWidth="3.5" strokeLinecap="round" />
|
||||
<circle cx="100" cy="0" r="5" fill="#FFD166" stroke={BODY_STROKE} strokeWidth="3" />
|
||||
</g>
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** Small static mascot face used as the toolbar toggle icon. */
|
||||
export function MascotFaceIcon({ className }: { className?: string }) {
|
||||
return (
|
||||
<svg viewBox="0 0 24 24" width="16" height="16" className={className} aria-hidden="true">
|
||||
<circle cx="12" cy="12" r="9.5" fill="none" stroke="currentColor" strokeWidth="1.8" />
|
||||
<ellipse cx="8.6" cy="10.5" rx="1.3" ry="1.8" fill="currentColor" />
|
||||
<ellipse cx="15.4" cy="10.5" rx="1.3" ry="1.8" fill="currentColor" />
|
||||
<path d="M 9 14.5 Q 12 17 15 14.5" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" />
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
262
apps/x/apps/renderer/src/components/tour-vignettes.tsx
Normal file
262
apps/x/apps/renderer/src/components/tour-vignettes.tsx
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
import { useEffect } from 'react'
|
||||
|
||||
export type MascotVignetteKind = 'email' | 'meetings' | 'brain'
|
||||
export type TourVignetteKind = MascotVignetteKind | 'agents'
|
||||
|
||||
/**
|
||||
* Little looping "shows" staged around the mascot while it presents a section
|
||||
* during the product tour. Purely decorative: everything is pointer-events-none
|
||||
* and rendered on the tour's own layers, never inside the section's real UI.
|
||||
*/
|
||||
export function MascotVignette({ kind, playDing }: { kind: MascotVignetteKind; playDing?: () => void }) {
|
||||
// One round of dings as the first envelopes land, then let the loop run silent
|
||||
useEffect(() => {
|
||||
if (kind !== 'email' || !playDing) return
|
||||
const timers = [900, 1900, 2900].map((ms) => setTimeout(playDing, ms))
|
||||
return () => timers.forEach(clearTimeout)
|
||||
}, [kind, playDing])
|
||||
|
||||
return (
|
||||
<div className="pointer-events-none absolute left-1/2 top-0 z-0 -translate-x-1/2" aria-hidden="true">
|
||||
<style>{`
|
||||
@keyframes tour-env-left {
|
||||
0% { opacity: 0; transform: translate(-150px, -130px) rotate(-26deg); }
|
||||
10% { opacity: 1; }
|
||||
52% { opacity: 1; transform: translate(0px, 0px) rotate(5deg); }
|
||||
64% { opacity: 0; transform: translate(2px, 12px) rotate(5deg); }
|
||||
100% { opacity: 0; transform: translate(2px, 12px) rotate(5deg); }
|
||||
}
|
||||
@keyframes tour-env-right {
|
||||
0% { opacity: 0; transform: translate(150px, -140px) rotate(26deg); }
|
||||
10% { opacity: 1; }
|
||||
52% { opacity: 1; transform: translate(0px, 0px) rotate(-5deg); }
|
||||
64% { opacity: 0; transform: translate(-2px, 12px) rotate(-5deg); }
|
||||
100% { opacity: 0; transform: translate(-2px, 12px) rotate(-5deg); }
|
||||
}
|
||||
@keyframes tour-badge-pulse {
|
||||
0%, 100% { transform: scale(1); }
|
||||
50% { transform: scale(1.15); }
|
||||
}
|
||||
@keyframes tour-note-line {
|
||||
0% { width: 0; }
|
||||
18% { width: var(--w); }
|
||||
80% { width: var(--w); opacity: 1; }
|
||||
92%, 100% { width: var(--w); opacity: 0; }
|
||||
}
|
||||
@keyframes tour-quill-bob {
|
||||
0% { transform: translate(4px, 26px) rotate(-8deg); }
|
||||
25% { transform: translate(46px, 30px) rotate(4deg); }
|
||||
50% { transform: translate(6px, 40px) rotate(-8deg); }
|
||||
75% { transform: translate(48px, 44px) rotate(4deg); }
|
||||
100% { transform: translate(4px, 26px) rotate(-8deg); }
|
||||
}
|
||||
@keyframes tour-voice-bar {
|
||||
0%, 100% { transform: scaleY(0.35); }
|
||||
50% { transform: scaleY(1); }
|
||||
}
|
||||
@keyframes tour-orb {
|
||||
0% { opacity: 0; transform: translate(var(--from-x), var(--from-y)) scale(0.5); }
|
||||
15% { opacity: 0.9; }
|
||||
70% { opacity: 0.9; transform: translate(0, 0) scale(1); }
|
||||
85%, 100% { opacity: 0; transform: translate(0, 6px) scale(0.3); }
|
||||
}
|
||||
@keyframes tour-node-pulse {
|
||||
0%, 100% { opacity: 0.55; }
|
||||
50% { opacity: 1; }
|
||||
}
|
||||
@keyframes tour-edge-draw {
|
||||
from { stroke-dashoffset: 60; }
|
||||
to { stroke-dashoffset: 0; }
|
||||
}
|
||||
`}</style>
|
||||
|
||||
{kind === 'email' && (
|
||||
<div className="relative" style={{ width: 240, height: 150 }}>
|
||||
{[
|
||||
{ anim: 'tour-env-left', delay: 0, x: 78, y: 96 },
|
||||
{ anim: 'tour-env-right', delay: 1.0, x: 128, y: 102 },
|
||||
{ anim: 'tour-env-left', delay: 2.0, x: 104, y: 92 },
|
||||
{ anim: 'tour-env-right', delay: 3.0, x: 90, y: 104 },
|
||||
].map((e, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className="absolute"
|
||||
style={{ left: e.x, top: e.y, animation: `${e.anim} 4s ease-in-out ${e.delay}s infinite both` }}
|
||||
>
|
||||
<svg width="34" height="24" viewBox="0 0 34 24">
|
||||
<rect x="1.5" y="1.5" width="31" height="21" rx="3" fill="#FFF8E7" stroke="#17171B" strokeWidth="2.5" />
|
||||
<path d="M 2 4 L 17 14 L 32 4" fill="none" stroke="#17171B" strokeWidth="2.5" strokeLinejoin="round" />
|
||||
</svg>
|
||||
</div>
|
||||
))}
|
||||
<div
|
||||
className="absolute"
|
||||
style={{ left: 148, top: 78, animation: 'tour-badge-pulse 2s ease-in-out infinite' }}
|
||||
>
|
||||
<svg width="22" height="22" viewBox="0 0 22 22">
|
||||
<circle cx="11" cy="11" r="9.5" fill="#3FA95C" stroke="#17171B" strokeWidth="2.5" />
|
||||
<path d="M 6.5 11.5 L 9.5 14.5 L 15.5 8" fill="none" stroke="#FFFFFF" strokeWidth="2.5" strokeLinecap="round" strokeLinejoin="round" />
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{kind === 'meetings' && (
|
||||
<div className="relative" style={{ width: 220, height: 160, top: -110 }}>
|
||||
{/* someone's talking */}
|
||||
<div className="absolute left-1/2 flex -translate-x-1/2 items-end gap-1" style={{ top: 0, height: 26 }}>
|
||||
{[0.9, 1.1, 0.7, 1.3, 0.8].map((dur, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className="w-1.5 origin-bottom rounded-full bg-[#8FB6D9]"
|
||||
style={{ height: 22, animation: `tour-voice-bar ${dur}s ease-in-out ${i * 0.12}s infinite` }}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
{/* ...and the notepad writes itself */}
|
||||
<div
|
||||
className="absolute left-1/2 rounded-md border-2 border-[#17171B] bg-[#FFFDF6] shadow-md"
|
||||
style={{ top: 36, width: 96, height: 104, transform: 'translateX(-50%) rotate(-3deg)' }}
|
||||
>
|
||||
<div className="mx-2 mt-2 h-1.5 rounded bg-[#D9534F]/70" />
|
||||
{[64, 52, 60, 44].map((w, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className="ml-2 mt-3 h-1.5 rounded bg-[#9AA1AE]"
|
||||
style={{ ['--w' as string]: `${w}px`, animation: `tour-note-line 5s ease-out ${i * 1.1}s infinite both` }}
|
||||
/>
|
||||
))}
|
||||
<svg
|
||||
className="absolute left-0 top-0"
|
||||
width="26"
|
||||
height="26"
|
||||
viewBox="0 0 26 26"
|
||||
style={{ animation: 'tour-quill-bob 5s ease-in-out infinite' }}
|
||||
>
|
||||
<path d="M 4 22 L 10 12 Q 14 4 22 2 Q 18 10 12 14 Z" fill="#6B5138" stroke="#17171B" strokeWidth="2" strokeLinejoin="round" />
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{kind === 'brain' && (
|
||||
<div className="relative" style={{ width: 260, height: 180, top: -128 }}>
|
||||
{/* constellation assembling above the head */}
|
||||
<svg className="absolute left-1/2 -translate-x-1/2" width="140" height="80" viewBox="0 0 140 80" style={{ top: 0 }}>
|
||||
{[
|
||||
'M 22 58 L 52 24',
|
||||
'M 52 24 L 88 40',
|
||||
'M 88 40 L 120 18',
|
||||
'M 88 40 L 70 66',
|
||||
].map((d, i) => (
|
||||
<path
|
||||
key={i}
|
||||
d={d}
|
||||
fill="none"
|
||||
stroke="#9CCBEA"
|
||||
strokeWidth="2"
|
||||
strokeDasharray="60"
|
||||
style={{ animation: `tour-edge-draw 1.2s ease-out ${0.4 + i * 0.35}s both` }}
|
||||
/>
|
||||
))}
|
||||
{[
|
||||
[22, 58], [52, 24], [88, 40], [120, 18], [70, 66],
|
||||
].map(([cx, cy], i) => (
|
||||
<circle
|
||||
key={i}
|
||||
cx={cx}
|
||||
cy={cy}
|
||||
r={5}
|
||||
fill="#BFE0F5"
|
||||
stroke="#17171B"
|
||||
strokeWidth="2"
|
||||
style={{ animation: `tour-node-pulse 2.4s ease-in-out ${i * 0.3}s infinite` }}
|
||||
/>
|
||||
))}
|
||||
</svg>
|
||||
{/* thought-orbs drifting into the head */}
|
||||
{[
|
||||
{ fx: '-120px', fy: '-30px', delay: 0 },
|
||||
{ fx: '120px', fy: '-40px', delay: 1.1 },
|
||||
{ fx: '-90px', fy: '50px', delay: 2.2 },
|
||||
{ fx: '110px', fy: '46px', delay: 3.3 },
|
||||
].map((o, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className="absolute rounded-full"
|
||||
style={{
|
||||
left: 122,
|
||||
top: 118,
|
||||
width: 16,
|
||||
height: 16,
|
||||
background: 'radial-gradient(circle, #FFF3B8 20%, #FFD166 60%, transparent 75%)',
|
||||
['--from-x' as string]: o.fx,
|
||||
['--from-y' as string]: o.fy,
|
||||
animation: `tour-orb 4.4s ease-in-out ${o.delay}s infinite both`,
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Background-agents vignette: a fleet of tiny mascots rowing across the water
|
||||
* at the bottom of the screen while the big one takes a break.
|
||||
*/
|
||||
export function AgentsFleet() {
|
||||
return (
|
||||
<div className="pointer-events-none fixed inset-x-0 bottom-0 z-[67] h-28 overflow-hidden" aria-hidden="true">
|
||||
<style>{`
|
||||
@keyframes tour-fleet-right {
|
||||
from { transform: translateX(-18vw); }
|
||||
to { transform: translateX(112vw); }
|
||||
}
|
||||
@keyframes tour-fleet-left {
|
||||
from { transform: translateX(112vw); }
|
||||
to { transform: translateX(-18vw); }
|
||||
}
|
||||
@keyframes tour-fleet-bob {
|
||||
0%, 100% { transform: translateY(0) rotate(-2deg); }
|
||||
50% { transform: translateY(-3px) rotate(2deg); }
|
||||
}
|
||||
`}</style>
|
||||
{[
|
||||
{ size: 46, bottom: 4, dur: 16, delay: 0, dir: 'tour-fleet-right' },
|
||||
{ size: 34, bottom: 22, dur: 22, delay: -8, dir: 'tour-fleet-left' },
|
||||
{ size: 28, bottom: 14, dur: 27, delay: -3, dir: 'tour-fleet-right' },
|
||||
].map((b, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className="absolute left-0"
|
||||
style={{ bottom: b.bottom, animation: `${b.dir} ${b.dur}s linear ${b.delay}s infinite` }}
|
||||
>
|
||||
<div style={{ animation: 'tour-fleet-bob 1.4s ease-in-out infinite', transform: b.dir === 'tour-fleet-left' ? 'scaleX(-1)' : undefined }}>
|
||||
<MiniRower size={b.size} flipped={b.dir === 'tour-fleet-left'} />
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function MiniRower({ size, flipped }: { size: number; flipped: boolean }) {
|
||||
return (
|
||||
<svg
|
||||
width={size}
|
||||
height={size * 0.75}
|
||||
viewBox="0 0 60 45"
|
||||
style={{ transform: flipped ? 'scaleX(-1)' : undefined }}
|
||||
>
|
||||
<circle cx="27" cy="13" r="10" fill="#E8E9F5" stroke="#17171B" strokeWidth="3" />
|
||||
<circle cx="23.5" cy="11.5" r="1.4" fill="#17171B" />
|
||||
<circle cx="30.5" cy="11.5" r="1.4" fill="#17171B" />
|
||||
<path d="M 23 16 Q 27 19 31 16" fill="none" stroke="#17171B" strokeWidth="1.6" strokeLinecap="round" />
|
||||
<line x1="40" y1="14" x2="20" y2="38" stroke="#17171B" strokeWidth="4.5" strokeLinecap="round" />
|
||||
<line x1="40" y1="14" x2="20" y2="38" stroke="#54402F" strokeWidth="2.5" strokeLinecap="round" />
|
||||
<path d="M 7 25 C 17 31 43 31 53 25 C 51 37 43 42 30 42 C 17 42 9 37 7 25 Z" fill="#54402F" stroke="#17171B" strokeWidth="3" strokeLinejoin="round" />
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
|
@ -43,4 +43,17 @@ function PopoverAnchor({
|
|||
return <PopoverPrimitive.Anchor data-slot="popover-anchor" {...props} />
|
||||
}
|
||||
|
||||
export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor }
|
||||
function PopoverArrow({
|
||||
className,
|
||||
...props
|
||||
}: React.ComponentProps<typeof PopoverPrimitive.Arrow>) {
|
||||
return (
|
||||
<PopoverPrimitive.Arrow
|
||||
data-slot="popover-arrow"
|
||||
className={cn("fill-popover", className)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor, PopoverArrow }
|
||||
|
|
|
|||
274
apps/x/apps/renderer/src/components/video-call-view.tsx
Normal file
274
apps/x/apps/renderer/src/components/video-call-view.tsx
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
import { useEffect, useRef, useState } from 'react'
|
||||
import { Mic, MicOff, Minimize2, MonitorUp, PhoneOff, Presentation, Square, User, Video, VideoOff } from 'lucide-react'
|
||||
|
||||
import { MascotFaceIcon, TalkingHead } from '@/components/talking-head'
|
||||
import type { TTSState } from '@/hooks/useVoiceTTS'
|
||||
import { cn } from '@/lib/utils'
|
||||
|
||||
export type VideoCallStatus = 'listening' | 'thinking' | 'speaking'
|
||||
|
||||
interface VideoCallViewProps {
|
||||
/** Live camera stream from useVideoMode — attached to the user's tile. */
|
||||
streamRef: React.MutableRefObject<MediaStream | null>
|
||||
cameraOn: boolean
|
||||
onToggleCamera: () => void
|
||||
/** User mute = full input pause: no mic audio AND no frame capture. */
|
||||
micMuted: boolean
|
||||
onToggleMic: () => void
|
||||
/** Starting a share collapses this view into the floating popout (the
|
||||
* surface is derived from devices — see App.tsx). */
|
||||
onToggleScreenShare: () => void
|
||||
/** Practice preset: the assistant is coaching this session. */
|
||||
practiceMode?: boolean
|
||||
/** Shrink to the floating pill without touching any devices. */
|
||||
onMinimize: () => void
|
||||
/** Stop the assistant: silence speech and abort the run if still going. */
|
||||
onInterrupt: () => void
|
||||
ttsState: TTSState
|
||||
/** Live TTS output level — drives the mascot's mouth animation. */
|
||||
getTtsLevel: () => number
|
||||
status: VideoCallStatus
|
||||
/** Live transcript of the user's in-progress utterance. */
|
||||
interimText?: string
|
||||
/** The assistant line currently being spoken aloud. */
|
||||
assistantCaption?: string
|
||||
onLeave: () => void
|
||||
}
|
||||
|
||||
const STATUS_DISPLAY: Record<VideoCallStatus, { label: string; dotClass: string }> = {
|
||||
listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' },
|
||||
thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' },
|
||||
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-screen call surface: a Meet-style two-tile layout with the user's
|
||||
* webcam on one side and the mascot as the other participant. Shown only
|
||||
* while the camera is on with no screen share (the derived-surface rule in
|
||||
* App.tsx) — sharing or muting the camera moves the call into the floating
|
||||
* popout. The mascot animates with the assistant's speech; dismissing it
|
||||
* swaps in a Meet-style letter avatar ("R"). Live captions run along the
|
||||
* bottom.
|
||||
*/
|
||||
export function VideoCallView({
|
||||
streamRef,
|
||||
cameraOn,
|
||||
onToggleCamera,
|
||||
micMuted,
|
||||
onToggleMic,
|
||||
onToggleScreenShare,
|
||||
practiceMode,
|
||||
onMinimize,
|
||||
onInterrupt,
|
||||
ttsState,
|
||||
getTtsLevel,
|
||||
status,
|
||||
interimText,
|
||||
assistantCaption,
|
||||
onLeave,
|
||||
}: VideoCallViewProps) {
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
const [mascotVisible, setMascotVisible] = useState(true)
|
||||
|
||||
useEffect(() => {
|
||||
if (!cameraOn) return
|
||||
const videoEl = videoRef.current
|
||||
if (!videoEl) return
|
||||
videoEl.srcObject = streamRef.current
|
||||
videoEl.play().catch(() => {})
|
||||
return () => {
|
||||
videoEl.srcObject = null
|
||||
}
|
||||
}, [streamRef, cameraOn])
|
||||
|
||||
const userSpeaking = status === 'listening' && Boolean(interimText)
|
||||
const assistantSpeaking = ttsState === 'speaking'
|
||||
|
||||
const caption = assistantSpeaking && assistantCaption
|
||||
? { who: 'Rowboat', text: assistantCaption }
|
||||
: interimText
|
||||
? { who: 'You', text: interimText }
|
||||
: null
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-[100] flex flex-col bg-neutral-950">
|
||||
{practiceMode && (
|
||||
<span className="absolute left-4 top-4 z-10 flex items-center gap-1.5 rounded-full bg-violet-600/90 px-3 py-1 text-xs font-medium text-white">
|
||||
<Presentation className="h-3.5 w-3.5" />
|
||||
Practice session
|
||||
</span>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={onMinimize}
|
||||
className="absolute right-4 top-4 z-10 flex h-8 w-8 items-center justify-center rounded-full bg-neutral-800 text-white/80 transition-colors hover:bg-neutral-700 hover:text-white"
|
||||
aria-label="Minimize call (shares your screen)"
|
||||
title="Minimize — shares your screen so it can help you work"
|
||||
>
|
||||
<Minimize2 className="h-4 w-4" />
|
||||
</button>
|
||||
|
||||
{/* Participant tiles */}
|
||||
<div className="grid min-h-0 flex-1 grid-cols-1 gap-3 p-4 pb-2 md:grid-cols-2">
|
||||
{/* User */}
|
||||
<div
|
||||
className={cn(
|
||||
'relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
|
||||
userSpeaking && 'ring-2 ring-green-500/80'
|
||||
)}
|
||||
>
|
||||
{cameraOn ? (
|
||||
<video
|
||||
ref={videoRef}
|
||||
muted
|
||||
playsInline
|
||||
className="h-full w-full object-cover"
|
||||
style={{ transform: 'scaleX(-1)' }}
|
||||
/>
|
||||
) : (
|
||||
<span className="flex h-40 w-40 items-center justify-center rounded-full bg-neutral-700 text-neutral-400" aria-label="Camera off">
|
||||
<User className="h-20 w-20" />
|
||||
</span>
|
||||
)}
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
You
|
||||
</span>
|
||||
{micMuted && (
|
||||
<span className="absolute bottom-3 right-3 flex items-center gap-1.5 rounded-md bg-red-600/90 px-2 py-0.5 text-sm font-medium text-white">
|
||||
<MicOff className="h-3.5 w-3.5" />
|
||||
Muted — nothing is heard or captured
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Assistant */}
|
||||
<div
|
||||
className={cn(
|
||||
'group relative flex items-center justify-center overflow-hidden rounded-2xl bg-neutral-900 transition-shadow',
|
||||
assistantSpeaking && 'ring-2 ring-sky-400/80'
|
||||
)}
|
||||
>
|
||||
{mascotVisible ? (
|
||||
<TalkingHead ttsState={ttsState} getLevel={getTtsLevel} size={220} />
|
||||
) : (
|
||||
<span
|
||||
className="flex h-40 w-40 items-center justify-center rounded-full bg-sky-600 text-7xl font-medium text-white"
|
||||
aria-label="Rowboat"
|
||||
>
|
||||
R
|
||||
</span>
|
||||
)}
|
||||
<span className="absolute bottom-3 left-3 rounded-md bg-black/50 px-2 py-0.5 text-sm text-white">
|
||||
Rowboat
|
||||
</span>
|
||||
{status !== 'listening' && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onInterrupt}
|
||||
className="absolute bottom-3 right-3 flex items-center gap-1.5 rounded-md bg-red-600/90 px-2.5 py-1 text-sm font-medium text-white transition-colors hover:bg-red-500"
|
||||
aria-label="Stop the assistant"
|
||||
title={status === 'speaking' ? 'Stop speaking' : 'Stop responding'}
|
||||
>
|
||||
<Square className="h-3 w-3 fill-current" />
|
||||
Stop
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMascotVisible((v) => !v)}
|
||||
className="absolute right-3 top-3 rounded-md bg-black/50 px-2 py-1 text-xs text-white/80 opacity-0 transition-opacity hover:text-white group-hover:opacity-100"
|
||||
>
|
||||
{mascotVisible ? 'Hide mascot' : 'Show mascot'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Captions */}
|
||||
<div className="flex h-14 items-center justify-center px-6">
|
||||
{caption && (
|
||||
<div className="max-w-3xl truncate rounded-lg bg-black/60 px-4 py-2 text-sm text-white/90">
|
||||
<span className="mr-2 font-semibold text-white">{caption.who}:</span>
|
||||
{caption.text}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Control bar */}
|
||||
<div className="flex items-center justify-center gap-4 pb-5">
|
||||
<span className="flex items-center gap-2 rounded-full bg-neutral-800 px-3 py-1.5 text-xs font-medium text-white/90">
|
||||
{/* Muted overrides "Listening" — the green pulse would be a lie.
|
||||
Thinking/speaking still show: output continues while muted. */}
|
||||
{micMuted && status === 'listening' ? (
|
||||
<>
|
||||
<span className="block h-2 w-2 rounded-full bg-red-500" />
|
||||
Muted
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<span className={cn('block h-2 w-2 rounded-full', STATUS_DISPLAY[status].dotClass)} />
|
||||
{STATUS_DISPLAY[status].label}
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleMic}
|
||||
className={cn(
|
||||
'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
|
||||
micMuted
|
||||
? 'bg-red-600 text-white hover:bg-red-500'
|
||||
: 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
|
||||
)}
|
||||
aria-label={micMuted ? 'Unmute' : 'Mute (pauses mic and frame capture)'}
|
||||
title={micMuted ? 'Unmute' : 'Mute — pauses your mic and all frame capture'}
|
||||
>
|
||||
{micMuted ? <MicOff className="h-5 w-5" /> : <Mic className="h-5 w-5" />}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleCamera}
|
||||
className={cn(
|
||||
'flex h-10 w-10 items-center justify-center rounded-full transition-colors',
|
||||
cameraOn
|
||||
? 'bg-neutral-800 text-white/90 hover:bg-neutral-700'
|
||||
: 'bg-red-600 text-white hover:bg-red-500'
|
||||
)}
|
||||
aria-label={cameraOn ? 'Turn off camera' : 'Turn on camera'}
|
||||
title={cameraOn ? 'Turn off camera' : 'Turn on camera'}
|
||||
>
|
||||
{cameraOn ? <Video className="h-5 w-5" /> : <VideoOff className="h-5 w-5" />}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleScreenShare}
|
||||
className="flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700"
|
||||
aria-label="Present your screen"
|
||||
title="Present your screen"
|
||||
>
|
||||
<MonitorUp className="h-5 w-5" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setMascotVisible((v) => !v)}
|
||||
className="relative flex h-10 w-10 items-center justify-center rounded-full bg-neutral-800 text-white/90 transition-colors hover:bg-neutral-700"
|
||||
aria-label={mascotVisible ? 'Hide mascot' : 'Show mascot'}
|
||||
>
|
||||
<MascotFaceIcon />
|
||||
{!mascotVisible && (
|
||||
<span className="absolute inset-0 flex items-center justify-center pointer-events-none">
|
||||
<span className="block h-[1.5px] w-6 -rotate-45 rounded-full bg-white/80" />
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onLeave}
|
||||
className="flex h-10 w-14 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
|
||||
aria-label="End call"
|
||||
>
|
||||
<PhoneOff className="h-5 w-5" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
239
apps/x/apps/renderer/src/components/video-popout.tsx
Normal file
239
apps/x/apps/renderer/src/components/video-popout.tsx
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { Maximize2, Mic, MicOff, MonitorUp, PhoneOff, Square, User, Video, VideoOff } from 'lucide-react'
|
||||
|
||||
import { TalkingHead } from '@/components/talking-head'
|
||||
|
||||
type PopoutState = {
|
||||
ttsState: 'idle' | 'synthesizing' | 'speaking'
|
||||
status: 'listening' | 'thinking' | 'speaking' | null
|
||||
cameraOn: boolean
|
||||
/** User mute = full input pause: no mic audio AND no frame capture. */
|
||||
micMuted: boolean
|
||||
screenSharing: boolean
|
||||
interimText: string | null
|
||||
}
|
||||
|
||||
const STATUS_DISPLAY: Record<NonNullable<PopoutState['status']>, { label: string; dotClass: string }> = {
|
||||
listening: { label: 'Listening', dotClass: 'bg-green-500 animate-pulse' },
|
||||
thinking: { label: 'Thinking…', dotClass: 'bg-amber-400' },
|
||||
speaking: { label: 'Speaking', dotClass: 'bg-sky-400 animate-pulse' },
|
||||
}
|
||||
|
||||
const dragRegion = { WebkitAppRegion: 'drag' } as React.CSSProperties
|
||||
const noDragRegion = { WebkitAppRegion: 'no-drag' } as React.CSSProperties
|
||||
|
||||
/**
|
||||
* Content of the always-on-top popout window shown for the whole duration of
|
||||
* a screen share (Meet-style floating mini-call) — it floats over every app,
|
||||
* including Rowboat itself, and is the call's control surface while sharing:
|
||||
* camera toggle, share toggle, end-call. Rendered in its own BrowserWindow
|
||||
* (see `video:setPopout` in the main process); call state streams in over
|
||||
* the `video:popout-state` push channel and control actions round-trip back
|
||||
* through `video:popoutAction`. Captures its own webcam feed — MediaStreams
|
||||
* can't cross windows.
|
||||
*/
|
||||
export function VideoPopout() {
|
||||
// Camera defaults OFF: guessing "on" would flash the user's video for a
|
||||
// beat before the real state arrives — which reads as a bug. The true
|
||||
// state is fetched immediately below.
|
||||
const [state, setState] = useState<PopoutState>({ ttsState: 'idle', status: null, cameraOn: false, micMuted: false, screenSharing: false, interimText: null })
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
const cleanup = window.ipc.on('video:popout-state', (next) => setState(next))
|
||||
// The main process replays the cached state on did-finish-load, but that
|
||||
// can race this listener's registration — fetch it explicitly too.
|
||||
window.ipc
|
||||
.invoke('video:getPopoutState', null)
|
||||
.then(({ state: cached }) => {
|
||||
if (cached) setState(cached)
|
||||
})
|
||||
.catch(() => {})
|
||||
return cleanup
|
||||
}, [])
|
||||
|
||||
// Own camera feed, following the main window's camera-on/off state.
|
||||
useEffect(() => {
|
||||
if (!state.cameraOn) return
|
||||
let stream: MediaStream | null = null
|
||||
let cancelled = false
|
||||
navigator.mediaDevices
|
||||
.getUserMedia({ video: { width: { ideal: 640 }, facingMode: 'user' }, audio: false })
|
||||
.then((s) => {
|
||||
if (cancelled) {
|
||||
s.getTracks().forEach((t) => t.stop())
|
||||
return
|
||||
}
|
||||
stream = s
|
||||
if (videoRef.current) {
|
||||
videoRef.current.srcObject = s
|
||||
videoRef.current.play().catch(() => {})
|
||||
}
|
||||
})
|
||||
.catch((err) => console.error('[popout] camera failed:', err))
|
||||
return () => {
|
||||
cancelled = true
|
||||
stream?.getTracks().forEach((t) => t.stop())
|
||||
if (videoRef.current) videoRef.current.srcObject = null
|
||||
}
|
||||
}, [state.cameraOn])
|
||||
|
||||
// The popout has no TTS audio pipeline — synthesize a plausible mouth level
|
||||
// so the mascot still animates while the assistant speaks in the main window.
|
||||
const getLevel = useCallback(() => 0.45 + 0.35 * Math.sin(performance.now() / 90), [])
|
||||
|
||||
const sendAction = useCallback((action: 'toggle-mic' | 'toggle-camera' | 'toggle-share' | 'stop-speaking' | 'end-call' | 'expand') => {
|
||||
void window.ipc.invoke('video:popoutAction', { action }).catch(() => {})
|
||||
}, [])
|
||||
|
||||
const statusDisplay = state.status ? STATUS_DISPLAY[state.status] : null
|
||||
|
||||
return (
|
||||
<div
|
||||
className="relative flex h-screen w-screen select-none flex-col gap-1.5 bg-neutral-900 p-1.5"
|
||||
style={dragRegion}
|
||||
>
|
||||
<div className="flex min-h-0 flex-1 gap-1.5">
|
||||
<div className="relative flex-1 overflow-hidden rounded-lg bg-neutral-800">
|
||||
{state.cameraOn ? (
|
||||
<video
|
||||
ref={videoRef}
|
||||
muted
|
||||
playsInline
|
||||
className="h-full w-full object-cover"
|
||||
style={{ transform: 'scaleX(-1)' }}
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center">
|
||||
<span className="flex h-12 w-12 items-center justify-center rounded-full bg-neutral-700 text-neutral-400">
|
||||
<User className="h-6 w-6" />
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
|
||||
You
|
||||
</span>
|
||||
{/* Persistent consent badge — the user must always be able to see
|
||||
at a glance that their screen is going out. Muted pauses frame
|
||||
capture while keeping the share stream open, so say so. */}
|
||||
{state.screenSharing && (
|
||||
<span className="absolute left-1.5 top-1.5 flex items-center gap-1 rounded-full bg-sky-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
|
||||
<span className={`block h-1.5 w-1.5 rounded-full bg-white ${state.micMuted ? '' : 'animate-pulse'}`} />
|
||||
{state.micMuted ? 'Sharing paused' : 'Sharing screen'}
|
||||
</span>
|
||||
)}
|
||||
{state.micMuted && (
|
||||
<span className="absolute bottom-1 right-1.5 flex items-center gap-1 rounded bg-red-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white">
|
||||
<MicOff className="h-2.5 w-2.5" />
|
||||
Muted
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="relative flex flex-1 items-center justify-center overflow-hidden rounded-lg bg-neutral-800">
|
||||
<TalkingHead ttsState={state.ttsState} getLevel={getLevel} size={84} />
|
||||
<span className="absolute bottom-1 left-1.5 rounded bg-black/50 px-1 py-px text-[10px] text-white">
|
||||
Rowboat
|
||||
</span>
|
||||
{statusDisplay && (
|
||||
<span className="absolute right-1.5 top-1.5 flex items-center gap-1 rounded-full bg-black/50 px-1.5 py-0.5 text-[10px] font-medium text-white">
|
||||
{/* Muted overrides "Listening" — the green pulse would be a lie. */}
|
||||
{state.micMuted && state.status === 'listening' ? (
|
||||
<>
|
||||
<span className="block h-1.5 w-1.5 rounded-full bg-red-500" />
|
||||
Muted
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<span className={`block h-1.5 w-1.5 rounded-full ${statusDisplay.dotClass}`} />
|
||||
{statusDisplay.label}
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
)}
|
||||
{(state.status === 'speaking' || state.status === 'thinking') && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('stop-speaking')}
|
||||
className="absolute bottom-1 right-1.5 flex items-center gap-1 rounded bg-red-600/90 px-1.5 py-0.5 text-[10px] font-medium text-white hover:bg-red-500"
|
||||
style={noDragRegion}
|
||||
aria-label="Stop the assistant"
|
||||
title={state.status === 'speaking' ? 'Stop speaking' : 'Stop responding'}
|
||||
>
|
||||
<Square className="h-2.5 w-2.5 fill-current" />
|
||||
Stop
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
{/* Live caption of the in-progress utterance, floating over the tiles */}
|
||||
{state.interimText && (
|
||||
<div className="pointer-events-none absolute inset-x-1.5 bottom-9 flex justify-center">
|
||||
<span className="max-w-full truncate rounded bg-black/70 px-1.5 py-0.5 text-[10px] text-white/90">
|
||||
{state.interimText}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Control bar — actions execute in the main app window */}
|
||||
<div className="flex h-7 shrink-0 items-center justify-center gap-2" style={noDragRegion}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('toggle-mic')}
|
||||
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
|
||||
state.micMuted
|
||||
? 'bg-red-600 text-white hover:bg-red-500'
|
||||
: 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
|
||||
}`}
|
||||
aria-label={state.micMuted ? 'Unmute' : 'Mute (pauses mic and frame capture)'}
|
||||
title={state.micMuted ? 'Unmute' : 'Mute — pauses your mic and all frame capture'}
|
||||
>
|
||||
{state.micMuted ? <MicOff className="h-3.5 w-3.5" /> : <Mic className="h-3.5 w-3.5" />}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('toggle-camera')}
|
||||
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
|
||||
state.cameraOn
|
||||
? 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
|
||||
: 'bg-red-600 text-white hover:bg-red-500'
|
||||
}`}
|
||||
aria-label={state.cameraOn ? 'Turn off camera' : 'Turn on camera'}
|
||||
title={state.cameraOn ? 'Turn off camera' : 'Turn on camera'}
|
||||
>
|
||||
{state.cameraOn ? <Video className="h-3.5 w-3.5" /> : <VideoOff className="h-3.5 w-3.5" />}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('toggle-share')}
|
||||
className={`flex h-6 w-6 items-center justify-center rounded-full transition-colors ${
|
||||
state.screenSharing
|
||||
? 'bg-sky-600 text-white hover:bg-sky-500'
|
||||
: 'bg-neutral-700 text-white/90 hover:bg-neutral-600'
|
||||
}`}
|
||||
aria-label={state.screenSharing ? 'Stop sharing screen' : 'Share your screen'}
|
||||
title={state.screenSharing ? 'Stop sharing screen' : 'Share your screen'}
|
||||
>
|
||||
<MonitorUp className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('end-call')}
|
||||
className="flex h-6 w-8 items-center justify-center rounded-full bg-red-600 text-white transition-colors hover:bg-red-500"
|
||||
aria-label="End call"
|
||||
title="End call"
|
||||
>
|
||||
<PhoneOff className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => sendAction('expand')}
|
||||
className="flex h-6 w-6 items-center justify-center rounded-full bg-neutral-700 text-white/90 transition-colors hover:bg-neutral-600"
|
||||
aria-label="Expand to full screen (stops screen sharing)"
|
||||
title="Expand to full screen (stops sharing)"
|
||||
>
|
||||
<Maximize2 className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
326
apps/x/apps/renderer/src/hooks/useVideoMode.ts
Normal file
326
apps/x/apps/renderer/src/hooks/useVideoMode.ts
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
export type VideoModeState = 'idle' | 'starting' | 'live';
|
||||
export type ScreenShareState = 'idle' | 'starting' | 'live';
|
||||
|
||||
export interface CapturedVideoFrame {
|
||||
/** base64-encoded JPEG bytes (no data: prefix) — shape of the UserImagePart wire format */
|
||||
data: string;
|
||||
mediaType: string;
|
||||
capturedAt: string; // ISO timestamp
|
||||
/** data: URL of the same frame, for direct display in the transcript */
|
||||
dataUrl: string;
|
||||
source: 'camera' | 'screen';
|
||||
}
|
||||
|
||||
// Frames are grabbed once per second — dense enough to catch expression and
|
||||
// posture changes while the user talks. Per message we attach at most
|
||||
// MAX_*_FRAMES_PER_MESSAGE frames, evenly sampled across the window since the
|
||||
// last send, so long monologues don't balloon the request.
|
||||
const CAPTURE_INTERVAL_MS = 1000;
|
||||
const MAX_CAMERA_FRAMES_PER_MESSAGE = 12;
|
||||
// Screen frames are ~4x the resolution (and tokens) of camera frames, and the
|
||||
// latest view matters far more than the trajectory — keep the cap small.
|
||||
const MAX_SCREEN_FRAMES_PER_MESSAGE = 4;
|
||||
// Rolling buffer bound (~2 minutes). The buffer only needs to cover the gap
|
||||
// between two sends; anything older is stale context anyway.
|
||||
const MAX_BUFFERED_FRAMES = 120;
|
||||
// Downscale targets. 512px wide JPEG keeps a webcam frame around 20-40KB —
|
||||
// cheap enough to inline a dozen per message as multimodal image parts.
|
||||
// Screen captures keep 1280px so on-screen text stays legible to the model.
|
||||
const CAMERA_FRAME_WIDTH = 512;
|
||||
const SCREEN_FRAME_WIDTH = 1280;
|
||||
const CAMERA_JPEG_QUALITY = 0.65;
|
||||
const SCREEN_JPEG_QUALITY = 0.7;
|
||||
|
||||
interface BufferedFrame {
|
||||
dataUrl: string;
|
||||
capturedAt: string;
|
||||
ts: number;
|
||||
}
|
||||
|
||||
// One capture pipeline: stream → offscreen <video> → canvas JPEG → ring buffer.
|
||||
interface CapturePipe {
|
||||
stream: MediaStream | null;
|
||||
videoEl: HTMLVideoElement | null;
|
||||
canvas: HTMLCanvasElement | null;
|
||||
interval: ReturnType<typeof setInterval> | null;
|
||||
frames: BufferedFrame[];
|
||||
lastCollectTs: number;
|
||||
}
|
||||
|
||||
const emptyPipe = (): CapturePipe => ({
|
||||
stream: null,
|
||||
videoEl: null,
|
||||
canvas: null,
|
||||
interval: null,
|
||||
frames: [],
|
||||
lastCollectTs: 0,
|
||||
});
|
||||
|
||||
function capturePipeFrame(pipe: CapturePipe, width: number, quality: number) {
|
||||
const videoEl = pipe.videoEl;
|
||||
if (!videoEl || videoEl.readyState < 2 || videoEl.videoWidth === 0) return;
|
||||
if (!pipe.canvas) {
|
||||
pipe.canvas = document.createElement('canvas');
|
||||
}
|
||||
const canvas = pipe.canvas;
|
||||
const scale = Math.min(1, width / videoEl.videoWidth);
|
||||
canvas.width = Math.round(videoEl.videoWidth * scale);
|
||||
canvas.height = Math.round(videoEl.videoHeight * scale);
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
ctx.drawImage(videoEl, 0, 0, canvas.width, canvas.height);
|
||||
const dataUrl = canvas.toDataURL('image/jpeg', quality);
|
||||
// A near-empty data URL means the frame was blank (source still warming up)
|
||||
if (dataUrl.length < 100) return;
|
||||
pipe.frames.push({ dataUrl, capturedAt: new Date().toISOString(), ts: Date.now() });
|
||||
if (pipe.frames.length > MAX_BUFFERED_FRAMES) {
|
||||
pipe.frames.splice(0, pipe.frames.length - MAX_BUFFERED_FRAMES);
|
||||
}
|
||||
}
|
||||
|
||||
function attachPipeSource(pipe: CapturePipe, stream: MediaStream, grab: () => void) {
|
||||
pipe.stream = stream;
|
||||
// Offscreen <video> that feeds the capture canvas; any visible preview
|
||||
// attaches to the same MediaStream separately.
|
||||
const videoEl = document.createElement('video');
|
||||
videoEl.muted = true;
|
||||
videoEl.playsInline = true;
|
||||
videoEl.srcObject = stream;
|
||||
pipe.videoEl = videoEl;
|
||||
videoEl.play().catch(() => {});
|
||||
// First frame as soon as the source delivers data, then steady-state cadence.
|
||||
videoEl.addEventListener('loadeddata', () => grab(), { once: true });
|
||||
pipe.interval = setInterval(grab, CAPTURE_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function teardownPipe(pipe: CapturePipe) {
|
||||
if (pipe.interval) {
|
||||
clearInterval(pipe.interval);
|
||||
pipe.interval = null;
|
||||
}
|
||||
if (pipe.videoEl) {
|
||||
pipe.videoEl.srcObject = null;
|
||||
pipe.videoEl = null;
|
||||
}
|
||||
if (pipe.stream) {
|
||||
pipe.stream.getTracks().forEach((t) => t.stop());
|
||||
pipe.stream = null;
|
||||
}
|
||||
pipe.frames = [];
|
||||
pipe.lastCollectTs = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain frames captured since the previous collection, evenly sampled down to
|
||||
* `max` (always keeping the newest). Falls back to the single most recent
|
||||
* frame when nothing new accumulated (rapid-fire messages), so every message
|
||||
* carries at least one frame once the source has warmed up.
|
||||
*/
|
||||
function drainPipe(pipe: CapturePipe, max: number, source: CapturedVideoFrame['source']): CapturedVideoFrame[] {
|
||||
const all = pipe.frames;
|
||||
if (all.length === 0) return [];
|
||||
|
||||
let window_ = all.filter((f) => f.ts > pipe.lastCollectTs);
|
||||
if (window_.length === 0) {
|
||||
window_ = [all[all.length - 1]];
|
||||
}
|
||||
pipe.lastCollectTs = window_[window_.length - 1].ts;
|
||||
|
||||
let sampled: BufferedFrame[];
|
||||
if (window_.length <= max) {
|
||||
sampled = window_;
|
||||
} else {
|
||||
sampled = [];
|
||||
const step = (window_.length - 1) / (max - 1);
|
||||
for (let i = 0; i < max; i++) {
|
||||
sampled.push(window_[Math.round(i * step)]);
|
||||
}
|
||||
}
|
||||
|
||||
return sampled.map((f) => ({
|
||||
data: f.dataUrl.slice(f.dataUrl.indexOf(',') + 1),
|
||||
mediaType: 'image/jpeg',
|
||||
capturedAt: f.capturedAt,
|
||||
dataUrl: f.dataUrl,
|
||||
source,
|
||||
}));
|
||||
}
|
||||
|
||||
export function useVideoMode() {
|
||||
const [state, setState] = useState<VideoModeState>('idle');
|
||||
const [screenState, setScreenState] = useState<ScreenShareState>('idle');
|
||||
// Camera can be turned off mid-session (Meet-style) while the mode — and
|
||||
// any screen share — keeps running. Resets to on for the next session.
|
||||
const [cameraOn, setCameraOn] = useState(true);
|
||||
// In-call mute pauses capture entirely: nothing lands in the ring buffers
|
||||
// and collectFrames() returns nothing, so a muted stretch can never ride
|
||||
// along with a later message. Streams stay open for instant resume.
|
||||
const capturePausedRef = useRef(false);
|
||||
const cameraPipeRef = useRef<CapturePipe>(emptyPipe());
|
||||
const screenPipeRef = useRef<CapturePipe>(emptyPipe());
|
||||
// Stable stream refs for preview components (<video srcObject>).
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const screenStreamRef = useRef<MediaStream | null>(null);
|
||||
const stateRef = useRef<VideoModeState>('idle');
|
||||
stateRef.current = state;
|
||||
const screenStateRef = useRef<ScreenShareState>('idle');
|
||||
screenStateRef.current = screenState;
|
||||
|
||||
const captureCameraFrame = useCallback(() => {
|
||||
if (capturePausedRef.current) return;
|
||||
capturePipeFrame(cameraPipeRef.current, CAMERA_FRAME_WIDTH, CAMERA_JPEG_QUALITY);
|
||||
}, []);
|
||||
|
||||
const captureScreenFrame = useCallback(() => {
|
||||
if (capturePausedRef.current) return;
|
||||
capturePipeFrame(screenPipeRef.current, SCREEN_FRAME_WIDTH, SCREEN_JPEG_QUALITY);
|
||||
}, []);
|
||||
|
||||
const setCapturePaused = useCallback((paused: boolean) => {
|
||||
capturePausedRef.current = paused;
|
||||
}, []);
|
||||
|
||||
const stopScreenShare = useCallback(() => {
|
||||
teardownPipe(screenPipeRef.current);
|
||||
screenStreamRef.current = null;
|
||||
setScreenState('idle');
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
teardownPipe(cameraPipeRef.current);
|
||||
streamRef.current = null;
|
||||
setState('idle');
|
||||
setCameraOn(true);
|
||||
capturePausedRef.current = false;
|
||||
stopScreenShare();
|
||||
}, [stopScreenShare]);
|
||||
|
||||
// Acquire the webcam and start its capture pipeline. Shared by start()
|
||||
// and by re-enabling the camera mid-session.
|
||||
const acquireCamera = useCallback(async (): Promise<boolean> => {
|
||||
// Settle the macOS TCC camera permission before getUserMedia, same as
|
||||
// voice mode does for the mic — otherwise the first click silently
|
||||
// fails while the native prompt is still up.
|
||||
const access = await window.ipc
|
||||
.invoke('voice:ensureCameraAccess', null)
|
||||
.catch(() => ({ granted: true }));
|
||||
if (!access.granted) {
|
||||
console.error('[video] Camera access denied');
|
||||
return false;
|
||||
}
|
||||
|
||||
let stream: MediaStream | null = null;
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: 'user' },
|
||||
audio: false,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[video] Camera access failed:', err);
|
||||
return false;
|
||||
}
|
||||
|
||||
streamRef.current = stream;
|
||||
attachPipeSource(cameraPipeRef.current, stream, captureCameraFrame);
|
||||
return true;
|
||||
}, [captureCameraFrame]);
|
||||
|
||||
/**
|
||||
* Turn the camera off/on without leaving video mode (Meet-style). While
|
||||
* off, no webcam frames are captured or attached; screen-share frames
|
||||
* (if presenting) keep flowing.
|
||||
*/
|
||||
const setCameraEnabled = useCallback(async (enabled: boolean): Promise<boolean> => {
|
||||
if (stateRef.current !== 'live') return false;
|
||||
if (enabled) {
|
||||
const ok = await acquireCamera();
|
||||
if (ok) setCameraOn(true);
|
||||
return ok;
|
||||
}
|
||||
teardownPipe(cameraPipeRef.current);
|
||||
streamRef.current = null;
|
||||
setCameraOn(false);
|
||||
return true;
|
||||
}, [acquireCamera]);
|
||||
|
||||
/**
|
||||
* Start video mode. `camera: false` starts a camera-less session (voice
|
||||
* call / screen-share-only) — the mode is live so frames can flow from
|
||||
* other sources, and the camera can be enabled later via setCameraEnabled.
|
||||
*/
|
||||
const start = useCallback(async ({ camera = true }: { camera?: boolean } = {}): Promise<boolean> => {
|
||||
if (stateRef.current !== 'idle') return true;
|
||||
setState('starting');
|
||||
if (camera) {
|
||||
const ok = await acquireCamera();
|
||||
if (!ok) {
|
||||
setState('idle');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
setCameraOn(camera);
|
||||
setState('live');
|
||||
return true;
|
||||
}, [acquireCamera]);
|
||||
|
||||
/**
|
||||
* Share the screen. The main process auto-approves getDisplayMedia with
|
||||
* the primary screen (see setDisplayMediaRequestHandler in main.ts), so
|
||||
* no source picker appears. Returns false if capture couldn't start
|
||||
* (usually the macOS Screen Recording permission).
|
||||
*/
|
||||
const startScreenShare = useCallback(async (): Promise<boolean> => {
|
||||
if (screenStateRef.current !== 'idle') return true;
|
||||
setScreenState('starting');
|
||||
|
||||
// Surfaces the macOS Screen Recording permission state and, on first
|
||||
// use, registers the app in System Settings (same flow meetings use).
|
||||
await window.ipc.invoke('meeting:checkScreenPermission', null).catch(() => null);
|
||||
|
||||
let stream: MediaStream | null = null;
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getDisplayMedia({
|
||||
video: { frameRate: { ideal: 5 } },
|
||||
audio: false,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[video] Screen share failed:', err);
|
||||
setScreenState('idle');
|
||||
return false;
|
||||
}
|
||||
|
||||
screenStreamRef.current = stream;
|
||||
// The capture can end outside our UI (display unplugged, OS revokes) —
|
||||
// tear down cleanly so the UI doesn't show a dead share.
|
||||
stream.getVideoTracks()[0]?.addEventListener('ended', () => stopScreenShare(), { once: true });
|
||||
attachPipeSource(screenPipeRef.current, stream, captureScreenFrame);
|
||||
setScreenState('live');
|
||||
return true;
|
||||
}, [captureScreenFrame, stopScreenShare]);
|
||||
|
||||
/**
|
||||
* Drain webcam + screen-share frames buffered since the last send, tagged
|
||||
* by source. Webcam frames come first, then screen frames.
|
||||
*/
|
||||
const collectFrames = useCallback((): CapturedVideoFrame[] => {
|
||||
if (stateRef.current !== 'live') return [];
|
||||
// Muted: no frames at all — not even pre-mute buffered ones — so a
|
||||
// typed message during a mute carries nothing captured around it.
|
||||
if (capturePausedRef.current) return [];
|
||||
// Grab a frame right now so the message always includes the moment of send.
|
||||
captureCameraFrame();
|
||||
const frames = drainPipe(cameraPipeRef.current, MAX_CAMERA_FRAMES_PER_MESSAGE, 'camera');
|
||||
if (screenStateRef.current === 'live') {
|
||||
captureScreenFrame();
|
||||
frames.push(...drainPipe(screenPipeRef.current, MAX_SCREEN_FRAMES_PER_MESSAGE, 'screen'));
|
||||
}
|
||||
return frames;
|
||||
}, [captureCameraFrame, captureScreenFrame]);
|
||||
|
||||
// Release the camera/screen if the component unmounts with video mode on.
|
||||
useEffect(() => stop, [stop]);
|
||||
|
||||
return { state, screenState, cameraOn, streamRef, screenStreamRef, start, stop, startScreenShare, stopScreenShare, setCameraEnabled, setCapturePaused, collectFrames };
|
||||
}
|
||||
|
|
@ -19,7 +19,35 @@ const DEEPGRAM_PARAMS = new URLSearchParams({
|
|||
endpointing: '100',
|
||||
no_delay: 'true',
|
||||
});
|
||||
const DEEPGRAM_LISTEN_URL = `wss://api.deepgram.com/v1/listen?${DEEPGRAM_PARAMS.toString()}`;
|
||||
// Hands-free (continuous) mode: Deepgram's endpoint fires FAST (600ms of
|
||||
// silence) and we apply smart hold logic on our side — if the transcript
|
||||
// already reads as a complete thought (terminal punctuation) the utterance
|
||||
// fires immediately, otherwise we hold INCOMPLETE_HOLD_MS longer in case the
|
||||
// user was mid-thought. Net effect: complete sentences turn around ~1.2s
|
||||
// faster than the old fixed 1800ms endpoint, while thinking pauses still get
|
||||
// the same total grace (~1.8s).
|
||||
const CONTINUOUS_ENDPOINTING_MS = 600;
|
||||
const INCOMPLETE_HOLD_MS = 1200;
|
||||
// While the mic is paused (assistant speaking), keep the idle Deepgram socket
|
||||
// alive — it closes after ~10s without audio otherwise.
|
||||
const KEEPALIVE_INTERVAL_MS = 5000;
|
||||
|
||||
// Deepgram punctuates finals (punctuate=true) — a transcript ending in
|
||||
// terminal punctuation (optionally inside a closing quote/paren) is treated
|
||||
// as a complete thought.
|
||||
const COMPLETE_THOUGHT_RE = /[.!?…]["')\]]*\s*$/;
|
||||
|
||||
function deepgramParams(continuous: boolean): URLSearchParams {
|
||||
if (!continuous) return DEEPGRAM_PARAMS;
|
||||
const params = new URLSearchParams(DEEPGRAM_PARAMS);
|
||||
params.set('endpointing', String(CONTINUOUS_ENDPOINTING_MS));
|
||||
// Second end-of-speech signal: speech_final can be missed (it often rides
|
||||
// on a result with an empty transcript, or never fires when background
|
||||
// noise keeps the endpointer engaged). UtteranceEnd is word-timing based
|
||||
// and arrives as its own message type, so we listen for both.
|
||||
params.set('utterance_end_ms', '1000');
|
||||
return params;
|
||||
}
|
||||
|
||||
// Cap on retained per-frame amplitude samples (~64ms/frame ⇒ ~5 min of history).
|
||||
// The waveform only ever displays the most recent window, so older samples are dropped.
|
||||
|
|
@ -53,6 +81,13 @@ export function useVoiceMode() {
|
|||
const audioLevelsRef = useRef<number[]>([]);
|
||||
// Running peak amplitude for the waveform auto-gain (see PEAK_DECAY/MIN_PEAK).
|
||||
const audioPeakRef = useRef(0);
|
||||
// Hands-free mode: invoked with each completed utterance (speech_final).
|
||||
const continuousCbRef = useRef<((text: string) => void) | null>(null);
|
||||
// While true (assistant is speaking), mic audio is dropped instead of streamed.
|
||||
const pausedRef = useRef(false);
|
||||
const keepAliveTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
// Pending mid-thought hold (smart endpointing) — see maybeEndUtterance.
|
||||
const holdTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
// Refresh cached auth details (called on warmup, not on mic click)
|
||||
const refreshAuth = useCallback(async () => {
|
||||
|
|
@ -71,9 +106,44 @@ export function useVoiceMode() {
|
|||
}
|
||||
}, [refreshRowboatAccount]);
|
||||
|
||||
// Hands-free mode: flush the accumulated utterance to the callback.
|
||||
// Both end-of-speech signals may fire for the same utterance — the second
|
||||
// finds an empty buffer and is a no-op.
|
||||
const fireContinuousUtterance = useCallback(() => {
|
||||
if (holdTimerRef.current) {
|
||||
clearTimeout(holdTimerRef.current);
|
||||
holdTimerRef.current = null;
|
||||
}
|
||||
if (!continuousCbRef.current || pausedRef.current) return;
|
||||
const utterance = transcriptBufferRef.current.trim();
|
||||
transcriptBufferRef.current = '';
|
||||
interimRef.current = '';
|
||||
setInterimText('');
|
||||
if (utterance) continuousCbRef.current(utterance);
|
||||
}, []);
|
||||
|
||||
// Smart endpoint: Deepgram's endpoint fires fast (600ms). If the
|
||||
// transcript reads as a complete thought, hand it off immediately; if it
|
||||
// trails off mid-sentence ("so what I want is…"), hold a little longer —
|
||||
// resumed speech cancels the hold and the utterance keeps growing.
|
||||
const maybeEndUtterance = useCallback(() => {
|
||||
if (!continuousCbRef.current || pausedRef.current) return;
|
||||
const buffered = transcriptBufferRef.current.trim();
|
||||
if (!buffered) return;
|
||||
if (COMPLETE_THOUGHT_RE.test(buffered)) {
|
||||
fireContinuousUtterance();
|
||||
return;
|
||||
}
|
||||
if (holdTimerRef.current) clearTimeout(holdTimerRef.current);
|
||||
holdTimerRef.current = setTimeout(() => {
|
||||
holdTimerRef.current = null;
|
||||
fireContinuousUtterance();
|
||||
}, INCOMPLETE_HOLD_MS);
|
||||
}, [fireContinuousUtterance]);
|
||||
|
||||
// Create and connect a Deepgram WebSocket using cached auth.
|
||||
// Starts the connection and returns immediately (does not wait for open).
|
||||
const connectWs = useCallback(async () => {
|
||||
const connectWs = useCallback(async (continuous = false) => {
|
||||
if (wsRef.current && (wsRef.current.readyState === WebSocket.OPEN || wsRef.current.readyState === WebSocket.CONNECTING)) return;
|
||||
|
||||
// Refresh auth if we don't have it cached yet
|
||||
|
|
@ -82,12 +152,13 @@ export function useVoiceMode() {
|
|||
}
|
||||
if (!cachedAuth) return;
|
||||
|
||||
const params = deepgramParams(continuous);
|
||||
let ws: WebSocket;
|
||||
if (cachedAuth.type === 'rowboat') {
|
||||
const listenUrl = buildDeepgramListenUrl(cachedAuth.url, DEEPGRAM_PARAMS);
|
||||
const listenUrl = buildDeepgramListenUrl(cachedAuth.url, params);
|
||||
ws = new WebSocket(listenUrl, ['bearer', cachedAuth.token]);
|
||||
} else {
|
||||
ws = new WebSocket(DEEPGRAM_LISTEN_URL, ['token', cachedAuth.apiKey]);
|
||||
ws = new WebSocket(`wss://api.deepgram.com/v1/listen?${params.toString()}`, ['token', cachedAuth.apiKey]);
|
||||
}
|
||||
wsRef.current = ws;
|
||||
|
||||
|
|
@ -103,16 +174,43 @@ export function useVoiceMode() {
|
|||
|
||||
ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
if (!data.channel?.alternatives?.[0]) return;
|
||||
|
||||
// Hands-free mode: word-timing based end-of-speech marker.
|
||||
if (data.type === 'UtteranceEnd') {
|
||||
maybeEndUtterance();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!data.channel?.alternatives?.[0]) return;
|
||||
const transcript = data.channel.alternatives[0].transcript;
|
||||
if (!transcript) return;
|
||||
|
||||
// The user resumed speaking — cancel any pending mid-thought hold
|
||||
// so the utterance keeps growing instead of firing under them.
|
||||
if (transcript && holdTimerRef.current) {
|
||||
clearTimeout(holdTimerRef.current);
|
||||
holdTimerRef.current = null;
|
||||
}
|
||||
|
||||
if (data.is_final) {
|
||||
transcriptBufferRef.current += (transcriptBufferRef.current ? ' ' : '') + transcript;
|
||||
interimRef.current = '';
|
||||
setInterimText(transcriptBufferRef.current);
|
||||
} else {
|
||||
// NOTE: the endpoint marker (speech_final) usually arrives on a
|
||||
// result whose transcript is EMPTY — the silence after the user
|
||||
// stops talking. Empty finals must still reach the speech_final
|
||||
// check below or hands-free utterances never complete.
|
||||
if (transcript) {
|
||||
transcriptBufferRef.current += (transcriptBufferRef.current ? ' ' : '') + transcript;
|
||||
interimRef.current = '';
|
||||
}
|
||||
// Hands-free mode: an endpoint may complete the utterance —
|
||||
// immediately for complete thoughts, after a short hold for
|
||||
// mid-sentence trails.
|
||||
if (continuousCbRef.current && data.speech_final) {
|
||||
maybeEndUtterance();
|
||||
return;
|
||||
}
|
||||
if (transcript) {
|
||||
setInterimText(transcriptBufferRef.current);
|
||||
}
|
||||
} else if (transcript) {
|
||||
interimRef.current = transcript;
|
||||
setInterimText(transcriptBufferRef.current + (transcriptBufferRef.current ? ' ' : '') + transcript);
|
||||
}
|
||||
|
|
@ -127,8 +225,17 @@ export function useVoiceMode() {
|
|||
ws.onclose = () => {
|
||||
console.log('[voice] WebSocket closed');
|
||||
wsRef.current = null;
|
||||
// A hands-free call is long-lived — if the socket drops while the
|
||||
// call is still on, reconnect instead of silently going deaf.
|
||||
if (continuousCbRef.current) {
|
||||
setTimeout(() => {
|
||||
if (continuousCbRef.current && !wsRef.current) {
|
||||
void connectWs(true);
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
};
|
||||
}, [refreshAuth]);
|
||||
}, [refreshAuth, maybeEndUtterance]);
|
||||
|
||||
const waitForWsOpen = useCallback(async (timeoutMs = 1500): Promise<boolean> => {
|
||||
const ws = wsRef.current;
|
||||
|
|
@ -191,6 +298,16 @@ export function useVoiceMode() {
|
|||
wsRef.current.close();
|
||||
wsRef.current = null;
|
||||
}
|
||||
continuousCbRef.current = null;
|
||||
pausedRef.current = false;
|
||||
if (holdTimerRef.current) {
|
||||
clearTimeout(holdTimerRef.current);
|
||||
holdTimerRef.current = null;
|
||||
}
|
||||
if (keepAliveTimerRef.current) {
|
||||
clearInterval(keepAliveTimerRef.current);
|
||||
keepAliveTimerRef.current = null;
|
||||
}
|
||||
audioBufferRef.current = [];
|
||||
audioLevelsRef.current = [];
|
||||
audioPeakRef.current = 0;
|
||||
|
|
@ -200,7 +317,7 @@ export function useVoiceMode() {
|
|||
setState('idle');
|
||||
}, [stopInputCapture]);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
const start = useCallback(async (continuous = false) => {
|
||||
if (state !== 'idle') return;
|
||||
|
||||
transcriptBufferRef.current = '';
|
||||
|
|
@ -235,7 +352,7 @@ export function useVoiceMode() {
|
|||
console.error('Microphone access denied:', err);
|
||||
return null;
|
||||
}),
|
||||
connectWs(),
|
||||
connectWs(continuous),
|
||||
]);
|
||||
|
||||
if (!stream) {
|
||||
|
|
@ -259,6 +376,9 @@ export function useVoiceMode() {
|
|||
processorRef.current = processor;
|
||||
|
||||
processor.onaudioprocess = (e) => {
|
||||
// Paused (assistant is speaking in a call): drop mic audio so the
|
||||
// assistant's own TTS never gets transcribed back at it.
|
||||
if (pausedRef.current) return;
|
||||
const float32 = e.inputBuffer.getChannelData(0);
|
||||
const int16 = new Int16Array(float32.length);
|
||||
let sumSquares = 0;
|
||||
|
|
@ -283,8 +403,13 @@ export function useVoiceMode() {
|
|||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(buffer);
|
||||
} else {
|
||||
// WebSocket still connecting — buffer the audio
|
||||
// WebSocket still connecting (or reconnecting mid-call) —
|
||||
// buffer the audio, bounded so an unreachable server during a
|
||||
// long call can't grow it without limit (~30s at 64ms/chunk).
|
||||
audioBufferRef.current.push(buffer);
|
||||
if (audioBufferRef.current.length > 500) {
|
||||
audioBufferRef.current.shift();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -318,10 +443,46 @@ export function useVoiceMode() {
|
|||
stopAudioCapture();
|
||||
}, [stopAudioCapture]);
|
||||
|
||||
/**
|
||||
* Hands-free (call) mode: listen continuously and invoke `onUtterance`
|
||||
* with each completed utterance. Runs until cancel()/stop.
|
||||
*/
|
||||
const startContinuous = useCallback(async (onUtterance: (text: string) => void) => {
|
||||
continuousCbRef.current = onUtterance;
|
||||
await start(true);
|
||||
}, [start]);
|
||||
|
||||
/**
|
||||
* Mute/unmute the continuous stream (used while the assistant is
|
||||
* thinking/speaking). Keeps the Deepgram socket alive with KeepAlives and
|
||||
* discards any half-heard utterance from before the pause.
|
||||
*/
|
||||
const setPaused = useCallback((paused: boolean) => {
|
||||
if (pausedRef.current === paused) return;
|
||||
pausedRef.current = paused;
|
||||
if (paused) {
|
||||
if (holdTimerRef.current) {
|
||||
clearTimeout(holdTimerRef.current);
|
||||
holdTimerRef.current = null;
|
||||
}
|
||||
transcriptBufferRef.current = '';
|
||||
interimRef.current = '';
|
||||
setInterimText('');
|
||||
keepAliveTimerRef.current = setInterval(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(JSON.stringify({ type: 'KeepAlive' }));
|
||||
}
|
||||
}, KEEPALIVE_INTERVAL_MS);
|
||||
} else if (keepAliveTimerRef.current) {
|
||||
clearInterval(keepAliveTimerRef.current);
|
||||
keepAliveTimerRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
/** Pre-cache auth details so mic click skips IPC round-trips */
|
||||
const warmup = useCallback(() => {
|
||||
refreshAuth().catch(() => {});
|
||||
}, [refreshAuth]);
|
||||
|
||||
return { state, interimText, audioLevelsRef, start, submit, cancel, warmup };
|
||||
return { state, interimText, audioLevelsRef, start, submit, cancel, warmup, startContinuous, setPaused };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { useCallback, useRef, useState } from 'react';
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { dispatchCreditReplenished } from '@/lib/credit-status';
|
||||
|
||||
export type TTSState = 'idle' | 'synthesizing' | 'speaking';
|
||||
|
||||
|
|
@ -8,20 +9,32 @@ interface SynthesizedAudio {
|
|||
|
||||
function synthesize(text: string): Promise<SynthesizedAudio> {
|
||||
return window.ipc.invoke('voice:synthesize', { text }).then(
|
||||
(result: { audioBase64: string; mimeType: string }) => ({
|
||||
dataUrl: `data:${result.mimeType};base64,${result.audioBase64}`,
|
||||
})
|
||||
(result: { audioBase64: string; mimeType: string }) => {
|
||||
// A successful Rowboat voice synth is a cost-incurring call that
|
||||
// returned OK, so it proves credits are available again.
|
||||
dispatchCreditReplenished();
|
||||
return { dataUrl: `data:${result.mimeType};base64,${result.audioBase64}` };
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function playAudio(dataUrl: string, audioRef: React.MutableRefObject<HTMLAudioElement | null>): Promise<void> {
|
||||
function playAudio(
|
||||
dataUrl: string,
|
||||
audioRef: React.MutableRefObject<HTMLAudioElement | null>,
|
||||
onAudioElement?: (audio: HTMLAudioElement) => void
|
||||
): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const audio = new Audio(dataUrl);
|
||||
audioRef.current = audio;
|
||||
onAudioElement?.(audio);
|
||||
audio.onended = () => {
|
||||
console.log('[tts] audio ended');
|
||||
resolve();
|
||||
};
|
||||
// pause() (from cancel) must settle this promise too, or the queue
|
||||
// loop stays parked on it forever. Natural end also fires 'pause'
|
||||
// just before 'ended'; double-resolve is harmless.
|
||||
audio.onpause = () => resolve();
|
||||
audio.onerror = (e) => {
|
||||
console.error('[tts] audio error:', e);
|
||||
reject(new Error('Audio playback failed'));
|
||||
|
|
@ -35,47 +48,282 @@ function playAudio(dataUrl: string, audioRef: React.MutableRefObject<HTMLAudioEl
|
|||
});
|
||||
}
|
||||
|
||||
/** A queue entry: text to synthesize, or a ready-to-play audio URL (e.g. a bundled clip). */
|
||||
type QueueItem = { text: string } | { url: string };
|
||||
|
||||
type TtsChunkMsg = { requestId: string; chunkBase64?: string; done: boolean; error?: string };
|
||||
|
||||
export function useVoiceTTS() {
|
||||
const [state, setState] = useState<TTSState>('idle');
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const queueRef = useRef<string[]>([]);
|
||||
const queueRef = useRef<QueueItem[]>([]);
|
||||
const processingRef = useRef(false);
|
||||
// Pre-fetched audio ready to play immediately
|
||||
const prefetchedRef = useRef<Promise<SynthesizedAudio> | null>(null);
|
||||
// Streaming synthesis: per-request chunk handlers + the in-flight request
|
||||
// id (so cancel() can abort the main-process fetch).
|
||||
const streamHandlersRef = useRef<Map<string, (msg: TtsChunkMsg) => void>>(new Map());
|
||||
const activeStreamIdRef = useRef<string | null>(null);
|
||||
// Bumped by cancel(). A queue loop that awaited across a cancel sees a
|
||||
// stale generation and exits instead of playing audio that was cancelled
|
||||
// while still synthesizing (which would overlap the next utterance).
|
||||
const generationRef = useRef(0);
|
||||
// Web Audio analyser tap for lip-sync (talking head)
|
||||
const audioCtxRef = useRef<AudioContext | null>(null);
|
||||
const analyserRef = useRef<AnalyserNode | null>(null);
|
||||
const levelBufferRef = useRef<Uint8Array<ArrayBuffer> | null>(null);
|
||||
|
||||
// Route playback through an AnalyserNode so consumers can read the live
|
||||
// output level. If Web Audio wiring fails, the element still plays directly.
|
||||
const connectAnalyser = useCallback((audio: HTMLAudioElement) => {
|
||||
try {
|
||||
let ctx = audioCtxRef.current;
|
||||
if (!ctx) {
|
||||
ctx = new AudioContext();
|
||||
audioCtxRef.current = ctx;
|
||||
const analyser = ctx.createAnalyser();
|
||||
analyser.fftSize = 512;
|
||||
analyser.smoothingTimeConstant = 0.5;
|
||||
analyser.connect(ctx.destination);
|
||||
analyserRef.current = analyser;
|
||||
}
|
||||
if (ctx.state === 'suspended') {
|
||||
void ctx.resume();
|
||||
}
|
||||
const source = ctx.createMediaElementSource(audio);
|
||||
source.connect(analyserRef.current!);
|
||||
// Detach once this chunk is done (ended, cancelled via pause, or
|
||||
// failed) so source nodes don't accumulate over a long session.
|
||||
const disconnect = () => {
|
||||
try {
|
||||
source.disconnect();
|
||||
} catch {
|
||||
// already disconnected
|
||||
}
|
||||
};
|
||||
audio.addEventListener('ended', disconnect, { once: true });
|
||||
audio.addEventListener('pause', disconnect, { once: true });
|
||||
audio.addEventListener('error', disconnect, { once: true });
|
||||
} catch (err) {
|
||||
console.error('[tts] analyser hookup failed:', err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Current output level, 0..1. Safe to call every animation frame.
|
||||
// Release the audio graph when the owning component unmounts
|
||||
useEffect(() => () => {
|
||||
audioCtxRef.current?.close().catch(() => {});
|
||||
audioCtxRef.current = null;
|
||||
analyserRef.current = null;
|
||||
}, []);
|
||||
|
||||
// Route streaming TTS chunks to whichever request is waiting for them.
|
||||
useEffect(() => {
|
||||
return window.ipc.on('voice:tts-chunk', (msg) => {
|
||||
streamHandlersRef.current.get(msg.requestId)?.(msg);
|
||||
});
|
||||
}, []);
|
||||
|
||||
/**
|
||||
* Streaming synthesis + playback via MediaSource: audio starts on the
|
||||
* first chunk instead of after the full body. Rejects (for caller
|
||||
* fallback to non-streaming synth) if the stream fails before any audio
|
||||
* arrived; resolves when playback finishes.
|
||||
*/
|
||||
const streamSynthesizeAndPlay = useCallback((text: string, onStarted: () => void): Promise<void> => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported('audio/mpeg')) {
|
||||
reject(new Error('MSE audio/mpeg unsupported'));
|
||||
return;
|
||||
}
|
||||
const requestId = `tts-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
||||
const mediaSource = new MediaSource();
|
||||
const audio = new Audio();
|
||||
audio.src = URL.createObjectURL(mediaSource);
|
||||
audioRef.current = audio;
|
||||
connectAnalyser(audio);
|
||||
activeStreamIdRef.current = requestId;
|
||||
|
||||
let sourceBuffer: SourceBuffer | null = null;
|
||||
const pending: Uint8Array[] = [];
|
||||
let streamDone = false;
|
||||
let gotAudio = false;
|
||||
let settled = false;
|
||||
|
||||
const cleanup = () => {
|
||||
streamHandlersRef.current.delete(requestId);
|
||||
if (activeStreamIdRef.current === requestId) activeStreamIdRef.current = null;
|
||||
URL.revokeObjectURL(audio.src);
|
||||
};
|
||||
const finish = (err?: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
};
|
||||
|
||||
// Drain pending chunks into the SourceBuffer one at a time
|
||||
// (appendBuffer is async; only one append may be in flight).
|
||||
const pump = () => {
|
||||
if (!sourceBuffer || sourceBuffer.updating || settled) return;
|
||||
const chunk = pending.shift();
|
||||
if (chunk) {
|
||||
try {
|
||||
sourceBuffer.appendBuffer(chunk as BufferSource);
|
||||
} catch (e) {
|
||||
finish(e as Error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (streamDone && mediaSource.readyState === 'open') {
|
||||
try {
|
||||
mediaSource.endOfStream();
|
||||
} catch { /* already ended */ }
|
||||
}
|
||||
};
|
||||
|
||||
mediaSource.addEventListener('sourceopen', () => {
|
||||
try {
|
||||
sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg');
|
||||
} catch (e) {
|
||||
finish(e as Error);
|
||||
return;
|
||||
}
|
||||
sourceBuffer.addEventListener('updateend', pump);
|
||||
pump();
|
||||
}, { once: true });
|
||||
|
||||
streamHandlersRef.current.set(requestId, (msg) => {
|
||||
if (msg.error && !gotAudio) {
|
||||
streamDone = true;
|
||||
finish(new Error(msg.error));
|
||||
return;
|
||||
}
|
||||
if (msg.chunkBase64) {
|
||||
gotAudio = true;
|
||||
const bin = atob(msg.chunkBase64);
|
||||
const bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||||
pending.push(bytes);
|
||||
pump();
|
||||
}
|
||||
if (msg.done) {
|
||||
streamDone = true;
|
||||
pump();
|
||||
}
|
||||
});
|
||||
|
||||
audio.addEventListener('playing', () => onStarted(), { once: true });
|
||||
audio.onended = () => finish();
|
||||
// pause() (from cancel) must settle this promise too; natural end
|
||||
// also fires 'pause' just before 'ended'; double-settle is a no-op.
|
||||
audio.onpause = () => finish();
|
||||
audio.onerror = () => finish(new Error('stream playback failed'));
|
||||
|
||||
window.ipc
|
||||
.invoke('voice:synthesizeStreamStart', { requestId, text })
|
||||
.then((res) => {
|
||||
if (!res.ok) finish(new Error(res.error || 'stream start failed'));
|
||||
})
|
||||
.catch((e) => finish(e as Error));
|
||||
|
||||
// Starts as soon as the first appended data is decodable.
|
||||
audio.play().catch(() => { /* surfaced via onerror / chunk error */ });
|
||||
|
||||
// Nothing arrived at all — bail so the caller can fall back.
|
||||
setTimeout(() => {
|
||||
if (!gotAudio && !settled) finish(new Error('stream timeout'));
|
||||
}, 10_000);
|
||||
});
|
||||
}, [connectAnalyser]);
|
||||
|
||||
const getLevel = useCallback((): number => {
|
||||
const analyser = analyserRef.current;
|
||||
if (!analyser) return 0;
|
||||
let buffer = levelBufferRef.current;
|
||||
if (!buffer || buffer.length !== analyser.fftSize) {
|
||||
buffer = new Uint8Array(analyser.fftSize);
|
||||
levelBufferRef.current = buffer;
|
||||
}
|
||||
analyser.getByteTimeDomainData(buffer);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < buffer.length; i++) {
|
||||
const d = (buffer[i] - 128) / 128;
|
||||
sum += d * d;
|
||||
}
|
||||
const rms = Math.sqrt(sum / buffer.length);
|
||||
return Math.min(1, rms * 4);
|
||||
}, []);
|
||||
|
||||
const processQueue = useCallback(async () => {
|
||||
if (processingRef.current) return;
|
||||
processingRef.current = true;
|
||||
const gen = generationRef.current;
|
||||
|
||||
// Kick off full-body pre-fetch for the next queued text while the
|
||||
// current one plays — keeps sentence-to-sentence playback gapless.
|
||||
const prefetchNext = () => {
|
||||
const next = queueRef.current[0];
|
||||
if (next && 'text' in next && next.text.trim() && !prefetchedRef.current) {
|
||||
console.log('[tts] pre-fetching next:', next.text.substring(0, 80));
|
||||
prefetchedRef.current = synthesize(next.text);
|
||||
}
|
||||
};
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
const text = queueRef.current.shift()!;
|
||||
if (!text.trim()) continue;
|
||||
const item = queueRef.current.shift()!;
|
||||
if ('text' in item && !item.text.trim()) continue;
|
||||
|
||||
// Cold start (nothing playing, nothing pre-fetched): stream the
|
||||
// synthesis so audio begins on the first chunk instead of after
|
||||
// the full body — this is where first-response latency lives.
|
||||
if ('text' in item && !prefetchedRef.current) {
|
||||
setState('synthesizing');
|
||||
console.log('[tts] stream-synthesizing:', item.text.substring(0, 80));
|
||||
try {
|
||||
await streamSynthesizeAndPlay(item.text, () => {
|
||||
if (generationRef.current !== gen) return;
|
||||
setState('speaking');
|
||||
prefetchNext();
|
||||
});
|
||||
if (generationRef.current !== gen) return;
|
||||
continue;
|
||||
} catch (err) {
|
||||
if (generationRef.current !== gen) return;
|
||||
console.error('[tts] stream failed, falling back to full synth:', err);
|
||||
// fall through to the non-streaming path below
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Use pre-fetched result if available, otherwise synthesize now
|
||||
// Pre-recorded URL plays as-is; text uses the pre-fetched
|
||||
// result if available, otherwise synthesizes now.
|
||||
let audioPromise: Promise<SynthesizedAudio>;
|
||||
if (prefetchedRef.current) {
|
||||
if ('url' in item) {
|
||||
audioPromise = Promise.resolve({ dataUrl: item.url });
|
||||
} else if (prefetchedRef.current) {
|
||||
console.log('[tts] using pre-fetched audio');
|
||||
audioPromise = prefetchedRef.current;
|
||||
prefetchedRef.current = null;
|
||||
} else {
|
||||
setState('synthesizing');
|
||||
console.log('[tts] synthesizing:', text.substring(0, 80));
|
||||
audioPromise = synthesize(text);
|
||||
console.log('[tts] synthesizing:', item.text.substring(0, 80));
|
||||
audioPromise = synthesize(item.text);
|
||||
}
|
||||
|
||||
const audio = await audioPromise;
|
||||
// Cancelled while synthesizing — cancel() already reset all
|
||||
// state (and a new loop may be running), so just bail.
|
||||
if (generationRef.current !== gen) return;
|
||||
setState('speaking');
|
||||
|
||||
// Kick off pre-fetch for next chunk while this one plays
|
||||
const nextText = queueRef.current[0];
|
||||
if (nextText?.trim()) {
|
||||
console.log('[tts] pre-fetching next:', nextText.substring(0, 80));
|
||||
prefetchedRef.current = synthesize(nextText);
|
||||
}
|
||||
prefetchNext();
|
||||
|
||||
await playAudio(audio.dataUrl, audioRef);
|
||||
await playAudio(audio.dataUrl, audioRef, connectAnalyser);
|
||||
if (generationRef.current !== gen) return;
|
||||
} catch (err) {
|
||||
if (generationRef.current !== gen) return;
|
||||
console.error('[tts] error:', err);
|
||||
prefetchedRef.current = null;
|
||||
}
|
||||
|
|
@ -85,17 +333,33 @@ export function useVoiceTTS() {
|
|||
prefetchedRef.current = null;
|
||||
processingRef.current = false;
|
||||
setState('idle');
|
||||
}, []);
|
||||
}, [connectAnalyser, streamSynthesizeAndPlay]);
|
||||
|
||||
const speak = useCallback((text: string) => {
|
||||
console.log('[tts] speak() called:', text.substring(0, 80));
|
||||
queueRef.current.push(text);
|
||||
queueRef.current.push({ text });
|
||||
processQueue();
|
||||
}, [processQueue]);
|
||||
|
||||
// Play a pre-recorded clip (e.g. bundled tour narration) through the same
|
||||
// queue, so lip-sync levels, state, and cancel() all work unchanged.
|
||||
const speakUrl = useCallback((url: string) => {
|
||||
console.log('[tts] speakUrl() called:', url.substring(0, 120));
|
||||
queueRef.current.push({ url });
|
||||
processQueue();
|
||||
}, [processQueue]);
|
||||
|
||||
const cancel = useCallback(() => {
|
||||
generationRef.current++;
|
||||
queueRef.current = [];
|
||||
prefetchedRef.current = null;
|
||||
// Abort any in-flight streaming synthesis in the main process.
|
||||
if (activeStreamIdRef.current) {
|
||||
void window.ipc
|
||||
.invoke('voice:synthesizeStreamCancel', { requestId: activeStreamIdRef.current })
|
||||
.catch(() => {});
|
||||
activeStreamIdRef.current = null;
|
||||
}
|
||||
if (audioRef.current) {
|
||||
audioRef.current.pause();
|
||||
audioRef.current = null;
|
||||
|
|
@ -104,5 +368,5 @@ export function useVoiceTTS() {
|
|||
setState('idle');
|
||||
}, []);
|
||||
|
||||
return { state, speak, cancel };
|
||||
return { state, speak, speakUrl, cancel, getLevel };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,6 +53,10 @@ export function chatMessageSent(props: {
|
|||
})
|
||||
}
|
||||
|
||||
export function appOpened(folder: string) {
|
||||
posthog.capture('app_opened', { folder })
|
||||
}
|
||||
|
||||
export function oauthConnected(provider: string) {
|
||||
posthog.capture('oauth_connected', { provider })
|
||||
}
|
||||
|
|
@ -65,6 +69,26 @@ export function voiceInputStarted() {
|
|||
posthog.capture('voice_input_started')
|
||||
}
|
||||
|
||||
export function callStarted(preset: 'voice' | 'video' | 'share' | 'practice') {
|
||||
posthog.capture('call_started', { preset })
|
||||
}
|
||||
|
||||
// Voice-to-voice latency breakdown for one call turn (all milliseconds):
|
||||
// utterance accepted → message submitted → first TTS speak() → audio playing.
|
||||
export function callTurnLatency(props: {
|
||||
endpointToSubmitMs: number
|
||||
submitToSpeakMs: number
|
||||
speakToAudioMs: number
|
||||
totalMs: number
|
||||
}) {
|
||||
posthog.capture('call_turn_latency', {
|
||||
endpoint_to_submit_ms: Math.round(props.endpointToSubmitMs),
|
||||
submit_to_speak_ms: Math.round(props.submitToSpeakMs),
|
||||
speak_to_audio_ms: Math.round(props.speakToAudioMs),
|
||||
total_ms: Math.round(props.totalMs),
|
||||
})
|
||||
}
|
||||
|
||||
export function searchExecuted(types: string[]) {
|
||||
posthog.capture('search_executed', { types })
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,20 @@
|
|||
export const BILLING_ERROR_PATTERNS = [
|
||||
{
|
||||
kind: 'subscription_required',
|
||||
pattern: /upgrade required/i,
|
||||
title: 'A subscription is required',
|
||||
subtitle: 'Get started with a plan to access AI features in Rowboat.',
|
||||
cta: 'Subscribe',
|
||||
},
|
||||
{
|
||||
kind: 'out_of_credits',
|
||||
pattern: /not enough credits/i,
|
||||
title: "You've run out of credits",
|
||||
subtitle: 'Upgrade your plan for more usage. Daily usage resets at 00:00 UTC.',
|
||||
cta: 'Upgrade plan',
|
||||
},
|
||||
{
|
||||
kind: 'subscription_inactive',
|
||||
pattern: /subscription not active/i,
|
||||
title: 'Your subscription is inactive',
|
||||
subtitle: 'Reactivate your subscription to continue using AI features.',
|
||||
|
|
|
|||
30
apps/x/apps/renderer/src/lib/call-sounds.ts
Normal file
30
apps/x/apps/renderer/src/lib/call-sounds.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
// Tiny synthesized UI sounds for calls — no audio assets, one lazy context.
|
||||
|
||||
let ctx: AudioContext | null = null
|
||||
|
||||
/**
|
||||
* Soft rising blip played the instant an utterance is accepted — sub-second
|
||||
* acknowledgment makes the (still ongoing) model turn feel responsive
|
||||
* instead of dead air.
|
||||
*/
|
||||
export function playAckCue() {
|
||||
try {
|
||||
if (!ctx) ctx = new AudioContext()
|
||||
if (ctx.state === 'suspended') void ctx.resume()
|
||||
const t = ctx.currentTime
|
||||
const osc = ctx.createOscillator()
|
||||
const gain = ctx.createGain()
|
||||
osc.type = 'sine'
|
||||
osc.frequency.setValueAtTime(880, t)
|
||||
osc.frequency.exponentialRampToValueAtTime(1320, t + 0.08)
|
||||
gain.gain.setValueAtTime(0.0001, t)
|
||||
gain.gain.exponentialRampToValueAtTime(0.08, t + 0.015)
|
||||
gain.gain.exponentialRampToValueAtTime(0.0001, t + 0.12)
|
||||
osc.connect(gain)
|
||||
gain.connect(ctx.destination)
|
||||
osc.start(t)
|
||||
osc.stop(t + 0.13)
|
||||
} catch {
|
||||
// cosmetic — never let a sound failure affect the call
|
||||
}
|
||||
}
|
||||
|
|
@ -10,6 +10,9 @@ export interface MessageAttachment {
|
|||
mimeType: string
|
||||
size?: number
|
||||
thumbnailUrl?: string
|
||||
/** Live webcam frame from video chat mode — rendered as a compact filmstrip.
|
||||
* Carries no path; thumbnailUrl holds the frame as a data: URL. */
|
||||
isVideoFrame?: boolean
|
||||
}
|
||||
|
||||
export interface ChatMessage {
|
||||
|
|
@ -118,6 +121,19 @@ export const normalizeToolOutput = (
|
|||
return output
|
||||
}
|
||||
|
||||
export const getToolErrorText = (tool: ToolCall): string | undefined => {
|
||||
if (tool.status !== 'error') return undefined
|
||||
if (typeof tool.result === 'string' && tool.result.trim()) return tool.result
|
||||
if (tool.result !== undefined) {
|
||||
try {
|
||||
return JSON.stringify(tool.result, null, 2)
|
||||
} catch {
|
||||
// Fall through to the generic label for non-serializable legacy values.
|
||||
}
|
||||
}
|
||||
return 'Tool error'
|
||||
}
|
||||
|
||||
export type WebSearchCardResult = { title: string; url: string; description: string }
|
||||
|
||||
export type WebSearchCardData = {
|
||||
|
|
@ -198,6 +214,22 @@ const summarizeFilterUpdates = (updates: Record<string, unknown>): string => {
|
|||
return parts.length > 0 ? parts.join(', ') : 'Updated view'
|
||||
}
|
||||
|
||||
const APP_VIEW_LABELS: Record<string, string> = {
|
||||
home: 'home',
|
||||
email: 'email',
|
||||
meetings: 'meetings',
|
||||
'live-notes': 'live notes',
|
||||
'bg-tasks': 'background agents',
|
||||
'chat-history': 'chat history',
|
||||
knowledge: 'knowledge',
|
||||
workspace: 'workspace',
|
||||
code: 'code',
|
||||
bases: 'bases',
|
||||
graph: 'graph',
|
||||
}
|
||||
|
||||
const appViewLabel = (view: unknown): string => APP_VIEW_LABELS[view as string] ?? String(view ?? 'view')
|
||||
|
||||
export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null => {
|
||||
if (tool.name !== 'app-navigation') return null
|
||||
const result = tool.result as Record<string, unknown> | undefined
|
||||
|
|
@ -209,7 +241,10 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null =
|
|||
const action = input.action as string
|
||||
switch (action) {
|
||||
case 'open-note': return { action, label: `Opening ${(input.path as string || '').split('/').pop()?.replace(/\.md$/, '') || 'note'}...` }
|
||||
case 'open-view': return { action, label: `Opening ${input.view} view...` }
|
||||
case 'open-view': return { action, label: `Opening ${appViewLabel(input.view)}...` }
|
||||
case 'open-app': return { action, label: `Opening ${input.appId || 'app'}...` }
|
||||
case 'read-view': return { action, label: `Reading ${appViewLabel(input.view)}...` }
|
||||
case 'open-item': return { action, label: 'Opening...' }
|
||||
case 'update-base-view': return { action, label: 'Updating view...' }
|
||||
case 'create-base': return { action, label: `Creating "${input.name}"...` }
|
||||
case 'get-base-state': return null // renders as normal tool block
|
||||
|
|
@ -224,7 +259,33 @@ export const getAppActionCardData = (tool: ToolCall): AppActionCardData | null =
|
|||
return { action: 'open-note', label: `Opened ${name}` }
|
||||
}
|
||||
case 'open-view':
|
||||
return { action: 'open-view', label: `Opened ${result.view} view` }
|
||||
return { action: 'open-view', label: `Opened ${appViewLabel(result.view)}` }
|
||||
case 'open-app':
|
||||
return { action: 'open-app', label: `Opened ${result.appName || result.appId || 'app'}` }
|
||||
case 'read-view': {
|
||||
const counted =
|
||||
(result.threads as unknown[] | undefined)?.length ??
|
||||
(result.agents as unknown[] | undefined)?.length ??
|
||||
(result.sessions as unknown[] | undefined)?.length
|
||||
return {
|
||||
action: 'read-view',
|
||||
label: counted !== undefined
|
||||
? `Read ${appViewLabel(result.view)} (${counted} item${counted === 1 ? '' : 's'})`
|
||||
: `Read ${appViewLabel(result.view)}`,
|
||||
}
|
||||
}
|
||||
case 'open-item': {
|
||||
switch (result.kind) {
|
||||
case 'email-thread': return { action: 'open-item', label: 'Opened email thread' }
|
||||
case 'note': {
|
||||
const name = (result.path as string || '').split('/').pop()?.replace(/\.md$/, '') || 'note'
|
||||
return { action: 'open-item', label: `Opened ${name}` }
|
||||
}
|
||||
case 'bg-task': return { action: 'open-item', label: `Opened agent "${result.taskName}"` }
|
||||
case 'session': return { action: 'open-item', label: 'Opened chat' }
|
||||
default: return { action: 'open-item', label: 'Opened item' }
|
||||
}
|
||||
}
|
||||
case 'update-base-view':
|
||||
return {
|
||||
action: 'update-base-view',
|
||||
|
|
|
|||
52
apps/x/apps/renderer/src/lib/code-run-feed.ts
Normal file
52
apps/x/apps/renderer/src/lib/code-run-feed.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
import { useSyncExternalStore } from 'react'
|
||||
import type { CodeRunEvent, CodeRunFeedEvent } from '@x/shared/src/code-mode.js'
|
||||
|
||||
// Renderer half of the ephemeral CodeRunFeed side-channel: buffers the live
|
||||
// `codeRun:events` broadcast per toolCallId so tool cards can render a
|
||||
// code_agent_run's activity while it streams. Module-level (not tied to any
|
||||
// session store) so the buffer survives session switches mid-run. Nothing here
|
||||
// is persisted — on settle the durable code-run-events-batch in the turn
|
||||
// record supersedes the buffer, which is then dropped.
|
||||
const buffers = new Map<string, CodeRunEvent[]>()
|
||||
const listeners = new Set<() => void>()
|
||||
const EMPTY: CodeRunEvent[] = []
|
||||
// Backstop so abandoned runs can't grow the map forever (a run's buffer is
|
||||
// normally dropped explicitly once its durable batch lands).
|
||||
const MAX_TRACKED_RUNS = 32
|
||||
|
||||
let attached = false
|
||||
function ensureAttached(): void {
|
||||
if (attached) return
|
||||
attached = true
|
||||
window.ipc.on('codeRun:events', ((raw: unknown) => {
|
||||
const { toolCallId, event } = raw as CodeRunFeedEvent
|
||||
if (!toolCallId || !event) return
|
||||
if (!buffers.has(toolCallId) && buffers.size >= MAX_TRACKED_RUNS) {
|
||||
const oldest = buffers.keys().next().value
|
||||
if (oldest !== undefined) buffers.delete(oldest)
|
||||
}
|
||||
// Immutable append: useSyncExternalStore consumers compare by reference.
|
||||
buffers.set(toolCallId, [...(buffers.get(toolCallId) ?? EMPTY), event])
|
||||
for (const listener of [...listeners]) listener()
|
||||
}) as never)
|
||||
}
|
||||
|
||||
function subscribe(onChange: () => void): () => void {
|
||||
ensureAttached()
|
||||
listeners.add(onChange)
|
||||
return () => {
|
||||
listeners.delete(onChange)
|
||||
}
|
||||
}
|
||||
|
||||
export function clearCodeRunBuffer(toolCallId: string): void {
|
||||
if (buffers.delete(toolCallId)) {
|
||||
for (const listener of [...listeners]) listener()
|
||||
}
|
||||
}
|
||||
|
||||
// Live events for one code_agent_run tool call, empty once the durable batch
|
||||
// takes over (or if the buffer never existed — e.g. after an app reload).
|
||||
export function useCodeRunFeed(toolCallId: string): CodeRunEvent[] {
|
||||
return useSyncExternalStore(subscribe, () => buffers.get(toolCallId) ?? EMPTY)
|
||||
}
|
||||
26
apps/x/apps/renderer/src/lib/credit-status.ts
Normal file
26
apps/x/apps/renderer/src/lib/credit-status.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import type { BillingInfo } from '@x/shared/dist/billing.js'
|
||||
|
||||
/**
|
||||
* A user is "out of credits" when EITHER the daily or the monthly bucket is
|
||||
* exhausted. Either exhaustion is what triggers the backend's "not enough
|
||||
* credits" API error, and this mirrors the two usage bars shown in Settings
|
||||
* (Plan usage = monthly, Daily use = daily). `availableCredits` already comes
|
||||
* down in the /v1/me payload, so no extra API work is needed.
|
||||
*/
|
||||
export function isOutOfCredits(billing: BillingInfo): boolean {
|
||||
return billing.daily.availableCredits <= 0 || billing.monthly.availableCredits <= 0
|
||||
}
|
||||
|
||||
/** Fired when we learn the user is out of credits (billing data or a usage API error). */
|
||||
export const CREDIT_EXHAUSTED_EVENT = 'credit-status:exhausted'
|
||||
|
||||
/** Fired when a successful cost-incurring call (LLM / voice) proves credits are available again. */
|
||||
export const CREDIT_REPLENISHED_EVENT = 'credit-status:replenished'
|
||||
|
||||
export function dispatchCreditExhausted(): void {
|
||||
window.dispatchEvent(new Event(CREDIT_EXHAUSTED_EVENT))
|
||||
}
|
||||
|
||||
export function dispatchCreditReplenished(): void {
|
||||
window.dispatchEvent(new Event(CREDIT_REPLENISHED_EVENT))
|
||||
}
|
||||
|
|
@ -41,6 +41,9 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
|
|||
filename?: string
|
||||
mimeType?: string
|
||||
size?: number
|
||||
data?: string
|
||||
mediaType?: string
|
||||
source?: string
|
||||
toolCallId?: string
|
||||
toolName?: string
|
||||
arguments?: unknown
|
||||
|
|
@ -52,13 +55,24 @@ export function runLogToConversation(log: RunLog): ConversationItem[] {
|
|||
.join('')
|
||||
|
||||
const attachmentParts = parts.filter((p) => p.type === 'attachment' && p.path)
|
||||
if (attachmentParts.length > 0) {
|
||||
msgAttachments = attachmentParts.map((p) => ({
|
||||
path: p.path!,
|
||||
filename: p.filename || p.path!.split('/').pop() || p.path!,
|
||||
mimeType: p.mimeType || 'application/octet-stream',
|
||||
size: p.size,
|
||||
}))
|
||||
// Video-mode webcam frames — inline base64 image parts, shown as a filmstrip
|
||||
const imageParts = parts.filter((p) => p.type === 'image' && p.data)
|
||||
if (attachmentParts.length > 0 || imageParts.length > 0) {
|
||||
msgAttachments = [
|
||||
...attachmentParts.map((p) => ({
|
||||
path: p.path!,
|
||||
filename: p.filename || p.path!.split('/').pop() || p.path!,
|
||||
mimeType: p.mimeType || 'application/octet-stream',
|
||||
size: p.size,
|
||||
})),
|
||||
...imageParts.map((p, index) => ({
|
||||
path: '',
|
||||
filename: `${p.source === 'screen' ? 'screen' : 'camera'}-frame-${index + 1}.jpg`,
|
||||
mimeType: p.mediaType || 'image/jpeg',
|
||||
thumbnailUrl: `data:${p.mediaType || 'image/jpeg'};base64,${p.data}`,
|
||||
isVideoFrame: true,
|
||||
})),
|
||||
]
|
||||
}
|
||||
|
||||
if (msg.role === 'assistant') {
|
||||
|
|
|
|||
|
|
@ -96,6 +96,25 @@ describe('voice output', () => {
|
|||
expect(overlay.voiceSegments).toEqual(['hello there', 'bye'])
|
||||
})
|
||||
|
||||
it('emits an early clause from a long open block, then the remainder on close', () => {
|
||||
let overlay = emptyOverlay()
|
||||
const longClause = 'Okay so the first thing I would look at here is the error message,'
|
||||
overlay = applyOverlay(overlay, delta(`<voice>${longClause} because`))
|
||||
// Open block crossed the early-speech threshold at a clause boundary.
|
||||
expect(overlay.voiceSegments).toEqual([longClause])
|
||||
overlay = applyOverlay(overlay, delta(' it tells you the root cause.</voice>'))
|
||||
// Remainder only — the early clause is not repeated.
|
||||
expect(overlay.voiceSegments).toEqual([longClause, 'because it tells you the root cause.'])
|
||||
})
|
||||
|
||||
it('does not emit early clauses from short open blocks', () => {
|
||||
let overlay = emptyOverlay()
|
||||
overlay = applyOverlay(overlay, delta('<voice>Sure, one sec'))
|
||||
expect(overlay.voiceSegments).toEqual([])
|
||||
overlay = applyOverlay(overlay, delta('.</voice>'))
|
||||
expect(overlay.voiceSegments).toEqual(['Sure, one sec.'])
|
||||
})
|
||||
|
||||
it('keeps segments but resets the scan on model_call_completed', () => {
|
||||
let overlay = emptyOverlay()
|
||||
overlay = applyOverlay(overlay, delta('<voice>one</voice>'))
|
||||
|
|
@ -171,6 +190,117 @@ describe('buildTurnConversation', () => {
|
|||
expect(items.map((i) => i.status)).toEqual(['pending', 'running'])
|
||||
})
|
||||
|
||||
it('uses the tool result envelope to determine error status', () => {
|
||||
const state = reduceTurn([
|
||||
created(T1, S1),
|
||||
requested(T1, 0),
|
||||
completed(
|
||||
T1,
|
||||
0,
|
||||
assistantCalls(
|
||||
toolCallPart('flagged', 'echo'),
|
||||
toolCallPart('normal', 'echo'),
|
||||
),
|
||||
),
|
||||
invocation(T1, 'flagged', 'echo'),
|
||||
{
|
||||
type: 'tool_result',
|
||||
turnId: T1,
|
||||
ts: TS,
|
||||
toolCallId: 'flagged',
|
||||
toolName: 'echo',
|
||||
source: 'sync',
|
||||
result: { output: 'boom', isError: true },
|
||||
},
|
||||
invocation(T1, 'normal', 'echo'),
|
||||
toolResult(T1, 'normal', 'echo', { success: false, message: 'payload data' }),
|
||||
])
|
||||
const items = buildTurnConversation(state).filter(isToolCall)
|
||||
expect(items.map((i) => i.status)).toEqual(['error', 'completed'])
|
||||
})
|
||||
|
||||
it('derives the code-run timeline from the settle-time batch and asks from request events', () => {
|
||||
const codeProgress = (progress: unknown): TEvent => ({
|
||||
type: 'tool_progress',
|
||||
turnId: T1,
|
||||
ts: TS,
|
||||
toolCallId: 'cr1',
|
||||
source: 'sync',
|
||||
progress: progress as never,
|
||||
})
|
||||
const state = reduceTurn([
|
||||
created(T1, S1),
|
||||
requested(T1, 0),
|
||||
completed(T1, 0, assistantCalls(toolCallPart('cr1', 'code_agent_run', { agent: 'codex' }))),
|
||||
invocation(T1, 'cr1', 'code_agent_run'),
|
||||
codeProgress({
|
||||
kind: 'code-run-permission-request',
|
||||
requestId: 'cpr-1',
|
||||
ask: { toolCallId: 'x', title: 'write file', options: [] },
|
||||
}),
|
||||
codeProgress({
|
||||
kind: 'code-run-events',
|
||||
events: [
|
||||
{ type: 'message', role: 'agent', text: 'hi' },
|
||||
{ type: 'tool_call', id: 'x', title: 'write file' },
|
||||
],
|
||||
}),
|
||||
])
|
||||
const tool = buildTurnConversation(state).filter(isToolCall)[0]
|
||||
expect(tool.status).toBe('running')
|
||||
expect(tool.codeRunEvents?.map((e) => e.type)).toEqual(['message', 'tool_call'])
|
||||
expect(tool.pendingCodePermission?.requestId).toBe('cpr-1')
|
||||
})
|
||||
|
||||
it('clears the pending code permission on the resolved marker and on tool result', () => {
|
||||
const codeProgress = (toolCallId: string, progress: unknown): TEvent => ({
|
||||
type: 'tool_progress',
|
||||
turnId: T1,
|
||||
ts: TS,
|
||||
toolCallId,
|
||||
source: 'sync',
|
||||
progress: progress as never,
|
||||
})
|
||||
const ask = { toolCallId: 'x', title: 'write file', options: [] }
|
||||
// resolved: the durable marker pairs off the request mid-run
|
||||
const resolvedState = reduceTurn([
|
||||
created(T1, S1),
|
||||
requested(T1, 0),
|
||||
completed(T1, 0, assistantCalls(toolCallPart('cr1', 'code_agent_run'))),
|
||||
invocation(T1, 'cr1', 'code_agent_run'),
|
||||
codeProgress('cr1', { kind: 'code-run-permission-request', requestId: 'cpr-1', ask }),
|
||||
codeProgress('cr1', { kind: 'code-run-permission-resolved' }),
|
||||
])
|
||||
const resolved = buildTurnConversation(resolvedState).filter(isToolCall)[0]
|
||||
expect(resolved.pendingCodePermission).toBeUndefined()
|
||||
|
||||
// a second ask after the first resolution is pending again
|
||||
const secondAskState = reduceTurn([
|
||||
created(T1, S1),
|
||||
requested(T1, 0),
|
||||
completed(T1, 0, assistantCalls(toolCallPart('cr1', 'code_agent_run'))),
|
||||
invocation(T1, 'cr1', 'code_agent_run'),
|
||||
codeProgress('cr1', { kind: 'code-run-permission-request', requestId: 'cpr-1', ask }),
|
||||
codeProgress('cr1', { kind: 'code-run-permission-resolved' }),
|
||||
codeProgress('cr1', { kind: 'code-run-permission-request', requestId: 'cpr-2', ask }),
|
||||
])
|
||||
const secondAsk = buildTurnConversation(secondAskState).filter(isToolCall)[0]
|
||||
expect(secondAsk.pendingCodePermission?.requestId).toBe('cpr-2')
|
||||
|
||||
// settled: an unanswered ask must not survive the tool's terminal result
|
||||
const settledState = reduceTurn([
|
||||
created(T1, S1),
|
||||
requested(T1, 0),
|
||||
completed(T1, 0, assistantCalls(toolCallPart('cr2', 'code_agent_run'))),
|
||||
invocation(T1, 'cr2', 'code_agent_run'),
|
||||
codeProgress('cr2', { kind: 'code-run-permission-request', requestId: 'cpr-2', ask }),
|
||||
toolResult(T1, 'cr2', 'code_agent_run', { success: false, stopReason: 'cancelled' }),
|
||||
])
|
||||
const settled = buildTurnConversation(settledState).filter(isToolCall)[0]
|
||||
expect(settled.pendingCodePermission).toBeUndefined()
|
||||
expect(settled.status).toBe('completed')
|
||||
})
|
||||
|
||||
it('renders user attachments and a failed turn as an error item', () => {
|
||||
const input = {
|
||||
role: 'user' as const,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
type TurnState,
|
||||
type TurnStreamEvent,
|
||||
} from '@x/shared/src/turns.js'
|
||||
import type { CodeRunEvent, PermissionAsk } from '@x/shared/src/code-mode.js'
|
||||
import type {
|
||||
ChatMessage,
|
||||
ConversationItem,
|
||||
|
|
@ -34,14 +35,20 @@ export type LiveOverlay = {
|
|||
text: string
|
||||
reasoning: string
|
||||
toolOutput: Record<string, string>
|
||||
// Contents of completed <voice>…</voice> blocks seen while streaming, in
|
||||
// order, monotonically growing for the lifetime of the overlay (i.e. one
|
||||
// active turn). Consumers speak segments beyond what they've already
|
||||
// spoken; the overlay reset on turn switch starts a fresh list.
|
||||
// Speakable segments seen while streaming, in order, monotonically growing
|
||||
// for the lifetime of the overlay (i.e. one active turn). Usually the
|
||||
// contents of completed <voice>…</voice> blocks, but a long still-open
|
||||
// block may emit an early clause (see EARLY_SPEECH_MIN_CHARS) so speech
|
||||
// can start before the sentence finishes generating. Consumers speak
|
||||
// segments beyond what they've already spoken; the overlay reset on turn
|
||||
// switch starts a fresh list.
|
||||
voiceSegments: string[]
|
||||
// Scan cursor into `text` — everything before it has been checked for
|
||||
// complete voice blocks.
|
||||
voiceScanIndex: number
|
||||
// Chars of the currently-open voice block's content already emitted as an
|
||||
// early clause — the block's remainder (on close) excludes them.
|
||||
voicePartialConsumed: number
|
||||
}
|
||||
|
||||
export const emptyOverlay = (): LiveOverlay => ({
|
||||
|
|
@ -50,6 +57,7 @@ export const emptyOverlay = (): LiveOverlay => ({
|
|||
toolOutput: {},
|
||||
voiceSegments: [],
|
||||
voiceScanIndex: 0,
|
||||
voicePartialConsumed: 0,
|
||||
})
|
||||
|
||||
// The model emits <voice>…</voice> around speakable text when voice output
|
||||
|
|
@ -59,6 +67,17 @@ export function stripVoiceTags(text: string): string {
|
|||
}
|
||||
|
||||
const VOICE_BLOCK = /<voice>([\s\S]*?)<\/voice>/g
|
||||
const VOICE_OPEN_TAG = '<voice>'
|
||||
|
||||
// Early speech: once an open block has this many unconsumed chars, its last
|
||||
// complete clause is emitted immediately instead of waiting for </voice> —
|
||||
// TTS starts on the first clause while the rest of the sentence generates.
|
||||
const EARLY_SPEECH_MIN_CHARS = 60
|
||||
// ...but never emit a fragment shorter than this (prosody suffers).
|
||||
const EARLY_SPEECH_MIN_EMIT = 30
|
||||
// Clause boundaries (punctuation, optionally inside closing quote/paren,
|
||||
// followed by whitespace or end-of-buffer).
|
||||
const CLAUSE_BOUNDARY = /[,;:.!?…—]["')\]]*(?=\s|$)/g
|
||||
|
||||
// Accumulates deltas; canonical durable events supersede the buffers (the
|
||||
// committed transcript now contains what was streaming).
|
||||
|
|
@ -68,15 +87,43 @@ export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): Live
|
|||
const text = overlay.text + event.delta
|
||||
// Extract complete voice blocks past the scan cursor. Incomplete
|
||||
// blocks (opening tag seen, closing not yet) stay unconsumed until a
|
||||
// later delta completes them.
|
||||
// later delta completes them. The first complete block may have had an
|
||||
// early clause emitted while it was open — skip those chars.
|
||||
const segments: string[] = []
|
||||
let scanIndex = overlay.voiceScanIndex
|
||||
let partialConsumed = overlay.voicePartialConsumed
|
||||
VOICE_BLOCK.lastIndex = scanIndex
|
||||
for (let m = VOICE_BLOCK.exec(text); m; m = VOICE_BLOCK.exec(text)) {
|
||||
const content = m[1].trim()
|
||||
const content = m[1].slice(partialConsumed).trim()
|
||||
partialConsumed = 0
|
||||
if (content) segments.push(content)
|
||||
scanIndex = m.index + m[0].length
|
||||
}
|
||||
|
||||
// Early speech: if a voice block is still open and has accumulated a
|
||||
// long unconsumed run, emit its last complete clause now — speech can
|
||||
// start while the rest of the sentence is still generating.
|
||||
const openIdx = text.indexOf(VOICE_OPEN_TAG, scanIndex)
|
||||
if (openIdx !== -1) {
|
||||
const unconsumed = text.slice(openIdx + VOICE_OPEN_TAG.length + partialConsumed)
|
||||
if (unconsumed.length >= EARLY_SPEECH_MIN_CHARS) {
|
||||
let lastBoundaryEnd = -1
|
||||
CLAUSE_BOUNDARY.lastIndex = 0
|
||||
for (let b = CLAUSE_BOUNDARY.exec(unconsumed); b; b = CLAUSE_BOUNDARY.exec(unconsumed)) {
|
||||
lastBoundaryEnd = b.index + b[0].length
|
||||
}
|
||||
if (lastBoundaryEnd >= EARLY_SPEECH_MIN_EMIT) {
|
||||
const clause = unconsumed.slice(0, lastBoundaryEnd).trim()
|
||||
if (clause) segments.push(clause)
|
||||
partialConsumed += lastBoundaryEnd
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No open block — any partial bookkeeping belongs to a block that
|
||||
// has since closed.
|
||||
partialConsumed = 0
|
||||
}
|
||||
|
||||
return {
|
||||
...overlay,
|
||||
text,
|
||||
|
|
@ -84,12 +131,13 @@ export function applyOverlay(overlay: LiveOverlay, event: TurnStreamEvent): Live
|
|||
? { voiceSegments: [...overlay.voiceSegments, ...segments] }
|
||||
: {}),
|
||||
voiceScanIndex: scanIndex,
|
||||
voicePartialConsumed: partialConsumed,
|
||||
}
|
||||
}
|
||||
case 'reasoning_delta':
|
||||
return { ...overlay, reasoning: overlay.reasoning + event.delta }
|
||||
case 'model_call_completed':
|
||||
return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0 }
|
||||
return { ...overlay, text: '', reasoning: '', voiceScanIndex: 0, voicePartialConsumed: 0 }
|
||||
case 'tool_progress': {
|
||||
const progress = event.progress
|
||||
if (
|
||||
|
|
@ -153,11 +201,46 @@ function extractAttachments(content: UserContent): MessageAttachment[] | undefin
|
|||
}
|
||||
|
||||
function toolStatus(tc: ToolCallState): ToolCall['status'] {
|
||||
if (tc.result) return 'completed'
|
||||
if (tc.result) return tc.result.result.isError ? 'error' : 'completed'
|
||||
if (tc.permission && !tc.permission.resolved) return 'pending'
|
||||
return 'running'
|
||||
}
|
||||
|
||||
// code_agent_run's durable trail in tool_progress (see the publish bridge in
|
||||
// real-tool-registry.ts): ONE settle-time 'code-run-events' batch carrying the
|
||||
// whole timeline (the live per-event stream travels over the ephemeral
|
||||
// CodeRunFeed and never reaches turn state), plus per-ask
|
||||
// 'code-run-permission-request' / 'code-run-permission-resolved' pairs. An ask
|
||||
// is pending while requests outnumber resolutions and the tool hasn't settled.
|
||||
function codeRunViewOf(
|
||||
tc: ToolCallState,
|
||||
): Pick<ToolCall, 'codeRunEvents' | 'pendingCodePermission'> {
|
||||
let events: CodeRunEvent[] | undefined
|
||||
let pending: { requestId: string; ask: PermissionAsk } | null = null
|
||||
let unresolved = 0
|
||||
for (const p of tc.progress) {
|
||||
const entry = p.progress
|
||||
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) continue
|
||||
const kind = (entry as { kind?: unknown }).kind
|
||||
if (kind === 'code-run-events') {
|
||||
const batch = (entry as { events?: unknown }).events
|
||||
if (Array.isArray(batch)) events = batch as CodeRunEvent[]
|
||||
} else if (kind === 'code-run-permission-request') {
|
||||
const { requestId, ask } = entry as { requestId?: unknown; ask?: unknown }
|
||||
if (typeof requestId === 'string' && ask) {
|
||||
pending = { requestId, ask: ask as PermissionAsk }
|
||||
unresolved += 1
|
||||
}
|
||||
} else if (kind === 'code-run-permission-resolved') {
|
||||
unresolved -= 1
|
||||
}
|
||||
}
|
||||
return {
|
||||
...(events && events.length > 0 ? { codeRunEvents: events } : {}),
|
||||
...(pending && unresolved > 0 && !tc.result ? { pendingCodePermission: pending } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
// One turn's contribution to the conversation: the user input, then per
|
||||
// completed model call its text and tool calls (with live status/results).
|
||||
export function buildTurnConversation(state: TurnState): ConversationItem[] {
|
||||
|
|
@ -211,6 +294,7 @@ export function buildTurnConversation(state: TurnState): ConversationItem[] {
|
|||
...(tc?.result ? { result: tc.result.result.output as ToolCall['result'] } : {}),
|
||||
status: tc ? toolStatus(tc) : 'running',
|
||||
timestamp: ts(),
|
||||
...(tc ? codeRunViewOf(tc) : {}),
|
||||
} satisfies ToolCall)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
109
apps/x/apps/renderer/src/lib/tour-sounds.ts
Normal file
109
apps/x/apps/renderer/src/lib/tour-sounds.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
/**
|
||||
* Tiny synthesized sound effects for the product tour — oar splashes, dock
|
||||
* bumps, and an arrival fanfare. Everything is generated with Web Audio
|
||||
* oscillators/noise so no audio assets are needed. All methods fail silently
|
||||
* if audio is unavailable.
|
||||
*/
|
||||
export class TourSounds {
|
||||
private ctx: AudioContext | null = null
|
||||
private master: GainNode | null = null
|
||||
|
||||
private ensure(): AudioContext | null {
|
||||
try {
|
||||
if (!this.ctx) {
|
||||
this.ctx = new AudioContext()
|
||||
this.master = this.ctx.createGain()
|
||||
this.master.gain.value = 0.5
|
||||
this.master.connect(this.ctx.destination)
|
||||
}
|
||||
if (this.ctx.state === 'suspended') void this.ctx.resume()
|
||||
return this.ctx
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/** Short filtered-noise burst with a falling pitch — an oar dipping. */
|
||||
splash() {
|
||||
const ctx = this.ensure()
|
||||
if (!ctx || !this.master) return
|
||||
const dur = 0.22
|
||||
const noise = ctx.createBufferSource()
|
||||
const buffer = ctx.createBuffer(1, Math.ceil(ctx.sampleRate * dur), ctx.sampleRate)
|
||||
const data = buffer.getChannelData(0)
|
||||
for (let i = 0; i < data.length; i++) data[i] = Math.random() * 2 - 1
|
||||
noise.buffer = buffer
|
||||
|
||||
const filter = ctx.createBiquadFilter()
|
||||
filter.type = 'bandpass'
|
||||
filter.Q.value = 1.2
|
||||
filter.frequency.setValueAtTime(1600, ctx.currentTime)
|
||||
filter.frequency.exponentialRampToValueAtTime(350, ctx.currentTime + dur)
|
||||
|
||||
const gain = ctx.createGain()
|
||||
gain.gain.setValueAtTime(0.14, ctx.currentTime)
|
||||
gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + dur)
|
||||
|
||||
noise.connect(filter).connect(gain).connect(this.master)
|
||||
noise.start()
|
||||
noise.stop(ctx.currentTime + dur)
|
||||
}
|
||||
|
||||
/** Soft low thump — the boat nudging a dock. */
|
||||
bump() {
|
||||
const ctx = this.ensure()
|
||||
if (!ctx || !this.master) return
|
||||
const osc = ctx.createOscillator()
|
||||
osc.type = 'sine'
|
||||
osc.frequency.setValueAtTime(150, ctx.currentTime)
|
||||
osc.frequency.exponentialRampToValueAtTime(70, ctx.currentTime + 0.18)
|
||||
const gain = ctx.createGain()
|
||||
gain.gain.setValueAtTime(0.2, ctx.currentTime)
|
||||
gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + 0.25)
|
||||
osc.connect(gain).connect(this.master)
|
||||
osc.start()
|
||||
osc.stop(ctx.currentTime + 0.3)
|
||||
}
|
||||
|
||||
/** Gentle high blip — an email landing in the boat. */
|
||||
ding() {
|
||||
const ctx = this.ensure()
|
||||
if (!ctx || !this.master) return
|
||||
const osc = ctx.createOscillator()
|
||||
osc.type = 'sine'
|
||||
osc.frequency.setValueAtTime(880, ctx.currentTime)
|
||||
osc.frequency.exponentialRampToValueAtTime(1320, ctx.currentTime + 0.05)
|
||||
const gain = ctx.createGain()
|
||||
gain.gain.setValueAtTime(0.08, ctx.currentTime)
|
||||
gain.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + 0.3)
|
||||
osc.connect(gain).connect(this.master)
|
||||
osc.start()
|
||||
osc.stop(ctx.currentTime + 0.35)
|
||||
}
|
||||
|
||||
/** Little four-note arpeggio for the tour finale. */
|
||||
fanfare() {
|
||||
const ctx = this.ensure()
|
||||
if (!ctx || !this.master) return
|
||||
const notes = [523.25, 659.25, 783.99, 1046.5] // C5 E5 G5 C6
|
||||
notes.forEach((freq, i) => {
|
||||
const start = ctx.currentTime + i * 0.13
|
||||
const osc = ctx.createOscillator()
|
||||
osc.type = 'triangle'
|
||||
osc.frequency.value = freq
|
||||
const gain = ctx.createGain()
|
||||
gain.gain.setValueAtTime(0.0001, start)
|
||||
gain.gain.exponentialRampToValueAtTime(0.16, start + 0.02)
|
||||
gain.gain.exponentialRampToValueAtTime(0.001, start + (i === notes.length - 1 ? 0.7 : 0.3))
|
||||
osc.connect(gain).connect(this.master!)
|
||||
osc.start(start)
|
||||
osc.stop(start + 0.8)
|
||||
})
|
||||
}
|
||||
|
||||
dispose() {
|
||||
this.ctx?.close().catch(() => {})
|
||||
this.ctx = null
|
||||
this.master = null
|
||||
}
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import { PostHogProvider } from 'posthog-js/react'
|
|||
import type { CaptureResult } from 'posthog-js'
|
||||
import { ThemeProvider } from '@/contexts/theme-context'
|
||||
import { configureAnalyticsContext } from './lib/analytics'
|
||||
import { VideoPopout } from '@/components/video-popout'
|
||||
|
||||
// Fetch the stable installation ID from main so renderer + main share one
|
||||
// PostHog distinct_id. Falls back to PostHog's auto-generated anonymous ID
|
||||
|
|
@ -57,4 +58,14 @@ async function bootstrap() {
|
|||
// The loaded callback applies api_url/app_version once PostHog has initialized.
|
||||
}
|
||||
|
||||
bootstrap()
|
||||
// The video-mode popout window loads the same bundle with a hash route and
|
||||
// renders only the mini-call UI — no analytics or app bootstrap needed.
|
||||
if (window.location.hash === '#video-popout') {
|
||||
createRoot(document.getElementById('root')!).render(
|
||||
<StrictMode>
|
||||
<VideoPopout />
|
||||
</StrictMode>,
|
||||
)
|
||||
} else {
|
||||
bootstrap()
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue