From 4ca03daa4cf51d4ec06f9073e515baaf0c2356dc Mon Sep 17 00:00:00 2001 From: Gagancreates Date: Tue, 28 Apr 2026 20:10:13 +0530 Subject: [PATCH 01/22] feat: group consecutive tool calls into collapsible summary Consecutive plain tool calls are now grouped into a single collapsible row instead of rendering as individual items. - Header shows the currently-executing tool name live with a vertical ticker animation, then switches to "Ran N tools" on completion - Expanding the group reveals each tool call individually collapsible - Tool calls with pending permission requests render individually - Special cards (web search, composio connect, app actions) excluded --- apps/x/apps/renderer/src/App.tsx | 19 +++- .../src/components/ai-elements/tool.tsx | 89 +++++++++++++++++++ .../renderer/src/components/chat-sidebar.tsx | 19 +++- .../renderer/src/lib/chat-conversation.ts | 66 ++++++++++++++ 4 files changed, 189 insertions(+), 4 deletions(-) diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index 67f3f06a..0e925e2e 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -35,7 +35,7 @@ import { import { Shimmer } from '@/components/ai-elements/shimmer'; import { useSmoothedText } from './hooks/useSmoothedText'; -import { Tool, ToolContent, ToolHeader, ToolTabbedContent } from '@/components/ai-elements/tool'; +import { Tool, ToolContent, ToolGroupComponent, ToolHeader, ToolTabbedContent } from '@/components/ai-elements/tool'; import { WebSearchResult } from '@/components/ai-elements/web-search-result'; import { AppActionCard } from '@/components/ai-elements/app-action-card'; import { ComposioConnectCard } from '@/components/ai-elements/composio-connect-card'; @@ -76,10 +76,12 @@ import { getAppActionCardData, getComposioConnectCardData, getToolDisplayName, + groupConversationItems, inferRunTitleFromMessage, isChatMessage, isErrorMessage, isToolCall, + isToolGroup, normalizeToolInput, normalizeToolOutput, parseAttachedFiles, @@ -4578,7 +4580,20 @@ function App() { ) : ( <> - {tabState.conversation.map(item => { + {groupConversationItems( + tabState.conversation, + (id) => !!tabState.allPermissionRequests.get(id) + ).map(item => { + if (isToolGroup(item)) { + return ( + isToolOpenForTab(tab.id, toolId)} + onToolOpenChange={(toolId, open) => setToolOpenForTab(tab.id, toolId, open)} + /> + ) + } const rendered = renderConversationItem(item, tab.id) if (isToolCall(item)) { const permRequest = tabState.allPermissionRequests.get(item.id) diff --git a/apps/x/apps/renderer/src/components/ai-elements/tool.tsx b/apps/x/apps/renderer/src/components/ai-elements/tool.tsx index 66feb1c6..5f65fa32 100644 --- a/apps/x/apps/renderer/src/components/ai-elements/tool.tsx +++ b/apps/x/apps/renderer/src/components/ai-elements/tool.tsx @@ -17,6 +17,9 @@ import { XCircleIcon, } from "lucide-react"; import { type ComponentProps, type ReactNode, isValidElement, useState } from "react"; +import { AnimatePresence, motion } from "motion/react"; +import type { ToolCall, ToolGroup as ToolGroupType } from "@/lib/chat-conversation"; +import { getToolDisplayName, getToolGroupSummary, toToolState } from "@/lib/chat-conversation"; const formatToolValue = (value: unknown) => { if (typeof value === "string") return value; @@ -224,3 +227,89 @@ export const ToolTabbedContent = ({ ); }; + +export type ToolGroupProps = { + group: ToolGroupType + isToolOpen: (toolId: string) => boolean + onToolOpenChange: (toolId: string, open: boolean) => void +} + +const getGroupState = (tools: ToolCall[]): ToolUIPart["state"] => { + if (tools.some(t => t.status === 'error')) return 'output-error' + if (tools.some(t => t.status === 'running')) return 'input-available' + if (tools.some(t => t.status === 'pending')) return 'input-streaming' + return 'output-available' +} + +export const ToolGroupComponent = ({ group, isToolOpen, onToolOpenChange }: ToolGroupProps) => { + const [open, setOpen] = useState(false) + const state = getGroupState(group.items) + const isCompleted = state === 'output-available' || state === 'output-error' + const runningTool = group.items.find(t => t.status === 'running' || t.status === 'pending') + const currentTool = runningTool ?? group.items[group.items.length - 1] + const summary = isCompleted + ? `Ran ${group.items.length} tool${group.items.length !== 1 ? 's' : ''}` + : currentTool ? getToolDisplayName(currentTool) : getToolGroupSummary(group.items) + + return ( + + +
+ +
+ + + {summary} + + +
+
+
+ {getStatusBadge(state)} + +
+ + +
+ {group.items.map((tool) => { + const toolState = toToolState(tool.status) + const isOpen = isToolOpen(tool.id) + return ( + onToolOpenChange(tool.id, o)} + className="mb-0 border-border/60" + > + + + + + + ) + })} +
+ + + ) +} diff --git a/apps/x/apps/renderer/src/components/chat-sidebar.tsx b/apps/x/apps/renderer/src/components/chat-sidebar.tsx index 0a407d5d..07f1b637 100644 --- a/apps/x/apps/renderer/src/components/chat-sidebar.tsx +++ b/apps/x/apps/renderer/src/components/chat-sidebar.tsx @@ -16,7 +16,7 @@ import { MessageResponse, } from '@/components/ai-elements/message' import { Shimmer } from '@/components/ai-elements/shimmer' -import { Tool, ToolContent, ToolHeader, ToolTabbedContent } from '@/components/ai-elements/tool' +import { Tool, ToolContent, ToolGroupComponent, ToolHeader, ToolTabbedContent } from '@/components/ai-elements/tool' import { WebSearchResult } from '@/components/ai-elements/web-search-result' import { ComposioConnectCard } from '@/components/ai-elements/composio-connect-card' import { PermissionRequest } from '@/components/ai-elements/permission-request' @@ -40,9 +40,11 @@ import { getWebSearchCardData, getComposioConnectCardData, getToolDisplayName, + groupConversationItems, isChatMessage, isErrorMessage, isToolCall, + isToolGroup, normalizeToolInput, normalizeToolOutput, parseAttachedFiles, @@ -591,7 +593,20 @@ export function ChatSidebar({ ) : ( <> - {tabState.conversation.map((item) => { + {groupConversationItems( + tabState.conversation, + (id) => !!tabState.allPermissionRequests.get(id) + ).map((item) => { + if (isToolGroup(item)) { + return ( + isToolOpenForTab?.(tab.id, toolId) ?? false} + onToolOpenChange={(toolId, open) => onToolOpenChangeForTab?.(tab.id, toolId, open)} + /> + ) + } const rendered = renderConversationItem(item, tab.id) if (isToolCall(item) && onPermissionResponse) { const permRequest = tabState.allPermissionRequests.get(item.id) diff --git a/apps/x/apps/renderer/src/lib/chat-conversation.ts b/apps/x/apps/renderer/src/lib/chat-conversation.ts index 693961c9..150edacb 100644 --- a/apps/x/apps/renderer/src/lib/chat-conversation.ts +++ b/apps/x/apps/renderer/src/lib/chat-conversation.ts @@ -586,6 +586,72 @@ export const getComposioActionCardData = (tool: ToolCall): ComposioActionCardDat return null } +export type ToolGroup = { + type: 'tool-group' + items: ToolCall[] + groupId: string +} + +export type GroupedConversationItem = ConversationItem | ToolGroup + +export const isToolGroup = (item: GroupedConversationItem): item is ToolGroup => + 'type' in item && (item as ToolGroup).type === 'tool-group' + +const isPlainToolCall = (item: ConversationItem): item is ToolCall => { + if (!isToolCall(item)) return false + if (getWebSearchCardData(item)) return false + if (getComposioConnectCardData(item)) return false + if (getAppActionCardData(item)) return false + return true +} + +export const groupConversationItems = ( + items: ConversationItem[], + hasPermissionRequest: (id: string) => boolean +): GroupedConversationItem[] => { + const result: GroupedConversationItem[] = [] + let i = 0 + + while (i < items.length) { + const item = items[i] + if (isPlainToolCall(item) && !hasPermissionRequest(item.id)) { + const group: ToolCall[] = [item] + i++ + while ( + i < items.length && + isPlainToolCall(items[i] as ConversationItem) && + !hasPermissionRequest((items[i] as ToolCall).id) + ) { + group.push(items[i] as ToolCall) + i++ + } + if (group.length === 1) { + result.push(group[0]) + } else { + result.push({ type: 'tool-group', items: group, groupId: group[0].id }) + } + } else { + result.push(item) + i++ + } + } + + return result +} + +export const getToolGroupSummary = (tools: ToolCall[]): string => { + const seen = new Set() + const names: string[] = [] + for (const tool of tools) { + const name = getToolDisplayName(tool) + if (!seen.has(name)) { + seen.add(name) + names.push(name) + } + } + return names.join(' · ') +} + export const inferRunTitleFromMessage = (content: string): string | undefined => { const { message } = parseAttachedFiles(content) const normalized = message.replace(/\s+/g, ' ').trim() From 1c2b2ac1fc8d00fc7d4f09077a96b9e539e1b3a0 Mon Sep 17 00:00:00 2001 From: arkml <6592213+arkml@users.noreply.github.com> Date: Mon, 4 May 2026 15:47:30 +0530 Subject: [PATCH 02/22] feat: native desktop notifications + rowboat:// deep links Adds INotificationService with an Electron implementation, plus a deep-link dispatcher (rowboat://) for routing notification clicks back into the app. Notifications: - New `notify-user` skill + builtin tool. Title, message, optional primary link, optional secondary actions. Supports https:// (opens in browser) and rowboat:// (opens in app) targets. - ElectronNotificationService holds strong refs to active Notification instances so click handlers survive GC (otherwise macOS click silently no-ops). - Calendar meeting notifier fires 1-min warnings with "take notes" / "join + take notes" actions backed by deep links. Deep links (rowboat://): - forge.config.cjs declares the protocol; main.ts wires single-instance lock, setAsDefaultProtocolClient, open-url (mac), second-instance (win/ linux), and first-launch argv extraction. - New deeplink.ts dispatcher with dispatchUrl(url): main-handled actions (rowboat://action?type=...) vs renderer navigation (rowboat://open?...) via app:openUrl IPC. Includes pending-URL buffering for first-launch delivery before the renderer is ready. - Renderer parseDeepLink supports file / chat / graph / task / suggested-topics targets. - New app:consumePendingDeepLink IPC for renderer one-time drain on mount. Refactor: extractConferenceLink moved out of calendar-block.tsx into shared lib/calendar-event.ts (used by both the block and the take-notes deep-link handler) --- apps/x/apps/main/forge.config.cjs | 3 + apps/x/apps/main/src/deeplink.ts | 118 ++++++++++++ apps/x/apps/main/src/ipc.ts | 4 + apps/x/apps/main/src/main.ts | 54 +++++- .../electron-notification-service.ts | 84 ++++++++ apps/x/apps/renderer/src/App.tsx | 92 +++++++++ .../src/extensions/calendar-block.tsx | 20 +- .../x/apps/renderer/src/lib/calendar-event.ts | 15 ++ .../src/application/assistant/instructions.ts | 2 + .../src/application/assistant/skills/index.ts | 7 + .../assistant/skills/notify-user/skill.ts | 70 +++++++ .../core/src/application/lib/builtin-tools.ts | 41 ++++ .../src/application/notification/service.ts | 12 ++ apps/x/packages/core/src/di/container.ts | 7 + .../src/knowledge/notify_calendar_meetings.ts | 180 ++++++++++++++++++ .../core/src/knowledge/track/run-agent.ts | 1 + apps/x/packages/shared/src/ipc.ts | 22 +++ 17 files changed, 712 insertions(+), 20 deletions(-) create mode 100644 apps/x/apps/main/src/deeplink.ts create mode 100644 apps/x/apps/main/src/notification/electron-notification-service.ts create mode 100644 apps/x/apps/renderer/src/lib/calendar-event.ts create mode 100644 apps/x/packages/core/src/application/assistant/skills/notify-user/skill.ts create mode 100644 apps/x/packages/core/src/application/notification/service.ts create mode 100644 apps/x/packages/core/src/knowledge/notify_calendar_meetings.ts diff --git a/apps/x/apps/main/forge.config.cjs b/apps/x/apps/main/forge.config.cjs index 178cb7e1..ad639a86 100644 --- a/apps/x/apps/main/forge.config.cjs +++ b/apps/x/apps/main/forge.config.cjs @@ -11,6 +11,9 @@ module.exports = { icon: './icons/icon', // .icns extension added automatically appBundleId: 'com.rowboat.app', appCategoryType: 'public.app-category.productivity', + protocols: [ + { name: 'Rowboat', schemes: ['rowboat'] }, + ], extendInfo: { NSAudioCaptureUsageDescription: 'Rowboat needs access to system audio to transcribe meetings from other apps (Zoom, Meet, etc.)', }, diff --git a/apps/x/apps/main/src/deeplink.ts b/apps/x/apps/main/src/deeplink.ts new file mode 100644 index 00000000..605990d1 --- /dev/null +++ b/apps/x/apps/main/src/deeplink.ts @@ -0,0 +1,118 @@ +import { BrowserWindow } from "electron"; +import path from "node:path"; +import fs from "node:fs/promises"; +import { WorkDir } from "@x/core/dist/config/config.js"; + +export const DEEP_LINK_SCHEME = "rowboat"; +const URL_PREFIX = `${DEEP_LINK_SCHEME}://`; +const ACTION_HOST = "action"; + +let pendingUrl: string | null = null; +let mainWindowRef: BrowserWindow | null = null; + +export function setMainWindowForDeepLinks(win: BrowserWindow | null): void { + mainWindowRef = win; +} + +export function consumePendingDeepLink(): string | null { + const url = pendingUrl; + pendingUrl = null; + return url; +} + +export function extractDeepLinkFromArgv(argv: readonly string[]): string | null { + for (const arg of argv) { + if (typeof arg === "string" && arg.startsWith(URL_PREFIX)) return arg; + } + return null; +} + +/** + * Dispatch any rowboat:// URL — chooses navigation vs action automatically. + * Use this from notification click handlers and other URL entry points. + */ +export function dispatchUrl(url: string): void { + if (parseAction(url)) { + void dispatchAction(url); + } else { + dispatchDeepLink(url); + } +} + +export function dispatchDeepLink(url: string): void { + if (!url.startsWith(URL_PREFIX)) return; + + pendingUrl = url; + + const win = mainWindowRef; + if (!win || win.isDestroyed()) return; + focusWindow(win); + + if (win.webContents.isLoading()) return; + + win.webContents.send("app:openUrl", { url }); + pendingUrl = null; +} + +interface MeetingNotesAction { + type: "take-meeting-notes" | "join-and-take-meeting-notes"; + eventId: string; +} + +type ParsedAction = MeetingNotesAction; + +function parseAction(url: string): ParsedAction | null { + if (!url.startsWith(URL_PREFIX)) return null; + const rest = url.slice(URL_PREFIX.length); + const queryIdx = rest.indexOf("?"); + const host = (queryIdx >= 0 ? rest.slice(0, queryIdx) : rest).replace(/\/$/, ""); + if (host !== ACTION_HOST) return null; + const params = new URLSearchParams(queryIdx >= 0 ? rest.slice(queryIdx + 1) : ""); + const type = params.get("type"); + if (type === "take-meeting-notes" || type === "join-and-take-meeting-notes") { + const eventId = params.get("eventId"); + return eventId ? { type, eventId } : null; + } + return null; +} + +async function dispatchAction(url: string): Promise { + const parsed = parseAction(url); + if (!parsed) return; + + const openMeeting = parsed.type === "join-and-take-meeting-notes"; + await handleTakeMeetingNotes(parsed.eventId, openMeeting); +} + +async function handleTakeMeetingNotes(eventId: string, openMeeting: boolean): Promise { + const win = mainWindowRef; + if (!win || win.isDestroyed()) return; + focusWindow(win); + + const filePath = path.join(WorkDir, "calendar_sync", `${eventId}.json`); + let event: unknown; + try { + const raw = await fs.readFile(filePath, "utf-8"); + event = JSON.parse(raw); + } catch (err) { + console.error(`[deeplink] take-meeting-notes: failed to read ${filePath}`, err); + return; + } + + const payload = { event, openMeeting }; + + if (win.webContents.isLoading()) { + win.webContents.once("did-finish-load", () => { + win.webContents.send("app:takeMeetingNotes", payload); + }); + return; + } + + win.webContents.send("app:takeMeetingNotes", payload); +} + +function focusWindow(win: BrowserWindow): void { + if (win.isMinimized()) win.restore(); + win.show(); + win.focus(); +} diff --git a/apps/x/apps/main/src/ipc.ts b/apps/x/apps/main/src/ipc.ts index 5e62e8ee..d70192cc 100644 --- a/apps/x/apps/main/src/ipc.ts +++ b/apps/x/apps/main/src/ipc.ts @@ -34,6 +34,7 @@ import { triggerSync as triggerGranolaSync } from '@x/core/dist/knowledge/granol import { ISlackConfigRepo } from '@x/core/dist/slack/repo.js'; import { isOnboardingComplete, markOnboardingComplete } from '@x/core/dist/config/note_creation_config.js'; import * as composioHandler from './composio-handler.js'; +import { consumePendingDeepLink } from './deeplink.js'; import { IAgentScheduleRepo } from '@x/core/dist/agent-schedule/repo.js'; import { IAgentScheduleStateRepo } from '@x/core/dist/agent-schedule/state-repo.js'; import { triggerRun as triggerAgentScheduleRun } from '@x/core/dist/agent-schedule/runner.js'; @@ -417,6 +418,9 @@ export function setupIpcHandlers() { // args is null for this channel (no request payload) return getVersions(); }, + 'app:consumePendingDeepLink': async () => { + return { url: consumePendingDeepLink() }; + }, 'analytics:bootstrap': async () => { return { installationId: getInstallationId(), diff --git a/apps/x/apps/main/src/main.ts b/apps/x/apps/main/src/main.ts index 99c77589..cd0717a4 100644 --- a/apps/x/apps/main/src/main.ts +++ b/apps/x/apps/main/src/main.ts @@ -23,6 +23,7 @@ import { init as initNoteTagging } from "@x/core/dist/knowledge/tag_notes.js"; import { init as initInlineTasks } from "@x/core/dist/knowledge/inline_tasks.js"; import { init as initAgentRunner } from "@x/core/dist/agent-schedule/runner.js"; import { init as initAgentNotes } from "@x/core/dist/knowledge/agent_notes.js"; +import { init as initCalendarNotifications } from "@x/core/dist/knowledge/notify_calendar_meetings.js"; import { init as initTrackScheduler } from "@x/core/dist/knowledge/track/scheduler.js"; import { init as initTrackEventProcessor } from "@x/core/dist/knowledge/track/events.js"; import { init as initLocalSites, shutdown as shutdownLocalSites } from "@x/core/dist/local-sites/server.js"; @@ -34,10 +35,17 @@ import started from "electron-squirrel-startup"; import { execSync, exec, execFileSync } from "node:child_process"; import { promisify } from "node:util"; import { init as initChromeSync } from "@x/core/dist/knowledge/chrome-extension/server/server.js"; -import { registerBrowserControlService } from "@x/core/dist/di/container.js"; +import { registerBrowserControlService, registerNotificationService } from "@x/core/dist/di/container.js"; import { browserViewManager, BROWSER_PARTITION } from "./browser/view.js"; import { setupBrowserEventForwarding } from "./browser/ipc.js"; import { ElectronBrowserControlService } from "./browser/control-service.js"; +import { ElectronNotificationService } from "./notification/electron-notification-service.js"; +import { + DEEP_LINK_SCHEME, + dispatchDeepLink, + extractDeepLinkFromArgv, + setMainWindowForDeepLinks, +} from "./deeplink.js"; const execAsync = promisify(exec); @@ -47,6 +55,43 @@ const __dirname = dirname(__filename); // run this as early in the main process as possible if (started) app.quit(); +// Single-instance lock: route a second launch (e.g. clicking a rowboat:// link) +// back into the existing process via the 'second-instance' event. +if (!app.requestSingleInstanceLock()) { + app.quit(); + process.exit(0); +} + +// Register as the OS handler for rowboat:// URLs. +// In dev, point at the right argv so the OS can re-invoke us correctly. +if (process.defaultApp) { + if (process.argv.length >= 2) { + app.setAsDefaultProtocolClient(DEEP_LINK_SCHEME, process.execPath, [ + path.resolve(process.argv[1]), + ]); + } +} else { + app.setAsDefaultProtocolClient(DEEP_LINK_SCHEME); +} + +// First-launch URL on Windows/Linux comes through argv. +{ + const initialUrl = extractDeepLinkFromArgv(process.argv); + if (initialUrl) dispatchDeepLink(initialUrl); +} + +// macOS sends URLs via 'open-url' (both first launch and while running). +app.on("open-url", (event, url) => { + event.preventDefault(); + dispatchDeepLink(url); +}); + +// Subsequent launches on Windows/Linux land here via the single-instance lock. +app.on("second-instance", (_event, argv) => { + const url = extractDeepLinkFromArgv(argv); + if (url) dispatchDeepLink(url); +}); + // Fix PATH for packaged Electron apps on macOS/Linux. // Packaged apps inherit a minimal environment that doesn't include paths from // the user's shell profile (such as those provided by nvm, Homebrew, etc.). @@ -165,6 +210,9 @@ function createWindow() { configureSessionPermissions(session.defaultSession); configureSessionPermissions(session.fromPartition(BROWSER_PARTITION)); + setMainWindowForDeepLinks(win); + win.on("closed", () => setMainWindowForDeepLinks(null)); + // Show window when content is ready to prevent blank screen win.once("ready-to-show", () => { win.maximize(); @@ -240,6 +288,7 @@ app.whenReady().then(async () => { }); registerBrowserControlService(new ElectronBrowserControlService()); + registerNotificationService(new ElectronNotificationService()); setupIpcHandlers(); setupBrowserEventForwarding(); @@ -298,6 +347,9 @@ app.whenReady().then(async () => { // start agent notes learning service initAgentNotes(); + // start calendar meeting notification service (fires 1-minute warnings) + initCalendarNotifications(); + // start chrome extension sync server initChromeSync(); diff --git a/apps/x/apps/main/src/notification/electron-notification-service.ts b/apps/x/apps/main/src/notification/electron-notification-service.ts new file mode 100644 index 00000000..dd37e37d --- /dev/null +++ b/apps/x/apps/main/src/notification/electron-notification-service.ts @@ -0,0 +1,84 @@ +import { BrowserWindow, Notification, shell } from "electron"; +import type { INotificationService, NotifyInput } from "@x/core/dist/application/notification/service.js"; +import { dispatchUrl } from "../deeplink.js"; + +const HTTP_URL = /^https?:\/\//i; +const ROWBOAT_URL = /^rowboat:\/\//i; + +export class ElectronNotificationService implements INotificationService { + // Holds strong references to active Notification instances so the GC can't + // collect them while they're still visible — without this, the click handler + // gets dropped and macOS clicks just focus the app silently. + private active = new Set(); + + isSupported(): boolean { + return Notification.isSupported(); + } + + notify({ title = "Rowboat", message, link, actionLabel, secondaryActions }: NotifyInput): void { + // Build the actions array AND a parallel index → link map. + // macOS shows actions[0] inline (Banner) or all of them (Alert); + // additional ones live behind the chevron menu. + const actionDefs: Electron.NotificationConstructorOptions["actions"] = []; + const actionLinks: string[] = []; + + const primaryLabel = actionLabel?.trim(); + if (link && primaryLabel) { + actionDefs!.push({ type: "button", text: primaryLabel }); + actionLinks.push(link); + } + if (secondaryActions) { + for (const sa of secondaryActions) { + actionDefs!.push({ type: "button", text: sa.label }); + actionLinks.push(sa.link); + } + } + + const notification = new Notification({ + title, + body: message, + actions: actionDefs, + }); + + this.active.add(notification); + const release = () => { this.active.delete(notification); }; + + const openLink = (target: string | undefined) => { + if (target && ROWBOAT_URL.test(target)) { + dispatchUrl(target); + } else if (target && HTTP_URL.test(target)) { + shell.openExternal(target).catch((err) => { + console.error("[notification] failed to open link:", err); + }); + } else { + this.focusMainWindow(); + } + release(); + }; + + // Body click: always opens the primary `link` (or focuses the app if none). + notification.on("click", () => openLink(link)); + + // Action button click: dispatch by index into the actions array. + notification.on("action", (_event, index) => { + if (index >= 0 && index < actionLinks.length) { + openLink(actionLinks[index]); + } else { + openLink(undefined); + } + }); + + notification.on("close", release); + notification.on("failed", release); + + notification.show(); + } + + private focusMainWindow(): void { + const [win] = BrowserWindow.getAllWindows(); + if (!win) return; + if (win.isMinimized()) win.restore(); + win.show(); + win.focus(); + } +} diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index 0e925e2e..7c749664 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -54,6 +54,7 @@ import { Button } from "@/components/ui/button" import { Toaster } from "@/components/ui/sonner" import { stripKnowledgePrefix, toKnowledgePath, wikiLabel } from '@/lib/wiki-links' import { splitFrontmatter, joinFrontmatter } from '@/lib/frontmatter' +import { extractConferenceLink } from '@/lib/calendar-event' import { OnboardingModal } from '@/components/onboarding' import { CommandPalette, type CommandPaletteContext, type CommandPaletteMention } from '@/components/search-dialog' import { TrackModal } from '@/components/track-modal' @@ -515,6 +516,45 @@ function viewStatesEqual(a: ViewState, b: ViewState): boolean { return true // both graph } +/** + * Parse a rowboat:// deep link into a ViewState. Returns null if the URL is + * malformed or names an unknown target. + * + * Shape: rowboat://open?type=&... + * file: ?type=file&path=knowledge/foo.md + * chat: ?type=chat&runId=abc123 (runId optional) + * graph: ?type=graph + * task: ?type=task&name=daily-brief + * suggested-topics: ?type=suggested-topics + */ +function parseDeepLink(input: string): ViewState | null { + const SCHEME = 'rowboat://' + if (!input.startsWith(SCHEME)) return null + const rest = input.slice(SCHEME.length) + const queryIdx = rest.indexOf('?') + const host = (queryIdx >= 0 ? rest.slice(0, queryIdx) : rest).replace(/\/$/, '') + if (host !== 'open') return null + const params = new URLSearchParams(queryIdx >= 0 ? rest.slice(queryIdx + 1) : '') + switch (params.get('type')) { + case 'file': { + const path = params.get('path') + return path ? { type: 'file', path } : null + } + case 'chat': + return { type: 'chat', runId: params.get('runId') || null } + case 'graph': + return { type: 'graph' } + case 'task': { + const name = params.get('name') + return name ? { type: 'task', name } : null + } + case 'suggested-topics': + return { type: 'suggested-topics' } + default: + return null + } +} + /** Sidebar toggle (fixed position, top-left) */ function FixedSidebarToggle({ leftInsetPx, @@ -3050,6 +3090,58 @@ function App() { void navigateToView({ type: 'file', path }) }, [navigateToView]) + // Deep-link handler kept in a ref so the useEffect below can register the + // IPC listener (and run the one-time pending-link drain) just once on mount, + // rather than re-running on every navigation when navigateToView's identity + // changes. + const navigateToViewRef = useRef(navigateToView) + useEffect(() => { navigateToViewRef.current = navigateToView }, [navigateToView]) + + useEffect(() => { + const handle = (url: string) => { + const view = parseDeepLink(url) + if (view) void navigateToViewRef.current(view) + } + void window.ipc.invoke('app:consumePendingDeepLink', null).then(({ url }) => { + if (url) handle(url) + }) + return window.ipc.on('app:openUrl', ({ url }) => handle(url)) + }, []) + + // Triggered by main when the user clicks a calendar-meeting notification. + // Reuses the same flow as the in-app "Join meeting & take notes" button. + // When `openMeeting` is true, also opens the meeting URL in the system browser. + useEffect(() => { + return window.ipc.on('app:takeMeetingNotes', ({ event, openMeeting }) => { + const e = event as { + summary?: string + start?: { dateTime?: string; date?: string; timeZone?: string } + end?: { dateTime?: string; date?: string; timeZone?: string } + location?: string + htmlLink?: string + hangoutLink?: string + conferenceData?: { entryPoints?: Array<{ entryPointType?: string; uri?: string }> } + } + if (!e || typeof e !== 'object') return + const conferenceLink = extractConferenceLink(e as Record) + if (openMeeting && conferenceLink) { + window.open(conferenceLink, '_blank') + } else if (openMeeting) { + console.warn('[take-meeting-notes] openMeeting requested but event has no conference link', e) + } + window.__pendingCalendarEvent = { + summary: e.summary, + start: e.start, + end: e.end, + location: e.location, + htmlLink: e.htmlLink, + conferenceLink, + source: 'calendar-sync', + } + window.dispatchEvent(new Event('calendar-block:join-meeting')) + }) + }, []) + const handleBaseConfigChange = useCallback((path: string, config: BaseConfig) => { setBaseConfigByPath((prev) => ({ ...prev, [path]: config })) }, []) diff --git a/apps/x/apps/renderer/src/extensions/calendar-block.tsx b/apps/x/apps/renderer/src/extensions/calendar-block.tsx index 9f0eec02..ecc5403d 100644 --- a/apps/x/apps/renderer/src/extensions/calendar-block.tsx +++ b/apps/x/apps/renderer/src/extensions/calendar-block.tsx @@ -3,6 +3,7 @@ import { ReactNodeViewRenderer, NodeViewWrapper } from '@tiptap/react' import { X, Calendar, Video, ChevronDown, Mic } from 'lucide-react' import { blocks } from '@x/shared' import { useState, useEffect, useRef } from 'react' +import { extractConferenceLink } from '../lib/calendar-event' function formatTime(dateStr: string): string { const d = new Date(dateStr) @@ -40,25 +41,6 @@ function getTimeRange(event: blocks.CalendarEvent): string { return `${startTime} \u2013 ${endTime}` } -/** - * Extract a video conference link from raw Google Calendar event JSON. - * Checks conferenceData.entryPoints (video type), hangoutLink, then falls back - * to conferenceLink if already set. - */ -function extractConferenceLink(raw: Record): string | undefined { - // Check conferenceData.entryPoints for video entry - const confData = raw.conferenceData as { entryPoints?: { entryPointType?: string; uri?: string }[] } | undefined - if (confData?.entryPoints) { - const video = confData.entryPoints.find(ep => ep.entryPointType === 'video') - if (video?.uri) return video.uri - } - // Check hangoutLink (Google Meet shortcut) - if (typeof raw.hangoutLink === 'string') return raw.hangoutLink - // Fall back to conferenceLink if present - if (typeof raw.conferenceLink === 'string') return raw.conferenceLink - return undefined -} - interface ResolvedEvent { event: blocks.CalendarEvent loaded: blocks.CalendarEvent | null diff --git a/apps/x/apps/renderer/src/lib/calendar-event.ts b/apps/x/apps/renderer/src/lib/calendar-event.ts new file mode 100644 index 00000000..b7ace75a --- /dev/null +++ b/apps/x/apps/renderer/src/lib/calendar-event.ts @@ -0,0 +1,15 @@ +/** + * Extract a video conference link from raw Google Calendar event JSON. + * Checks conferenceData.entryPoints (video type), hangoutLink, then falls back + * to a top-level conferenceLink if present. + */ +export function extractConferenceLink(raw: Record): string | undefined { + const confData = raw.conferenceData as { entryPoints?: { entryPointType?: string; uri?: string }[] } | undefined + if (confData?.entryPoints) { + const video = confData.entryPoints.find(ep => ep.entryPointType === 'video') + if (video?.uri) return video.uri + } + if (typeof raw.hangoutLink === 'string') return raw.hangoutLink + if (typeof raw.conferenceLink === 'string') return raw.conferenceLink + return undefined +} diff --git a/apps/x/packages/core/src/application/assistant/instructions.ts b/apps/x/packages/core/src/application/assistant/instructions.ts index af2d7a20..a455d845 100644 --- a/apps/x/packages/core/src/application/assistant/instructions.ts +++ b/apps/x/packages/core/src/application/assistant/instructions.ts @@ -85,6 +85,8 @@ ${thirdPartyBlock}**Meeting Prep:** When users ask you to prepare for a meeting, **Tracks (Auto-Updating Note Blocks):** When users ask you to **track**, **monitor**, **watch**, or **keep an eye on** something in a note — or say things like "every morning tell me X", "show the current Y in this note", "pin live updates of Z here" — load the \`tracks\` skill first. Also load it when a user presses Cmd+K with a note open and requests auto-refreshing content at the cursor. Track blocks are YAML-fenced scheduled blocks whose output is rewritten on each run — useful for weather, news, prices, status pages, and personal dashboards. **Browser Control:** When users ask you to open a website, browse in-app, search the web in the embedded browser, or interact with a live webpage inside Rowboat, load the \`browser-control\` skill first. It explains the \`read-page -> indexed action -> refreshed page\` workflow for the browser pane. +**Notifications:** When you need to send a desktop notification — completion alert after a long task, time-sensitive update, or a clickable result that lands the user on a specific note/view — load the \`notify-user\` skill first. It documents the \`notify-user\` tool and the \`rowboat://\` deep links you can attach to it. + ## Learning About the User (save-to-memory) diff --git a/apps/x/packages/core/src/application/assistant/skills/index.ts b/apps/x/packages/core/src/application/assistant/skills/index.ts index cad23177..6d3cdc5b 100644 --- a/apps/x/packages/core/src/application/assistant/skills/index.ts +++ b/apps/x/packages/core/src/application/assistant/skills/index.ts @@ -14,6 +14,7 @@ import appNavigationSkill from "./app-navigation/skill.js"; import browserControlSkill from "./browser-control/skill.js"; import composioIntegrationSkill from "./composio-integration/skill.js"; import tracksSkill from "./tracks/skill.js"; +import notifyUserSkill from "./notify-user/skill.js"; const CURRENT_DIR = path.dirname(fileURLToPath(import.meta.url)); const CATALOG_PREFIX = "src/application/assistant/skills"; @@ -112,6 +113,12 @@ const definitions: SkillDefinition[] = [ summary: "Control the embedded browser pane - open sites, inspect page state, and interact with indexed page elements.", content: browserControlSkill, }, + { + id: "notify-user", + title: "Notify User", + summary: "Send native desktop notifications with optional clickable links — including rowboat:// deep links that open a specific note, chat, or view inside the app.", + content: notifyUserSkill, + }, ]; const skillEntries = definitions.map((definition) => ({ diff --git a/apps/x/packages/core/src/application/assistant/skills/notify-user/skill.ts b/apps/x/packages/core/src/application/assistant/skills/notify-user/skill.ts new file mode 100644 index 00000000..9bc619be --- /dev/null +++ b/apps/x/packages/core/src/application/assistant/skills/notify-user/skill.ts @@ -0,0 +1,70 @@ +export const skill = String.raw` +# Notify User + +Load this skill when you need to send a desktop notification to the user — e.g. after a long-running task completes, when a track block detects something noteworthy, or when an agent wants to ping the user with a clickable result. + +## When to use +- **Use it for**: completion alerts, threshold breaches, status changes, new items the user asked you to watch for, anything time-sensitive. +- **Don't use it for**: routine progress updates, anything the user can already see in the chat, or repeated pings inside a loop (there is no built-in rate limit — restraint is on you). + +## The tool: \`notify-user\` + +Triggers a native macOS notification. The call returns immediately; it does not block waiting for the user to click. + +### Parameters +- **\`title\`** (optional, defaults to \`"Rowboat"\`) — bold headline at the top. +- **\`message\`** (required) — body text. Keep it short — macOS truncates after a couple of lines. +- **\`link\`** (optional) — URL to open when the user clicks the notification. Two kinds accepted: + - **\`https://...\` / \`http://...\`** — opens in the default browser + - **\`rowboat://...\`** — opens a view inside Rowboat (see deep links below) + - If omitted, clicking the notification focuses the Rowboat app. + +### Examples + +Plain alert (no link — clicking focuses the app): +\`\`\`json +{ + "title": "Backup complete", + "message": "All 142 files synced to iCloud." +} +\`\`\` + +External link: +\`\`\`json +{ + "title": "New email from Monica", + "message": "Re: Q4 planning — needs your input by Friday", + "link": "https://mail.google.com/mail/u/0/#inbox/abc123" +} +\`\`\` + +Deep link into a Rowboat note: +\`\`\`json +{ + "message": "Daily brief is ready", + "link": "rowboat://open?type=file&path=knowledge/Daily/2026-04-25.md" +} +\`\`\` + +## Deep links: \`rowboat://\` + +Use these as the \`link\` parameter to land the user on a specific view in Rowboat instead of an external site. URL-encode paths/names that contain spaces or special characters. + +| Target | Format | Example | +|---|---|---| +| Open a file | \`rowboat://open?type=file&path=\` | \`rowboat://open?type=file&path=knowledge/People/Acme.md\` | +| Open chat | \`rowboat://open?type=chat\` (optional \`&runId=\`) | \`rowboat://open?type=chat&runId=abc123\` | +| Knowledge graph | \`rowboat://open?type=graph\` | — | +| Background task view | \`rowboat://open?type=task&name=\` | \`rowboat://open?type=task&name=daily-brief\` | +| Suggested topics | \`rowboat://open?type=suggested-topics\` | — | + +The \`type=file\` path is workspace-relative (the same path you'd pass to \`workspace-readFile\`). + +## Anti-patterns +- **Don't notify per step** of a multi-step task. Notify on completion, not on progress. +- **Don't repeat what's already on screen.** If the result is already in the chat or in a track block the user is viewing, skip the notification. +- **Don't dump the result into \`message\`.** Surface the headline; put the detail behind a deep link or external link. +- **Don't notify silently-failing things either.** If something failed, say so in the message — don't swallow the failure into a generic "done". +`; + +export default skill; diff --git a/apps/x/packages/core/src/application/lib/builtin-tools.ts b/apps/x/packages/core/src/application/lib/builtin-tools.ts index 4fd347b6..65b398a1 100644 --- a/apps/x/packages/core/src/application/lib/builtin-tools.ts +++ b/apps/x/packages/core/src/application/lib/builtin-tools.ts @@ -29,6 +29,7 @@ import { getAccessToken } from "../../auth/tokens.js"; import { API_URL } from "../../config/env.js"; import { updateContent, updateTrackBlock } from "../../knowledge/track/fileops.js"; import type { IBrowserControlService } from "../browser-control/service.js"; +import type { INotificationService } from "../notification/service.js"; // Parser libraries are loaded dynamically inside parseFile.execute() // to avoid pulling pdfjs-dist's DOM polyfills into the main bundle. // Import paths are computed so esbuild cannot statically resolve them. @@ -1526,4 +1527,44 @@ export const BuiltinTools: z.infer = { } }, }, + + 'notify-user': { + description: "Show a native OS notification to the user. Clicking the notification opens the provided link in the default browser, or focuses the Rowboat app if no link is given.", + inputSchema: z.object({ + title: z.string().min(1).max(120).optional().describe("Bold headline shown at the top of the notification. Defaults to 'Rowboat'."), + message: z.string().min(1).describe("Body text of the notification."), + link: z.string().url().refine((v) => /^(https?|rowboat):\/\//i.test(v), { + message: "link must be an http(s):// or rowboat:// URL", + }).optional().describe("Optional URL opened when the user clicks the notification. Accepts http(s):// (opens in browser) or rowboat:// (opens a view inside Rowboat — see the notify-user skill for deep-link shapes)."), + actionLabel: z.string().min(1).max(20).optional().describe("Optional label for an inline action button on the notification (e.g. 'Open', 'View', 'Take Notes'). Only shown when `link` is set. Click on the button triggers the same action as clicking the notification body."), + secondaryActions: z.array(z.object({ + label: z.string().min(1).max(30), + link: z.string().url().refine((v) => /^(https?|rowboat):\/\//i.test(v), { + message: "secondary action link must be an http(s):// or rowboat:// URL", + }), + })).max(4).optional().describe("Additional action buttons. macOS shows them in the chevron menu next to the primary button (or all inline in Alert style). Each has its own label and link — clicking the button triggers that link, independent of the primary `link`."), + }), + isAvailable: async () => { + try { + return container.resolve('notificationService').isSupported(); + } catch { + return false; + } + }, + execute: async ({ title, message, link, actionLabel, secondaryActions }: { title?: string; message: string; link?: string; actionLabel?: string; secondaryActions?: Array<{ label: string; link: string }> }) => { + try { + const service = container.resolve('notificationService'); + if (!service.isSupported()) { + return { success: false, error: 'Notifications are not supported on this system' }; + } + service.notify({ title, message, link, actionLabel, secondaryActions }); + return { success: true }; + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } + }, + }, }; diff --git a/apps/x/packages/core/src/application/notification/service.ts b/apps/x/packages/core/src/application/notification/service.ts new file mode 100644 index 00000000..195315b1 --- /dev/null +++ b/apps/x/packages/core/src/application/notification/service.ts @@ -0,0 +1,12 @@ +export interface NotifyInput { + title?: string; + message: string; + link?: string; + actionLabel?: string; + secondaryActions?: Array<{ label: string; link: string }>; +} + +export interface INotificationService { + isSupported(): boolean; + notify(input: NotifyInput): void; +} diff --git a/apps/x/packages/core/src/di/container.ts b/apps/x/packages/core/src/di/container.ts index 93ba9ebd..9382de8b 100644 --- a/apps/x/packages/core/src/di/container.ts +++ b/apps/x/packages/core/src/di/container.ts @@ -16,6 +16,7 @@ import { FSAgentScheduleRepo, IAgentScheduleRepo } from "../agent-schedule/repo. import { FSAgentScheduleStateRepo, IAgentScheduleStateRepo } from "../agent-schedule/state-repo.js"; import { FSSlackConfigRepo, ISlackConfigRepo } from "../slack/repo.js"; import type { IBrowserControlService } from "../application/browser-control/service.js"; +import type { INotificationService } from "../application/notification/service.js"; const container = createContainer({ injectionMode: InjectionMode.PROXY, @@ -49,3 +50,9 @@ export function registerBrowserControlService(service: IBrowserControlService): browserControlService: asValue(service), }); } + +export function registerNotificationService(service: INotificationService): void { + container.register({ + notificationService: asValue(service), + }); +} diff --git a/apps/x/packages/core/src/knowledge/notify_calendar_meetings.ts b/apps/x/packages/core/src/knowledge/notify_calendar_meetings.ts new file mode 100644 index 00000000..cca9d230 --- /dev/null +++ b/apps/x/packages/core/src/knowledge/notify_calendar_meetings.ts @@ -0,0 +1,180 @@ +import path from "node:path"; +import fs from "node:fs/promises"; +import type { Dirent } from "node:fs"; +import { WorkDir } from "../config/config.js"; +import container from "../di/container.js"; +import type { INotificationService } from "../application/notification/service.js"; + +const TICK_INTERVAL_MS = 30_000; +// Notify when an event is between 30s in the past (started just now) and +// 90s in the future (about to start). The window is wider than 60s so we +// don't miss an event if the tick lands slightly off the start time. +const NOTIFY_LEAD_MS = 90_000; +const NOTIFY_GRACE_MS = 30_000; +// Drop state entries older than 24h so the file doesn't grow forever. +const STATE_TTL_MS = 24 * 60 * 60 * 1000; + +const CALENDAR_SYNC_DIR = path.join(WorkDir, "calendar_sync"); +const STATE_FILE = path.join(WorkDir, "calendar_notifications_state.json"); + +interface NotificationState { + notifiedEventIds: Record; +} + +interface CalendarEvent { + id?: string; + summary?: string; + status?: string; + start?: { dateTime?: string; date?: string; timeZone?: string }; + end?: { dateTime?: string; date?: string }; + attendees?: Array<{ email?: string; self?: boolean; responseStatus?: string }>; + hangoutLink?: string; + conferenceData?: { + entryPoints?: Array<{ entryPointType?: string; uri?: string }>; + }; +} + +async function loadState(): Promise { + try { + const raw = await fs.readFile(STATE_FILE, "utf-8"); + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === "object" && parsed.notifiedEventIds) { + return parsed as NotificationState; + } + } catch { + // No state file yet, or corrupt — start fresh. + } + return { notifiedEventIds: {} }; +} + +async function saveState(state: NotificationState): Promise { + // Write to a sibling tmp file then rename so a mid-write crash can't leave + // the JSON corrupt — a corrupt state file would make every event in the + // 90s notify window re-fire on next start. + const tmp = `${STATE_FILE}.tmp`; + await fs.writeFile(tmp, JSON.stringify(state, null, 2), "utf-8"); + await fs.rename(tmp, STATE_FILE); +} + +function gcState(state: NotificationState): NotificationState { + const cutoff = Date.now() - STATE_TTL_MS; + const fresh: NotificationState["notifiedEventIds"] = {}; + for (const [id, entry] of Object.entries(state.notifiedEventIds)) { + const ts = Date.parse(entry.notifiedAt); + if (Number.isFinite(ts) && ts >= cutoff) fresh[id] = entry; + } + return { notifiedEventIds: fresh }; +} + +function isAllDay(event: CalendarEvent): boolean { + // Google Calendar all-day events have `date` (YYYY-MM-DD) on start, not `dateTime`. + return !event.start?.dateTime; +} + +function isDeclinedBySelf(event: CalendarEvent): boolean { + if (!event.attendees) return false; + const self = event.attendees.find((a) => a.self); + return self?.responseStatus === "declined"; +} + +async function tick(state: NotificationState): Promise<{ state: NotificationState; dirty: boolean }> { + let entries: Dirent[]; + try { + entries = await fs.readdir(CALENDAR_SYNC_DIR, { withFileTypes: true }); + } catch { + return { state, dirty: false }; + } + + let service: INotificationService; + try { + service = container.resolve("notificationService"); + } catch { + // Notification service not registered yet (very early startup) — skip this tick. + return { state, dirty: false }; + } + if (!service.isSupported()) return { state, dirty: false }; + + const now = Date.now(); + let dirty = false; + + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith(".json")) continue; + if (entry.name === "sync_state.json" || entry.name.startsWith("sync_state")) continue; + + const eventId = entry.name.replace(/\.json$/, ""); + if (state.notifiedEventIds[eventId]) continue; + + const filePath = path.join(CALENDAR_SYNC_DIR, entry.name); + let event: CalendarEvent; + try { + event = JSON.parse(await fs.readFile(filePath, "utf-8")); + } catch { + continue; + } + + if (event.status === "cancelled") continue; + if (isAllDay(event)) continue; + if (isDeclinedBySelf(event)) continue; + + const startStr = event.start?.dateTime; + if (!startStr) continue; + const startMs = Date.parse(startStr); + if (!Number.isFinite(startMs)) continue; + + const msUntilStart = startMs - now; + if (msUntilStart > NOTIFY_LEAD_MS) continue; + if (msUntilStart < -NOTIFY_GRACE_MS) continue; + + const summary = event.summary?.trim() || "Untitled meeting"; + const eid = encodeURIComponent(eventId); + + try { + service.notify({ + title: "Upcoming meeting", + message: `${summary} starts in 1 minute. Click to join and take notes.`, + // Single labeled button — adding a secondary action would force + // macOS to bundle them into an "Options" dropdown, hiding the + // primary label. + link: `rowboat://action?type=join-and-take-meeting-notes&eventId=${eid}`, + actionLabel: "Join & Notes", + }); + console.log(`[CalendarNotify] notified for "${summary}" (${eventId})`); + } catch (err) { + console.error(`[CalendarNotify] notify failed for ${eventId}:`, err); + continue; + } + + state.notifiedEventIds[eventId] = { + notifiedAt: new Date().toISOString(), + startTime: startStr, + }; + dirty = true; + } + + return { state, dirty }; +} + +export async function init(): Promise { + console.log("[CalendarNotify] starting calendar notification service"); + console.log(`[CalendarNotify] tick every ${TICK_INTERVAL_MS / 1000}s`); + + let state = gcState(await loadState()); + + while (true) { + try { + const result = await tick(state); + state = result.state; + if (result.dirty) { + state = gcState(state); + try { + await saveState(state); + } catch (err) { + console.error("[CalendarNotify] failed to save state:", err); + } + } + } catch (err) { + console.error("[CalendarNotify] tick failed:", err); + } + await new Promise((resolve) => setTimeout(resolve, TICK_INTERVAL_MS)); + } +} diff --git a/apps/x/packages/core/src/knowledge/track/run-agent.ts b/apps/x/packages/core/src/knowledge/track/run-agent.ts index d93366f3..685305b2 100644 --- a/apps/x/packages/core/src/knowledge/track/run-agent.ts +++ b/apps/x/packages/core/src/knowledge/track/run-agent.ts @@ -263,6 +263,7 @@ You have the full workspace toolkit. Quick reference for common cases: - **\`parseFile\`, \`LLMParse\`** — parse PDFs, spreadsheets, Word docs if a track aggregates from attached files. - **\`composio-*\`, \`listMcpTools\`, \`executeMcpTool\`** — user-connected integrations (Gmail, Calendar, etc.). Prefer these when a track needs structured data from a connected service the user has authorized. - **\`browser-control\`** — only when a required source has no API / search alternative and requires JS rendering. +- **\`notify-user\`** — send a native desktop notification when this run produces something time-sensitive (threshold breach, urgent change, "the thing the user asked you to watch for just happened"). Skip it for routine refreshes — the note itself is the artifact. Load the \`notify-user\` skill via \`loadSkill\` for parameters and \`rowboat://\` deep-link shapes (so the click lands on the right note/view). # The Knowledge Graph diff --git a/apps/x/packages/shared/src/ipc.ts b/apps/x/packages/shared/src/ipc.ts index 575f8395..ab7d7f73 100644 --- a/apps/x/packages/shared/src/ipc.ts +++ b/apps/x/packages/shared/src/ipc.ts @@ -299,6 +299,28 @@ const ipcSchemas = { }), res: z.null(), }, + 'app:openUrl': { + req: z.object({ + url: z.string(), + }), + res: z.null(), + }, + 'app:takeMeetingNotes': { + req: z.object({ + // Pass the raw calendar event JSON through; renderer adapts to its existing flow. + event: z.unknown(), + // When true, the renderer should also open the meeting URL (Zoom/Meet/etc.) + // in addition to triggering the take-notes flow. + openMeeting: z.boolean().optional(), + }), + res: z.null(), + }, + 'app:consumePendingDeepLink': { + req: z.null(), + res: z.object({ + url: z.string().nullable(), + }), + }, 'granola:getConfig': { req: z.null(), res: z.object({ From 93feee15a051a0839141d2850c55e6fa80da7e0d Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Mon, 4 May 2026 17:20:19 +0530 Subject: [PATCH 03/22] fixed collapsed sidebar issue on chat --- apps/x/apps/renderer/src/App.tsx | 1 + .../x/apps/renderer/src/components/chat-sidebar.tsx | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index 7c749664..0321aaed 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -4850,6 +4850,7 @@ function App() { onToolOpenChangeForTab={setToolOpenForTab} onOpenKnowledgeFile={(path) => { navigateToFile(path) }} onActivate={() => setActiveShortcutPane('right')} + collapsedLeftPaddingPx={collapsedLeftPaddingPx} isRecording={isRecording} recordingText={voice.interimText} recordingState={voice.state === 'connecting' ? 'connecting' : 'listening'} diff --git a/apps/x/apps/renderer/src/components/chat-sidebar.tsx b/apps/x/apps/renderer/src/components/chat-sidebar.tsx index 07f1b637..6fa295b1 100644 --- a/apps/x/apps/renderer/src/components/chat-sidebar.tsx +++ b/apps/x/apps/renderer/src/components/chat-sidebar.tsx @@ -30,6 +30,7 @@ import remarkBreaks from 'remark-breaks' import { TabBar, type ChatTab } from '@/components/tab-bar' import { ChatInputWithMentions, type StagedAttachment, type SelectedModel } from '@/components/chat-input-with-mentions' import { ChatMessageAttachments } from '@/components/chat-message-attachments' +import { useSidebar } from '@/components/ui/sidebar' import { wikiLabel } from '@/lib/wiki-links' import { type ChatViewportAnchorState, @@ -177,6 +178,7 @@ interface ChatSidebarProps { onToolOpenChangeForTab?: (tabId: string, toolId: string, open: boolean) => void onOpenKnowledgeFile?: (path: string) => void onActivate?: () => void + collapsedLeftPaddingPx?: number // Voice / TTS props isRecording?: boolean recordingText?: string @@ -231,6 +233,7 @@ export function ChatSidebar({ onToolOpenChangeForTab, onOpenKnowledgeFile, onActivate, + collapsedLeftPaddingPx = 196, isRecording, recordingText, recordingState, @@ -245,6 +248,7 @@ export function ChatSidebar({ onTtsModeChange, onComposioConnected, }: ChatSidebarProps) { + const { state: sidebarState } = useSidebar() const [width, setWidth] = useState(() => getInitialPaneWidth(defaultWidth)) const [isResizing, setIsResizing] = useState(false) const [showContent, setShowContent] = useState(isOpen) @@ -519,7 +523,14 @@ export function ChatSidebar({ {showContent && ( <> -
+
Date: Mon, 4 May 2026 17:28:04 +0530 Subject: [PATCH 04/22] fix browser reload issue --- apps/x/apps/main/src/browser/view.ts | 35 ++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/apps/x/apps/main/src/browser/view.ts b/apps/x/apps/main/src/browser/view.ts index d319c5fb..b540809d 100644 --- a/apps/x/apps/main/src/browser/view.ts +++ b/apps/x/apps/main/src/browser/view.ts @@ -109,10 +109,31 @@ export class BrowserViewManager extends EventEmitter { private visible = false; private bounds: BrowserBounds = { x: 0, y: 0, width: 0, height: 0 }; private snapshotCache = new Map(); + private cleanupWindowListeners: (() => void) | null = null; attach(window: BrowserWindow): void { + this.cleanupWindowListeners?.(); this.window = window; - window.on('closed', () => { + + const resetForHostWindowNavigation = () => { + // Renderer refreshes do not run React unmount cleanup reliably, so the + // native browser view must be detached from the main process side. + this.visible = false; + this.bounds = { x: 0, y: 0, width: 0, height: 0 }; + this.syncAttachedView(); + }; + + const handleDidStartLoading = () => { + resetForHostWindowNavigation(); + }; + + const handleRenderProcessGone = () => { + resetForHostWindowNavigation(); + }; + + const handleClosed = () => { + this.cleanupWindowListeners?.(); + this.cleanupWindowListeners = null; this.window = null; this.browserSession = null; this.tabs.clear(); @@ -121,7 +142,17 @@ export class BrowserViewManager extends EventEmitter { this.attachedTabId = null; this.visible = false; this.snapshotCache.clear(); - }); + }; + + window.webContents.on('did-start-loading', handleDidStartLoading); + window.webContents.on('render-process-gone', handleRenderProcessGone); + window.on('closed', handleClosed); + + this.cleanupWindowListeners = () => { + window.webContents.removeListener('did-start-loading', handleDidStartLoading); + window.webContents.removeListener('render-process-gone', handleRenderProcessGone); + window.removeListener('closed', handleClosed); + }; } private getSession(): Session { From a76f8bae14a4f7208d513144a19dd2ce539bfb1b Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Tue, 5 May 2026 11:41:08 +0530 Subject: [PATCH 05/22] fix sticky browser issue --- apps/x/apps/renderer/src/App.tsx | 37 ++++++++++++++----- .../components/browser-pane/BrowserPane.tsx | 11 +++++- .../src/components/sidebar-content.tsx | 27 +++++++++++--- 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index 0321aaed..d0ed5284 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -2400,6 +2400,10 @@ function App() { } }, [runId]) + const dismissBrowserOverlay = useCallback(() => { + setIsBrowserOpen(false) + }, []) + const handleNewChat = useCallback(() => { // Invalidate any in-flight run loads (rapid switching can otherwise "pop" old conversations back in) loadRunRequestIdRef.current += 1 @@ -2623,6 +2627,7 @@ function App() { // File tab operations const openFileInNewTab = useCallback((path: string) => { + dismissBrowserOverlay() const existingTab = fileTabs.find(t => t.path === path) if (existingTab) { setActiveFileTabId(existingTab.id) @@ -2635,11 +2640,12 @@ function App() { setActiveFileTabId(id) setIsGraphOpen(false) setSelectedPath(path) - }, [fileTabs]) + }, [fileTabs, dismissBrowserOverlay]) const switchFileTab = useCallback((tabId: string) => { const tab = fileTabs.find(t => t.id === tabId) if (!tab) return + dismissBrowserOverlay() setActiveFileTabId(tabId) setSelectedBackgroundTask(null) setExpandedFrom(null) @@ -2662,7 +2668,7 @@ function App() { setIsGraphOpen(false) setIsSuggestedTopicsOpen(false) setSelectedPath(tab.path) - }, [fileTabs, isRightPaneMaximized]) + }, [fileTabs, isRightPaneMaximized, dismissBrowserOverlay]) const closeFileTab = useCallback((tabId: string) => { const closingTab = fileTabs.find(t => t.id === tabId) @@ -2734,8 +2740,9 @@ function App() { // Create a new tab const id = newChatTabId() setChatTabs(prev => [...prev, { id, runId: null }]) - setActiveChatTabId(id) + setActiveChatTabId(id) } + dismissBrowserOverlay() handleNewChat() // Left-pane "new chat" should always open full chat view. if (selectedPath || isGraphOpen || isSuggestedTopicsOpen) { @@ -2747,7 +2754,7 @@ function App() { setSelectedPath(null) setIsGraphOpen(false) setIsSuggestedTopicsOpen(false) - }, [chatTabs, activeChatTabId, handleNewChat, selectedPath, isGraphOpen, isSuggestedTopicsOpen]) + }, [chatTabs, activeChatTabId, dismissBrowserOverlay, handleNewChat, selectedPath, isGraphOpen, isSuggestedTopicsOpen]) // Sidebar variant: create/switch chat tab without leaving file/graph context. const handleNewChatTabInSidebar = useCallback(() => { @@ -2865,11 +2872,12 @@ function App() { if (selectedPath || isGraphOpen || isSuggestedTopicsOpen) { setExpandedFrom({ path: selectedPath, graph: isGraphOpen, suggestedTopics: isSuggestedTopicsOpen }) } + dismissBrowserOverlay() setIsRightPaneMaximized(false) setSelectedPath(null) setIsGraphOpen(false) setIsSuggestedTopicsOpen(false) - }, [selectedPath, isGraphOpen, isSuggestedTopicsOpen]) + }, [selectedPath, isGraphOpen, isSuggestedTopicsOpen, dismissBrowserOverlay]) const handleCloseFullScreenChat = useCallback(() => { if (expandedFrom) { @@ -3004,8 +3012,7 @@ function App() { case 'chat': setSelectedPath(null) setIsGraphOpen(false) - // Don't touch isBrowserOpen here — chat navigation should land in - // the right sidebar when the browser overlay is active. + setIsBrowserOpen(false) setExpandedFrom(null) setIsRightPaneMaximized(false) setSelectedBackgroundTask(null) @@ -3021,7 +3028,12 @@ function App() { const navigateToView = useCallback(async (nextView: ViewState) => { const current = currentViewState - if (viewStatesEqual(current, nextView)) return + if (viewStatesEqual(current, nextView)) { + if (isBrowserOpen) { + dismissBrowserOverlay() + } + return + } cancelRecordingIfActive() const nextHistory = { @@ -3030,7 +3042,7 @@ function App() { } setHistory(nextHistory) await applyViewState(nextView) - }, [appendUnique, applyViewState, cancelRecordingIfActive, currentViewState, setHistory]) + }, [appendUnique, applyViewState, cancelRecordingIfActive, currentViewState, setHistory, isBrowserOpen, dismissBrowserOverlay]) const navigateBack = useCallback(async () => { const { back, forward } = historyRef.current @@ -4329,6 +4341,8 @@ function App() { meetingSummarizing={meetingSummarizing} meetingAvailable={voiceAvailable} onToggleMeeting={() => { void handleToggleMeeting() }} + isSearchOpen={isSearchOpen} + isMeetingActionActive={showMeetingPermissions || meetingSummarizing || meetingTranscription.state !== 'idle'} isBrowserOpen={isBrowserOpen} onToggleBrowser={handleToggleBrowser} isSuggestedTopicsOpen={isSuggestedTopicsOpen} @@ -4463,7 +4477,10 @@ function App() { {isBrowserOpen ? ( - + ) : isSuggestedTopicsOpen ? (
void + forceHidden?: boolean } const getActiveTab = (state: BrowserState) => @@ -85,7 +86,7 @@ const getBrowserTabTitle = (tab: BrowserTabState) => { } } -export function BrowserPane({ onClose }: BrowserPaneProps) { +export function BrowserPane({ onClose, forceHidden = false }: BrowserPaneProps) { const [state, setState] = useState(EMPTY_STATE) const [addressValue, setAddressValue] = useState('') @@ -175,6 +176,12 @@ export function BrowserPane({ onClose }: BrowserPaneProps) { }, []) const syncView = useCallback(() => { + if (forceHidden) { + lastBoundsRef.current = null + setViewVisible(false) + return null + } + const doc = viewportRef.current?.ownerDocument if (doc && hasBlockingOverlay(doc)) { lastBoundsRef.current = null @@ -191,7 +198,7 @@ export function BrowserPane({ onClose }: BrowserPaneProps) { pushBounds(bounds) setViewVisible(true) return bounds - }, [measureBounds, pushBounds, setViewVisible]) + }, [forceHidden, measureBounds, pushBounds, setViewVisible]) useEffect(() => { syncView() diff --git a/apps/x/apps/renderer/src/components/sidebar-content.tsx b/apps/x/apps/renderer/src/components/sidebar-content.tsx index 41d6b622..7e204781 100644 --- a/apps/x/apps/renderer/src/components/sidebar-content.tsx +++ b/apps/x/apps/renderer/src/components/sidebar-content.tsx @@ -186,6 +186,8 @@ type SidebarContentPanelProps = { meetingSummarizing?: boolean meetingAvailable?: boolean onToggleMeeting?: () => void + isSearchOpen?: boolean + isMeetingActionActive?: boolean isBrowserOpen?: boolean onToggleBrowser?: () => void isSuggestedTopicsOpen?: boolean @@ -420,6 +422,8 @@ export function SidebarContentPanel({ meetingSummarizing = false, meetingAvailable = false, onToggleMeeting, + isSearchOpen = false, + isMeetingActionActive = false, isBrowserOpen = false, onToggleBrowser, isSuggestedTopicsOpen = false, @@ -436,6 +440,9 @@ export function SidebarContentPanel({ const [loggingIn, setLoggingIn] = useState(false) const [appUrl, setAppUrl] = useState(null) const { billing } = useBilling(isRowboatConnected) + const isMeetingQuickActionSelected = isMeetingActionActive + const isBrowserQuickActionSelected = isBrowserOpen && !isSearchOpen && !isMeetingQuickActionSelected + const isSuggestedTopicsQuickActionSelected = isSuggestedTopicsOpen && !isBrowserOpen const handleRowboatLogin = useCallback(async () => { try { @@ -533,7 +540,12 @@ export function SidebarContentPanel({ + +
+ + + ) +} diff --git a/apps/x/apps/renderer/src/components/onboarding-modal.tsx b/apps/x/apps/renderer/src/components/onboarding-modal.tsx index 469ac35d..33c89231 100644 --- a/apps/x/apps/renderer/src/components/onboarding-modal.tsx +++ b/apps/x/apps/renderer/src/components/onboarding-modal.tsx @@ -96,14 +96,14 @@ export function OnboardingModal({ open, onComplete }: OnboardingModalProps) { const [slackDiscovering, setSlackDiscovering] = useState(false) const [slackDiscoverError, setSlackDiscoverError] = useState(null) - // Composio/Gmail state - const [useComposioForGoogle, setUseComposioForGoogle] = useState(false) + // Composio Gmail/Calendar sync was removed — flags are seeded false and + // never flipped. Kept here so legacy gating expressions still type-check. + const [useComposioForGoogle] = useState(false) const [gmailConnected, setGmailConnected] = useState(false) const [gmailLoading, setGmailLoading] = useState(true) const [gmailConnecting, setGmailConnecting] = useState(false) - // Composio/Google Calendar state - const [useComposioForGoogleCalendar, setUseComposioForGoogleCalendar] = useState(false) + const [useComposioForGoogleCalendar] = useState(false) const [googleCalendarConnected, setGoogleCalendarConnected] = useState(false) const [googleCalendarLoading, setGoogleCalendarLoading] = useState(true) const [googleCalendarConnecting, setGoogleCalendarConnecting] = useState(false) @@ -151,25 +151,8 @@ export function OnboardingModal({ open, onComplete }: OnboardingModalProps) { setProvidersLoading(false) } } - async function loadComposioForGoogleFlag() { - try { - const result = await window.ipc.invoke('composio:use-composio-for-google', null) - setUseComposioForGoogle(result.enabled) - } catch (error) { - console.error('Failed to check composio-for-google flag:', error) - } - } - async function loadComposioForGoogleCalendarFlag() { - try { - const result = await window.ipc.invoke('composio:use-composio-for-google-calendar', null) - setUseComposioForGoogleCalendar(result.enabled) - } catch (error) { - console.error('Failed to check composio-for-google-calendar flag:', error) - } - } + // (Composio Gmail/Calendar flag fetches removed — sync was deleted.) loadProviders() - loadComposioForGoogleFlag() - loadComposioForGoogleCalendarFlag() }, [open]) // Load LLM models catalog on open @@ -622,12 +605,20 @@ export function OnboardingModal({ open, onComplete }: OnboardingModalProps) { // Connect to a provider const handleConnect = useCallback(async (provider: string) => { if (provider === 'google') { + // Signed-in users use the rowboat (managed-credentials) flow: opens + // the webapp in the browser, no BYOK modal. Falls back to BYOK modal + // for not-signed-in users. (Mirrors useConnectors.handleConnect.) + const isSignedIntoRowboat = providerStates.rowboat?.isConnected ?? false + if (isSignedIntoRowboat) { + await startConnect('google') + return + } setGoogleClientIdOpen(true) return } await startConnect(provider) - }, [startConnect]) + }, [startConnect, providerStates]) const handleGoogleClientIdSubmit = useCallback((clientId: string, clientSecret: string) => { setGoogleCredentials(clientId, clientSecret) diff --git a/apps/x/apps/renderer/src/components/onboarding/use-onboarding-state.ts b/apps/x/apps/renderer/src/components/onboarding/use-onboarding-state.ts index edb3616b..b06ec862 100644 --- a/apps/x/apps/renderer/src/components/onboarding/use-onboarding-state.ts +++ b/apps/x/apps/renderer/src/components/onboarding/use-onboarding-state.ts @@ -66,16 +66,16 @@ export function useOnboardingState(open: boolean, onComplete: () => void) { // Inline upsell callout dismissed const [upsellDismissed, setUpsellDismissed] = useState(false) - // Composio/Gmail state (used when signed in with Rowboat account) - const [useComposioForGoogle, setUseComposioForGoogle] = useState(false) + // Composio Gmail/Calendar sync was removed — flags are seeded false and + // never flipped. Kept here so legacy gating expressions still type-check. + const [useComposioForGoogle] = useState(false) const [gmailConnected, setGmailConnected] = useState(false) const [gmailLoading, setGmailLoading] = useState(true) const [gmailConnecting, setGmailConnecting] = useState(false) const [composioApiKeyOpen, setComposioApiKeyOpen] = useState(false) const [composioApiKeyTarget, setComposioApiKeyTarget] = useState<'slack' | 'gmail'>('gmail') - // Composio/Google Calendar state - const [useComposioForGoogleCalendar, setUseComposioForGoogleCalendar] = useState(false) + const [useComposioForGoogleCalendar] = useState(false) const [googleCalendarConnected, setGoogleCalendarConnected] = useState(false) const [googleCalendarLoading, setGoogleCalendarLoading] = useState(true) const [googleCalendarConnecting, setGoogleCalendarConnecting] = useState(false) @@ -123,25 +123,8 @@ export function useOnboardingState(open: boolean, onComplete: () => void) { setProvidersLoading(false) } } - async function loadComposioForGoogleFlag() { - try { - const result = await window.ipc.invoke('composio:use-composio-for-google', null) - setUseComposioForGoogle(result.enabled) - } catch (error) { - console.error('Failed to check composio-for-google flag:', error) - } - } - async function loadComposioForGoogleCalendarFlag() { - try { - const result = await window.ipc.invoke('composio:use-composio-for-google-calendar', null) - setUseComposioForGoogleCalendar(result.enabled) - } catch (error) { - console.error('Failed to check composio-for-google-calendar flag:', error) - } - } + // (Composio Gmail/Calendar flag fetches removed — sync was deleted; flags stay false.) loadProviders() - loadComposioForGoogleFlag() - loadComposioForGoogleCalendarFlag() }, [open]) // Load LLM models catalog on open @@ -539,17 +522,7 @@ export function useOnboardingState(open: boolean, onComplete: () => void) { const cleanup = window.ipc.on('oauth:didConnect', async (event) => { if (event.provider === 'rowboat' && event.success) { - // Re-check composio flags now that the account is connected - try { - const [googleResult, calendarResult] = await Promise.all([ - window.ipc.invoke('composio:use-composio-for-google', null), - window.ipc.invoke('composio:use-composio-for-google-calendar', null), - ]) - setUseComposioForGoogle(googleResult.enabled) - setUseComposioForGoogleCalendar(calendarResult.enabled) - } catch (error) { - console.error('Failed to re-check composio flags:', error) - } + // (Composio Gmail/Calendar flag re-check removed — sync was deleted.) setCurrentStep(2) // Go to Connect Accounts } }) @@ -609,12 +582,20 @@ export function useOnboardingState(open: boolean, onComplete: () => void) { // Connect to a provider const handleConnect = useCallback(async (provider: string) => { if (provider === 'google') { + // Signed-in users use the rowboat (managed-credentials) flow: opens + // the webapp in the browser, no BYOK modal. Falls back to BYOK modal + // for not-signed-in users. (Mirrors useConnectors.handleConnect.) + const isSignedIntoRowboat = providerStates.rowboat?.isConnected ?? false + if (isSignedIntoRowboat) { + await startConnect('google') + return + } setGoogleClientIdOpen(true) return } await startConnect(provider) - }, [startConnect]) + }, [startConnect, providerStates]) const handleGoogleClientIdSubmit = useCallback((clientId: string, clientSecret: string) => { setGoogleCredentials(clientId, clientSecret) diff --git a/apps/x/apps/renderer/src/hooks/useConnectors.ts b/apps/x/apps/renderer/src/hooks/useConnectors.ts index 7285fe04..af56b921 100644 --- a/apps/x/apps/renderer/src/hooks/useConnectors.ts +++ b/apps/x/apps/renderer/src/hooks/useConnectors.ts @@ -38,16 +38,21 @@ export function useConnectors(active: boolean) { const [slackDiscovering, setSlackDiscovering] = useState(false) const [slackDiscoverError, setSlackDiscoverError] = useState(null) - // Composio/Gmail state - const [useComposioForGoogle, setUseComposioForGoogle] = useState(false) + // Composio Gmail/Calendar sync was removed. These flags are seeded false + // and never flipped — the IPC that used to set them is gone. The setters + // remain so the legacy Composio-Gmail handlers below still type-check, + // but those handlers are no longer reachable in the UI (the gating + // condition `useComposioForGoogle` stays false). + // TODO follow-up: drop these flags entirely and prune the dead UI branches + // in connectors-popover, connected-accounts-settings, and onboarding-modal. + const [useComposioForGoogle] = useState(false) const [gmailConnected, setGmailConnected] = useState(false) - const [gmailLoading, setGmailLoading] = useState(true) + const [gmailLoading, setGmailLoading] = useState(false) const [gmailConnecting, setGmailConnecting] = useState(false) - // Composio/Google Calendar state - const [useComposioForGoogleCalendar, setUseComposioForGoogleCalendar] = useState(false) + const [useComposioForGoogleCalendar] = useState(false) const [googleCalendarConnected, setGoogleCalendarConnected] = useState(false) - const [googleCalendarLoading, setGoogleCalendarLoading] = useState(true) + const [googleCalendarLoading, setGoogleCalendarLoading] = useState(false) const [googleCalendarConnecting, setGoogleCalendarConnecting] = useState(false) // Load available providers on mount @@ -67,28 +72,7 @@ export function useConnectors(active: boolean) { loadProviders() }, []) - // Re-check composio-for-google flags when active - useEffect(() => { - if (!active) return - async function loadComposioForGoogleFlag() { - try { - const result = await window.ipc.invoke('composio:use-composio-for-google', null) - setUseComposioForGoogle(result.enabled) - } catch (error) { - console.error('Failed to check composio-for-google flag:', error) - } - } - async function loadComposioForGoogleCalendarFlag() { - try { - const result = await window.ipc.invoke('composio:use-composio-for-google-calendar', null) - setUseComposioForGoogleCalendar(result.enabled) - } catch (error) { - console.error('Failed to check composio-for-google-calendar flag:', error) - } - } - loadComposioForGoogleFlag() - loadComposioForGoogleCalendarFlag() - }, [active]) + // (Composio Gmail/Calendar flag-check effect removed — flags are constant false now.) // Load Granola config const refreshGranolaConfig = useCallback(async () => { @@ -346,13 +330,22 @@ export function useConnectors(active: boolean) { const handleConnect = useCallback(async (provider: string) => { if (provider === 'google') { + // Signed-in users use the rowboat (managed-credentials) flow: opens + // the webapp in the browser, no BYOK modal. Main process detects + // signed-in via isSignedIn() when oauth:connect arrives without creds. + // Falls back to the BYOK modal for not-signed-in users. + const isSignedIntoRowboat = providerStates.rowboat?.isConnected ?? false + if (isSignedIntoRowboat) { + await startConnect('google') + return + } setGoogleClientIdDescription(undefined) setGoogleClientIdOpen(true) return } await startConnect(provider) - }, [startConnect]) + }, [startConnect, providerStates]) const handleGoogleClientIdSubmit = useCallback((clientId: string, clientSecret: string) => { setGoogleCredentials(clientId, clientSecret) @@ -485,19 +478,6 @@ export function useConnectors(active: boolean) { toast.success(`Connected to ${displayName}`) } - if (provider === 'rowboat') { - try { - const [googleResult, calendarResult] = await Promise.all([ - window.ipc.invoke('composio:use-composio-for-google', null), - window.ipc.invoke('composio:use-composio-for-google-calendar', null), - ]) - setUseComposioForGoogle(googleResult.enabled) - setUseComposioForGoogleCalendar(calendarResult.enabled) - } catch (err) { - console.error('Failed to re-check composio flags:', err) - } - } - refreshAllStatuses() } }) diff --git a/apps/x/packages/core/src/auth/google-backend-oauth.ts b/apps/x/packages/core/src/auth/google-backend-oauth.ts new file mode 100644 index 00000000..a441d205 --- /dev/null +++ b/apps/x/packages/core/src/auth/google-backend-oauth.ts @@ -0,0 +1,113 @@ +import { API_URL } from "../config/env.js"; +import { getAccessToken } from "./tokens.js"; +import { OAuthTokens } from "./types.js"; + +/** + * Client for the rowboat-mode Google OAuth endpoints on the api: + * POST /v1/google-oauth/claim — one-shot retrieval of tokens parked by + * the webapp callback under a `state` ticket + * POST /v1/google-oauth/refresh — exchange a refresh_token for fresh tokens + * (the secret-requiring step that can't + * happen on the desktop) + * + * Both are called with the user's Rowboat Supabase bearer (via getAccessToken). + * + * The api response shape uses `scope: string` (space-delimited); we convert + * to the desktop's `scopes: string[]`. On refresh, api may omit `scope` and + * `refresh_token` — caller-provided existingScopes / refreshToken are + * preserved in those cases (Google rarely rotates refresh tokens). + */ + +/** Thrown when the api signals the user must reconnect (Google `invalid_grant`). */ +export class ReconnectRequiredError extends Error { + constructor(message: string) { + super(message); + this.name = "ReconnectRequiredError"; + } +} + +interface ApiTokenResponse { + access_token: string; + refresh_token?: string; + expires_at: number; + scope?: string; + token_type?: string; +} + +function toOAuthTokens( + body: ApiTokenResponse, + fallbackRefreshToken: string | null = null, + fallbackScopes?: string[], +): OAuthTokens { + const refresh_token = body.refresh_token ?? fallbackRefreshToken; + const scopes = body.scope + ? body.scope.split(" ").filter((s) => s.length > 0) + : fallbackScopes; + return { + access_token: body.access_token, + refresh_token, + expires_at: body.expires_at, + token_type: "Bearer", + scopes, + }; +} + +async function postWithBearer(path: string, body: unknown): Promise { + const bearer = await getAccessToken(); + return fetch(`${API_URL}${path}`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${bearer}`, + }, + body: JSON.stringify(body), + }); +} + +interface ErrorBody { + error?: string; + reconnectRequired?: boolean; +} + +async function readError(res: Response): Promise { + try { + return (await res.json()) as ErrorBody; + } catch { + return {}; + } +} + +/** Claim the tokens parked under `state` after the webapp finished its callback. */ +export async function claimTokensViaBackend(state: string): Promise { + const res = await postWithBearer("/v1/google-oauth/claim", { session: state }); + if (!res.ok) { + const err = await readError(res); + throw new Error(`claim failed: ${res.status} ${err.error ?? ""}`.trim()); + } + const body = (await res.json()) as ApiTokenResponse; + return toOAuthTokens(body); +} + +/** + * Refresh an access token via the api. Preserves caller's `refreshToken` and + * `existingScopes` when Google omits them on the refresh response. + */ +export async function refreshTokensViaBackend( + refreshToken: string, + existingScopes?: string[], +): Promise { + const res = await postWithBearer("/v1/google-oauth/refresh", { refreshToken }); + if (res.status === 409) { + const err = await readError(res); + if (err.reconnectRequired) { + throw new ReconnectRequiredError(err.error ?? "Reconnect required"); + } + throw new Error(`refresh failed: 409 ${err.error ?? ""}`.trim()); + } + if (!res.ok) { + const err = await readError(res); + throw new Error(`refresh failed: ${res.status} ${err.error ?? ""}`.trim()); + } + const body = (await res.json()) as ApiTokenResponse; + return toOAuthTokens(body, refreshToken, existingScopes); +} diff --git a/apps/x/packages/core/src/auth/repo.ts b/apps/x/packages/core/src/auth/repo.ts index 5276faea..08f6b56d 100644 --- a/apps/x/packages/core/src/auth/repo.ts +++ b/apps/x/packages/core/src/auth/repo.ts @@ -8,6 +8,13 @@ const ProviderConnectionSchema = z.object({ tokens: OAuthTokens.nullable().optional(), clientId: z.string().nullable().optional(), clientSecret: z.string().nullable().optional(), + /** + * `byok` (default for absent) — user provides their own client_id+secret; + * tokens stored locally; refresh handled locally via openid-client. + * `rowboat` — signed-in user; client_id+secret never on the desktop; + * tokens stored locally but refresh goes through the api. + */ + mode: z.enum(['byok', 'rowboat']).optional(), error: z.string().nullable().optional(), }); diff --git a/apps/x/packages/core/src/composio/client.ts b/apps/x/packages/core/src/composio/client.ts index 2844fc28..8080d923 100644 --- a/apps/x/packages/core/src/composio/client.ts +++ b/apps/x/packages/core/src/composio/client.ts @@ -49,8 +49,6 @@ async function getAuthHeaders(): Promise> { */ const ZComposioConfig = z.object({ apiKey: z.string().optional(), - use_composio_for_google: z.boolean().optional(), - use_composio_for_google_calendar: z.boolean().optional(), }); type ComposioConfig = z.infer; @@ -106,24 +104,6 @@ export async function isConfigured(): Promise { return !!getApiKey(); } -/** - * Check if Composio should be used for Google services (Gmail, etc.) - */ -export async function useComposioForGoogle(): Promise { - if (await isSignedIn()) return true; - const config = loadConfig(); - return config.use_composio_for_google === true; -} - -/** - * Check if Composio should be used for Google Calendar - */ -export async function useComposioForGoogleCalendar(): Promise { - if (await isSignedIn()) return true; - const config = loadConfig(); - return config.use_composio_for_google_calendar === true; -} - /** * Make an API call to Composio */ diff --git a/apps/x/packages/core/src/config/remote-config.ts b/apps/x/packages/core/src/config/remote-config.ts new file mode 100644 index 00000000..87174ef7 --- /dev/null +++ b/apps/x/packages/core/src/config/remote-config.ts @@ -0,0 +1,51 @@ +import { API_URL } from "./env.js"; + +/** + * Per-process cache of the unauthenticated `GET /v1/config` response from + * the api. The api returns `{ appUrl, supabaseUrl, websocketApiUrl }` — + * we use this to discover the webapp host (where the rowboat-mode OAuth + * flow runs) without hardcoding it on the desktop side. + * + * Cached as a Promise so concurrent first-callers all await the same fetch + * (no thundering herd). On failure the cache is cleared so the next call + * can retry. + */ + +interface RemoteConfig { + appUrl: string; + supabaseUrl: string; + websocketApiUrl: string; +} + +let _cached: Promise | null = null; + +async function fetchRemoteConfig(): Promise { + const res = await fetch(`${API_URL}/v1/config`); + if (!res.ok) { + throw new Error(`/v1/config returned ${res.status}`); + } + const body = (await res.json()) as Partial; + if (!body.appUrl) { + throw new Error("/v1/config response missing appUrl"); + } + return { + appUrl: body.appUrl, + supabaseUrl: body.supabaseUrl ?? "", + websocketApiUrl: body.websocketApiUrl ?? "", + }; +} + +export async function getRemoteConfig(): Promise { + if (!_cached) { + _cached = fetchRemoteConfig().catch((err) => { + _cached = null; // allow retry + throw err; + }); + } + return _cached; +} + +export async function getWebappUrl(): Promise { + const config = await getRemoteConfig(); + return config.appUrl; +} diff --git a/apps/x/packages/core/src/knowledge/agent_notes.ts b/apps/x/packages/core/src/knowledge/agent_notes.ts index 471bfecd..301c10a6 100644 --- a/apps/x/packages/core/src/knowledge/agent_notes.ts +++ b/apps/x/packages/core/src/knowledge/agent_notes.ts @@ -8,8 +8,6 @@ import { waitForRunCompletion } from '../agents/utils.js'; import { serviceLogger } from '../services/service_logger.js'; import { loadUserConfig, updateUserEmail } from '../config/user_config.js'; import { GoogleClientFactory } from './google-client-factory.js'; -import { useComposioForGoogle, executeAction } from '../composio/client.js'; -import { composioAccountsRepo } from '../composio/repo.js'; import { loadAgentNotesState, saveAgentNotesState, @@ -199,30 +197,7 @@ async function ensureUserEmail(): Promise { return existing.email; } - // Try Composio (used when signed in or composio configured) - try { - if (await useComposioForGoogle()) { - const account = composioAccountsRepo.getAccount('gmail'); - if (account && account.status === 'ACTIVE') { - const result = await executeAction('GMAIL_GET_PROFILE', { - connected_account_id: account.id, - user_id: 'rowboat-user', - version: 'latest', - arguments: { user_id: 'me' }, - }); - const email = (result.data as Record)?.emailAddress as string | undefined; - if (email) { - updateUserEmail(email); - console.log(`[AgentNotes] Auto-populated user email via Composio: ${email}`); - return email; - } - } - } - } catch (error) { - console.log('[AgentNotes] Could not fetch email via Composio:', error instanceof Error ? error.message : error); - } - - // Try direct Google OAuth + // Try direct Google OAuth (covers both BYOK and rowboat modes) try { const auth = await GoogleClientFactory.getClient(); if (auth) { diff --git a/apps/x/packages/core/src/knowledge/google-client-factory.ts b/apps/x/packages/core/src/knowledge/google-client-factory.ts index 9e0ad2d1..0c48ae37 100644 --- a/apps/x/packages/core/src/knowledge/google-client-factory.ts +++ b/apps/x/packages/core/src/knowledge/google-client-factory.ts @@ -6,20 +6,44 @@ import { getProviderConfig } from '../auth/providers.js'; import * as oauthClient from '../auth/oauth-client.js'; import type { Configuration } from '../auth/oauth-client.js'; import { OAuthTokens } from '../auth/types.js'; +import { + ReconnectRequiredError, + refreshTokensViaBackend, +} from '../auth/google-backend-oauth.js'; + +type Mode = 'byok' | 'rowboat'; /** * Factory for creating and managing Google OAuth2Client instances. * Handles caching, token refresh, and client reuse for Google API SDKs. + * + * Two connection modes share the same `oauth.json` provider entry: + * - `byok` user supplied client_id+secret; refresh runs locally via + * openid-client; OAuth2Client built with creds. + * - `rowboat` signed-in user; client_id+secret never on the desktop; + * refresh goes through the api at /v1/google-oauth/refresh; + * OAuth2Client built without creds and without refresh_token + * (we own all refreshes — see note below). + * + * **Auto-refresh disabled in rowboat mode:** google-auth-library's + * OAuth2Client will, on a 401 from a Google API call, try to refresh using + * the refresh_token + client secret it has on hand. In rowboat mode we have + * no secret, so that would 401-loop. We block this by passing only + * access_token + expiry_date in setCredentials (no refresh_token), which + * leaves the library nothing to refresh with. Our proactive expiry check + * in getClient() is the only refresh path. */ export class GoogleClientFactory { private static readonly PROVIDER_NAME = 'google'; private static cache: { + mode: Mode | null; config: Configuration | null; client: OAuth2Client | null; tokens: OAuthTokens | null; clientId: string | null; clientSecret: string | null; } = { + mode: null, config: null, client: null, tokens: null, @@ -27,7 +51,14 @@ export class GoogleClientFactory { clientSecret: null, }; - private static async resolveCredentials(): Promise<{ clientId: string; clientSecret?: string }> { + /** + * Promise singleton so a burst of getClient() calls during the brief + * expiry window all wait on a single refresh round-trip rather than + * fanning out parallel refreshes. + */ + private static refreshInFlight: Promise | null = null; + + private static async resolveByokCredentials(): Promise<{ clientId: string; clientSecret?: string }> { const oauthRepo = container.resolve('oauthRepo'); const connection = await oauthRepo.read(this.PROVIDER_NAME); if (!connection.clientId) { @@ -41,80 +72,116 @@ export class GoogleClientFactory { * Get or create OAuth2Client, reusing cached instance when possible */ static async getClient(): Promise { + if (this.refreshInFlight) { + return this.refreshInFlight; + } + const oauthRepo = container.resolve('oauthRepo'); - const { tokens } = await oauthRepo.read(this.PROVIDER_NAME); + const connection = await oauthRepo.read(this.PROVIDER_NAME); + const tokens = connection.tokens ?? null; + const mode: Mode = connection.mode ?? 'byok'; if (!tokens) { this.clearCache(); return null; } - // Initialize config cache if needed - try { - await this.initializeConfigCache(); - } catch (error) { - console.error("[OAuth] Failed to initialize Google OAuth configuration:", error); + // Mode flipped (e.g. user disconnected then reconnected differently) — invalidate. + if (this.cache.mode && this.cache.mode !== mode) { this.clearCache(); - return null; - } - if (!this.cache.config) { - return null; } - // Check if token is expired + // BYOK needs an openid-client Configuration for local refresh; rowboat doesn't. + if (mode === 'byok') { + try { + await this.initializeConfigCache(); + } catch (error) { + console.error('[OAuth] Failed to initialize Google OAuth configuration:', error); + this.clearCache(); + return null; + } + if (!this.cache.config) { + return null; + } + } + + // Check expiry against the cached tokens. Note: oauthClient.isTokenExpired + // applies a small clock-skew margin so we refresh slightly before real + // expiry — keeps long-running calls from racing the boundary. if (oauthClient.isTokenExpired(tokens)) { - // Token expired, try to refresh if (!tokens.refresh_token) { - console.log("[OAuth] Token expired and no refresh token available for Google."); + console.log('[OAuth] Token expired and no refresh token available for Google.'); await oauthRepo.upsert(this.PROVIDER_NAME, { error: 'Missing refresh token. Please reconnect.' }); this.clearCache(); return null; } - try { - console.log(`[OAuth] Token expired, refreshing access token...`); - const existingScopes = tokens.scopes; - const refreshedTokens = await oauthClient.refreshTokens( - this.cache.config, - tokens.refresh_token, - existingScopes - ); - await oauthRepo.upsert(this.PROVIDER_NAME, { tokens: refreshedTokens }); - - // Update cached tokens and recreate client - this.cache.tokens = refreshedTokens; - if (!this.cache.clientId) { - const creds = await this.resolveCredentials(); - this.cache.clientId = creds.clientId; - this.cache.clientSecret = creds.clientSecret ?? null; - } - this.cache.client = this.createClientFromTokens(refreshedTokens, this.cache.clientId, this.cache.clientSecret ?? undefined); - console.log(`[OAuth] Token refreshed successfully`); - return this.cache.client; - } catch (error) { - const message = error instanceof Error ? error.message : 'Failed to refresh token for Google'; - await oauthRepo.upsert(this.PROVIDER_NAME, { error: message }); - console.error("[OAuth] Failed to refresh token for Google:", error); - this.clearCache(); - return null; - } + this.refreshInFlight = this.refreshAndBuild(tokens, mode).finally(() => { + this.refreshInFlight = null; + }); + return this.refreshInFlight; } // Reuse client if tokens haven't changed - if (this.cache.client && this.cache.tokens && this.cache.tokens.access_token === tokens.access_token) { + if (this.cache.client && this.cache.tokens && this.cache.tokens.access_token === tokens.access_token && this.cache.mode === mode) { return this.cache.client; } - // Create new client with current tokens - console.log(`[OAuth] Creating new OAuth2Client instance`); - this.cache.tokens = tokens; - if (!this.cache.clientId) { - const creds = await this.resolveCredentials(); + // Build a fresh client for current tokens + return this.buildAndCacheClient(tokens, mode); + } + + private static async refreshAndBuild(tokens: OAuthTokens, mode: Mode): Promise { + const oauthRepo = container.resolve('oauthRepo'); + + try { + console.log(`[OAuth] Token expired, refreshing via ${mode}...`); + const existingScopes = tokens.scopes; + + let refreshedTokens: OAuthTokens; + if (mode === 'rowboat') { + refreshedTokens = await refreshTokensViaBackend(tokens.refresh_token!, existingScopes); + } else { + if (!this.cache.config) { + // Should not happen — initializeConfigCache ran above for byok. + throw new Error('Google OAuth config not initialized'); + } + refreshedTokens = await oauthClient.refreshTokens(this.cache.config, tokens.refresh_token!, existingScopes); + } + + await oauthRepo.upsert(this.PROVIDER_NAME, { tokens: refreshedTokens, error: null }); + console.log('[OAuth] Token refreshed successfully'); + return this.buildAndCacheClient(refreshedTokens, mode); + } catch (error) { + if (error instanceof ReconnectRequiredError) { + console.log('[OAuth] Reconnect required for Google'); + await oauthRepo.upsert(this.PROVIDER_NAME, { error: 'Reconnect Google' }); + this.clearCache(); + return null; + } + const message = error instanceof Error ? error.message : 'Failed to refresh token for Google'; + await oauthRepo.upsert(this.PROVIDER_NAME, { error: message }); + console.error('[OAuth] Failed to refresh token for Google:', error); + this.clearCache(); + return null; + } + } + + private static async buildAndCacheClient(tokens: OAuthTokens, mode: Mode): Promise { + if (mode === 'byok' && !this.cache.clientId) { + const creds = await this.resolveByokCredentials(); this.cache.clientId = creds.clientId; this.cache.clientSecret = creds.clientSecret ?? null; } - this.cache.client = this.createClientFromTokens(tokens, this.cache.clientId, this.cache.clientSecret ?? undefined); - return this.cache.client; + + const client = mode === 'rowboat' + ? this.createRowboatClient(tokens) + : this.createByokClient(tokens, this.cache.clientId!, this.cache.clientSecret ?? undefined); + + this.cache.mode = mode; + this.cache.tokens = tokens; + this.cache.client = client; + return client; } /** @@ -139,7 +206,8 @@ export class GoogleClientFactory { * Clear cache (useful for testing or when credentials are revoked) */ static clearCache(): void { - console.log(`[OAuth] Clearing Google auth cache`); + console.log('[OAuth] Clearing Google auth cache'); + this.cache.mode = null; this.cache.config = null; this.cache.client = null; this.cache.tokens = null; @@ -148,10 +216,10 @@ export class GoogleClientFactory { } /** - * Initialize cached configuration (called once) + * Initialize cached configuration for BYOK mode (rowboat doesn't need it). */ private static async initializeConfigCache(): Promise { - const { clientId, clientSecret } = await this.resolveCredentials(); + const { clientId, clientSecret } = await this.resolveByokCredentials(); if (this.cache.config && this.cache.clientId === clientId && this.cache.clientSecret === (clientSecret ?? null)) { return; // Already initialized for these credentials @@ -161,13 +229,13 @@ export class GoogleClientFactory { this.clearCache(); } - console.log(`[OAuth] Initializing Google OAuth configuration...`); + console.log('[OAuth] Initializing Google OAuth configuration...'); const providerConfig = await getProviderConfig(this.PROVIDER_NAME); if (providerConfig.discovery.mode === 'issuer') { if (providerConfig.client.mode === 'static') { // Discover endpoints, use static client ID - console.log(`[OAuth] Discovery mode: issuer with static client ID`); + console.log('[OAuth] Discovery mode: issuer with static client ID'); this.cache.config = await oauthClient.discoverConfiguration( providerConfig.discovery.issuer, clientId, @@ -175,7 +243,7 @@ export class GoogleClientFactory { ); } else { // DCR mode - need existing registration - console.log(`[OAuth] Discovery mode: issuer with DCR`); + console.log('[OAuth] Discovery mode: issuer with DCR'); const clientRepo = container.resolve('clientRegistrationRepo'); const existingRegistration = await clientRepo.getClientRegistration(this.PROVIDER_NAME); @@ -194,7 +262,7 @@ export class GoogleClientFactory { throw new Error('DCR requires discovery mode "issuer", not "static"'); } - console.log(`[OAuth] Using static endpoints (no discovery)`); + console.log('[OAuth] Using static endpoints (no discovery)'); this.cache.config = oauthClient.createStaticConfiguration( providerConfig.discovery.authorizationEndpoint, providerConfig.discovery.tokenEndpoint, @@ -206,27 +274,33 @@ export class GoogleClientFactory { this.cache.clientId = clientId; this.cache.clientSecret = clientSecret ?? null; - console.log(`[OAuth] Google OAuth configuration initialized`); + console.log('[OAuth] Google OAuth configuration initialized'); } - /** - * Create OAuth2Client from OAuthTokens - */ - private static createClientFromTokens(tokens: OAuthTokens, clientId: string, clientSecret?: string): OAuth2Client { - const client = new OAuth2Client( - clientId, - clientSecret ?? undefined, - undefined // redirect_uri not needed for token usage - ); - - // Set credentials + /** BYOK OAuth2Client — has client_id + secret + refresh_token. */ + private static createByokClient(tokens: OAuthTokens, clientId: string, clientSecret?: string): OAuth2Client { + const client = new OAuth2Client(clientId, clientSecret ?? undefined, undefined); client.setCredentials({ access_token: tokens.access_token, refresh_token: tokens.refresh_token || undefined, - expiry_date: tokens.expires_at * 1000, // Convert from seconds to milliseconds + expiry_date: tokens.expires_at * 1000, scope: tokens.scopes?.join(' ') || undefined, }); + return client; + } + /** + * Rowboat OAuth2Client — no client_id/secret, no refresh_token. + * Library auto-refresh is disabled by absence of refresh_token; our + * proactive refresh in getClient() is the only refresh path. + */ + private static createRowboatClient(tokens: OAuthTokens): OAuth2Client { + const client = new OAuth2Client(); + client.setCredentials({ + access_token: tokens.access_token, + expiry_date: tokens.expires_at * 1000, + scope: tokens.scopes?.join(' ') || undefined, + }); return client; } } diff --git a/apps/x/packages/core/src/knowledge/sync_calendar.ts b/apps/x/packages/core/src/knowledge/sync_calendar.ts index b6258975..b311dfa2 100644 --- a/apps/x/packages/core/src/knowledge/sync_calendar.ts +++ b/apps/x/packages/core/src/knowledge/sync_calendar.ts @@ -5,10 +5,8 @@ import { OAuth2Client } from 'google-auth-library'; import { NodeHtmlMarkdown } from 'node-html-markdown' import { WorkDir } from '../config/config.js'; import { GoogleClientFactory } from './google-client-factory.js'; -import { serviceLogger, type ServiceRunContext } from '../services/service_logger.js'; +import { serviceLogger } from '../services/service_logger.js'; import { limitEventItems } from './limit_event_items.js'; -import { executeAction, useComposioForGoogleCalendar } from '../composio/client.js'; -import { composioAccountsRepo } from '../composio/repo.js'; import { createEvent } from './track/events.js'; const MAX_EVENTS_IN_DIGEST = 50; @@ -138,7 +136,6 @@ async function publishCalendarSyncEvent( const SYNC_DIR = path.join(WorkDir, 'calendar_sync'); const SYNC_INTERVAL_MS = 5 * 60 * 1000; // Check every 5 minutes const LOOKBACK_DAYS = 7; -const COMPOSIO_LOOKBACK_DAYS = 7; const REQUIRED_SCOPES = [ 'https://www.googleapis.com/auth/calendar.events.readonly', 'https://www.googleapis.com/auth/drive.readonly' @@ -477,286 +474,17 @@ async function performSync(syncDir: string, lookbackDays: number) { } } -// --- Composio-based Sync --- - -interface ComposioCalendarState { - last_sync: string; // ISO string -} - -function loadComposioState(stateFile: string): ComposioCalendarState | null { - if (fs.existsSync(stateFile)) { - try { - const data = JSON.parse(fs.readFileSync(stateFile, 'utf-8')); - if (data.last_sync) { - return { last_sync: data.last_sync }; - } - } catch (e) { - console.error('[Calendar] Failed to load composio state:', e); - } - } - return null; -} - -function saveComposioState(stateFile: string, lastSync: string): void { - fs.writeFileSync(stateFile, JSON.stringify({ last_sync: lastSync }, null, 2)); -} - -/** - * Save a Composio calendar event as JSON (same format used by Google OAuth path). - * The event data from Composio is already structured similarly to Google Calendar API. - */ -function saveComposioEvent(eventData: Record, syncDir: string): { changed: boolean; isNew: boolean; title: string } { - const eventId = eventData.id as string | undefined; - if (!eventId) return { changed: false, isNew: false, title: 'Unknown' }; - - const filePath = path.join(syncDir, `${eventId}.json`); - const content = JSON.stringify(eventData, null, 2); - const exists = fs.existsSync(filePath); - - try { - if (exists) { - const existing = fs.readFileSync(filePath, 'utf-8'); - if (existing === content) { - return { changed: false, isNew: false, title: (eventData.summary as string) || eventId }; - } - } - - fs.writeFileSync(filePath, content); - return { changed: true, isNew: !exists, title: (eventData.summary as string) || eventId }; - } catch (e) { - console.error(`[Calendar] Error saving event ${eventId}:`, e); - return { changed: false, isNew: false, title: (eventData.summary as string) || eventId }; - } -} - -async function performSyncComposio() { - const STATE_FILE = path.join(SYNC_DIR, 'composio_state.json'); - - if (!fs.existsSync(SYNC_DIR)) fs.mkdirSync(SYNC_DIR, { recursive: true }); - - const account = composioAccountsRepo.getAccount('googlecalendar'); - if (!account || account.status !== 'ACTIVE') { - console.log('[Calendar] Google Calendar not connected via Composio. Skipping sync.'); - return; - } - - const connectedAccountId = account.id; - - // Calculate time window: lookback + 14 days forward - const now = new Date(); - const lookbackMs = COMPOSIO_LOOKBACK_DAYS * 24 * 60 * 60 * 1000; - const twoWeeksForwardMs = 14 * 24 * 60 * 60 * 1000; - - const timeMin = new Date(now.getTime() - lookbackMs).toISOString(); - const timeMax = new Date(now.getTime() + twoWeeksForwardMs).toISOString(); - - console.log(`[Calendar] Syncing via Composio from ${timeMin} to ${timeMax} (lookback: ${COMPOSIO_LOOKBACK_DAYS} days)...`); - - let run: ServiceRunContext | null = null; - const ensureRun = async (): Promise => { - if (!run) { - run = await serviceLogger.startRun({ - service: 'calendar', - message: 'Syncing calendar (Composio)', - trigger: 'timer', - }); - } - return run; - }; - - try { - const currentEventIds = new Set(); - let newCount = 0; - let updatedCount = 0; - const changedTitles: string[] = []; - const newEvents: AnyEvent[] = []; - const updatedEvents: AnyEvent[] = []; - let pageToken: string | null = null; - const MAX_PAGES = 20; - - for (let page = 0; page < MAX_PAGES; page++) { - // Re-check connection in case user disconnected mid-sync - if (!composioAccountsRepo.isConnected('googlecalendar')) { - console.log('[Calendar] Account disconnected during sync. Stopping.'); - return; - } - - const args: Record = { - calendar_id: 'primary', - time_min: timeMin, - time_max: timeMax, - single_events: true, - order_by: 'startTime', - }; - if (pageToken) { - args.page_token = pageToken; - } - - const result = await executeAction( - 'GOOGLECALENDAR_FIND_EVENT', - { - connected_account_id: connectedAccountId, - user_id: 'rowboat-user', - version: 'latest', - arguments: args, - } - ); - - if (!result.successful || !result.data) { - console.error('[Calendar] Failed to list events via Composio:', result.error); - return; - } - - const data = result.data as Record; - // Composio may return events in different structures - let events: Array> = []; - - if (Array.isArray(data.items)) { - events = data.items as Array>; - } else if (Array.isArray(data.events)) { - events = data.events as Array>; - } else if (data.event_data && typeof data.event_data === 'object') { - const nested = data.event_data as Record; - if (Array.isArray(nested.event_data)) { - events = nested.event_data as Array>; - } else if (Array.isArray(data.event_data)) { - events = data.event_data as Array>; - } - } else if (Array.isArray(data)) { - events = data as unknown as Array>; - } - - if (events.length === 0 && page === 0) { - console.log('[Calendar] No events found in this window.'); - } else if (events.length > 0) { - console.log(`[Calendar] Page ${page + 1}: found ${events.length} events.`); - for (const event of events) { - const eventId = event.id as string | undefined; - if (eventId) { - const saveResult = saveComposioEvent(event, SYNC_DIR); - currentEventIds.add(eventId); - - if (saveResult.changed) { - await ensureRun(); - changedTitles.push(saveResult.title); - if (saveResult.isNew) { - newCount++; - newEvents.push(event); - } else { - updatedCount++; - updatedEvents.push(event); - } - } - } - } - } - - // Check for next page - const nextToken = data.nextPageToken as string | undefined; - if (nextToken) { - pageToken = nextToken; - console.log(`[Calendar] Fetching next page...`); - } else { - break; - } - } - - // Clean up events no longer in the window - const deletedFiles = cleanUpOldFiles(currentEventIds, SYNC_DIR); - let deletedCount = 0; - if (deletedFiles.length > 0) { - await ensureRun(); - deletedCount = deletedFiles.length; - } - - // Publish a single bundled event capturing all changes from this sync. - await publishCalendarSyncEvent(newEvents, updatedEvents, deletedFiles); - - // Log results if any changes were detected (run was started by ensureRun) - if (run) { - const r = run as ServiceRunContext; - const totalChanges = newCount + updatedCount + deletedCount; - const limitedTitles = limitEventItems(changedTitles); - await serviceLogger.log({ - type: 'changes_identified', - service: r.service, - runId: r.runId, - level: 'info', - message: `Calendar updates: ${totalChanges} change${totalChanges === 1 ? '' : 's'}`, - counts: { - newEvents: newCount, - updatedEvents: updatedCount, - deletedFiles: deletedCount, - }, - items: limitedTitles.items, - truncated: limitedTitles.truncated, - }); - await serviceLogger.log({ - type: 'run_complete', - service: r.service, - runId: r.runId, - level: 'info', - message: `Calendar sync complete: ${totalChanges} change${totalChanges === 1 ? '' : 's'}`, - durationMs: Date.now() - r.startedAt, - outcome: 'ok', - summary: { - newEvents: newCount, - updatedEvents: updatedCount, - deletedFiles: deletedCount, - }, - }); - } - - // Save state - saveComposioState(STATE_FILE, new Date().toISOString()); - console.log(`[Calendar] Composio sync completed. ${newCount} new, ${updatedCount} updated, ${deletedCount} deleted.`); - } catch (error) { - console.error('[Calendar] Error during Composio sync:', error); - const errRun = await ensureRun(); - await serviceLogger.log({ - type: 'error', - service: errRun.service, - runId: errRun.runId, - level: 'error', - message: 'Calendar sync error', - error: error instanceof Error ? error.message : String(error), - }); - await serviceLogger.log({ - type: 'run_complete', - service: errRun.service, - runId: errRun.runId, - level: 'error', - message: 'Calendar sync failed', - durationMs: Date.now() - errRun.startedAt, - outcome: 'error', - }); - } -} - export async function init() { console.log("Starting Google Calendar & Notes Sync (TS)..."); console.log(`Will sync every ${SYNC_INTERVAL_MS / 1000} seconds.`); while (true) { try { - const composioMode = await useComposioForGoogleCalendar(); - if (composioMode) { - const isConnected = composioAccountsRepo.isConnected('googlecalendar'); - if (!isConnected) { - console.log('[Calendar] Google Calendar not connected via Composio. Sleeping...'); - } else { - await performSyncComposio(); - } + const hasCredentials = await GoogleClientFactory.hasValidCredentials(REQUIRED_SCOPES); + if (!hasCredentials) { + console.log("Google OAuth credentials not available or missing required Calendar/Drive scopes. Sleeping..."); } else { - // Check if credentials are available with required scopes - const hasCredentials = await GoogleClientFactory.hasValidCredentials(REQUIRED_SCOPES); - - if (!hasCredentials) { - console.log("Google OAuth credentials not available or missing required Calendar/Drive scopes. Sleeping..."); - } else { - // Perform one sync - await performSync(SYNC_DIR, LOOKBACK_DAYS); - } + await performSync(SYNC_DIR, LOOKBACK_DAYS); } } catch (error) { console.error("Error in main loop:", error); diff --git a/apps/x/packages/core/src/knowledge/sync_gmail.ts b/apps/x/packages/core/src/knowledge/sync_gmail.ts index 2aa48944..81a63edf 100644 --- a/apps/x/packages/core/src/knowledge/sync_gmail.ts +++ b/apps/x/packages/core/src/knowledge/sync_gmail.ts @@ -7,8 +7,6 @@ import { WorkDir } from '../config/config.js'; import { GoogleClientFactory } from './google-client-factory.js'; import { serviceLogger, type ServiceRunContext } from '../services/service_logger.js'; import { limitEventItems } from './limit_event_items.js'; -import { executeAction, useComposioForGoogle } from '../composio/client.js'; -import { composioAccountsRepo } from '../composio/repo.js'; import { createEvent } from './track/events.js'; // Configuration @@ -225,7 +223,7 @@ async function processThread(auth: OAuth2Client, threadId: string, syncDir: stri } } -function loadState(stateFile: string): { historyId?: string } { +function loadState(stateFile: string): { historyId?: string; last_sync?: string } { if (fs.existsSync(stateFile)) { return JSON.parse(fs.readFileSync(stateFile, 'utf-8')); } @@ -240,9 +238,24 @@ function saveState(historyId: string, stateFile: string) { } async function fullSync(auth: OAuth2Client, syncDir: string, attachmentsDir: string, stateFile: string, lookbackDays: number) { - console.log(`Performing full sync of last ${lookbackDays} days...`); const gmail = google.gmail({ version: 'v1', auth }); + // If the state file holds a last_sync timestamp (e.g. left over from a + // prior Composio sync, or from a previous successful native sync that + // we're falling back to after a history.list 404), use that as the + // floor instead of the default lookback. Carries forward Composio's + // last_sync on first migration so we don't refetch the last 7 days. + const state = loadState(stateFile); + let pastDate: Date; + if (state.last_sync) { + pastDate = new Date(state.last_sync); + console.log(`Performing full sync from last_sync=${state.last_sync}...`); + } else { + pastDate = new Date(); + pastDate.setDate(pastDate.getDate() - lookbackDays); + console.log(`Performing full sync of last ${lookbackDays} days...`); + } + let run: ServiceRunContext | null = null; const ensureRun = async () => { if (!run) { @@ -255,8 +268,6 @@ async function fullSync(auth: OAuth2Client, syncDir: string, attachmentsDir: str }; try { - const pastDate = new Date(); - pastDate.setDate(pastDate.getDate() - lookbackDays); const dateQuery = pastDate.toISOString().split('T')[0].replace(/-/g, '/'); // Get History ID @@ -498,386 +509,17 @@ async function performSync() { } } -// --- Composio-based Sync --- - -const COMPOSIO_LOOKBACK_DAYS = 7; - -interface ComposioSyncState { - last_sync: string; // ISO string -} - -function loadComposioState(stateFile: string): ComposioSyncState | null { - if (fs.existsSync(stateFile)) { - try { - const data = JSON.parse(fs.readFileSync(stateFile, 'utf-8')); - if (data.last_sync) { - return { last_sync: data.last_sync }; - } - } catch (e) { - console.error('[Gmail] Failed to load composio state:', e); - } - } - return null; -} - -function saveComposioState(stateFile: string, lastSync: string): void { - fs.writeFileSync(stateFile, JSON.stringify({ last_sync: lastSync }, null, 2)); -} - -function tryParseDate(dateStr: string): Date | null { - const d = new Date(dateStr); - return isNaN(d.getTime()) ? null : d; -} - -interface ParsedMessage { - from: string; - date: string; - subject: string; - body: string; -} - -function parseMessageData(messageData: Record): ParsedMessage { - const headers = messageData.payload && typeof messageData.payload === 'object' - ? (messageData.payload as Record).headers as Array<{ name: string; value: string }> | undefined - : undefined; - - const from = headers?.find(h => h.name === 'From')?.value || String(messageData.from || messageData.sender || 'Unknown'); - const date = headers?.find(h => h.name === 'Date')?.value || String(messageData.date || messageData.internalDate || 'Unknown'); - const subject = headers?.find(h => h.name === 'Subject')?.value || String(messageData.subject || '(No Subject)'); - - let body = ''; - - if (messageData.payload && typeof messageData.payload === 'object') { - body = extractBodyFromPayload(messageData.payload as Record); - } - - if (!body) { - if (typeof messageData.body === 'string') { - body = messageData.body; - } else if (typeof messageData.snippet === 'string') { - body = messageData.snippet; - } else if (typeof messageData.text === 'string') { - body = messageData.text; - } - } - - if (body && (body.includes(' !line.trim().startsWith('>')).join('\n'); - } - - return { from, date, subject, body }; -} - -function extractBodyFromPayload(payload: Record): string { - const parts = payload.parts as Array> | undefined; - - if (parts) { - for (const part of parts) { - const mimeType = part.mimeType as string | undefined; - const bodyData = part.body && typeof part.body === 'object' - ? (part.body as Record).data as string | undefined - : undefined; - - if ((mimeType === 'text/plain' || mimeType === 'text/html') && bodyData) { - const decoded = Buffer.from(bodyData, 'base64').toString('utf-8'); - if (mimeType === 'text/html') { - return nhm.translate(decoded); - } - return decoded; - } - - if (part.parts) { - const result = extractBodyFromPayload(part as Record); - if (result) return result; - } - } - } - - const bodyData = payload.body && typeof payload.body === 'object' - ? (payload.body as Record).data as string | undefined - : undefined; - - if (bodyData) { - const decoded = Buffer.from(bodyData, 'base64').toString('utf-8'); - const mimeType = payload.mimeType as string | undefined; - if (mimeType === 'text/html') { - return nhm.translate(decoded); - } - return decoded; - } - - return ''; -} - -interface ComposioThreadResult { - synced: SyncedThread | null; - newestIsoPlusOne: string | null; -} - -async function processThreadComposio(connectedAccountId: string, threadId: string, syncDir: string): Promise { - let threadResult; - try { - threadResult = await executeAction( - 'GMAIL_FETCH_MESSAGE_BY_THREAD_ID', - { - connected_account_id: connectedAccountId, - user_id: 'rowboat-user', - version: 'latest', - arguments: { thread_id: threadId, user_id: 'me' }, - } - ); - } catch (error) { - console.warn(`[Gmail] Skipping thread ${threadId} (fetch failed):`, error instanceof Error ? error.message : error); - return { synced: null, newestIsoPlusOne: null }; - } - - if (!threadResult.successful || !threadResult.data) { - console.error(`[Gmail] Failed to fetch thread ${threadId}:`, threadResult.error); - return { synced: null, newestIsoPlusOne: null }; - } - - const data = threadResult.data as Record; - const messages = data.messages as Array> | undefined; - - let newestDate: Date | null = null; - let mdContent: string; - let subjectForLog: string; - - if (!messages || messages.length === 0) { - const parsed = parseMessageData(data); - mdContent = `# ${parsed.subject}\n\n` + - `**Thread ID:** ${threadId}\n` + - `**Message Count:** 1\n\n---\n\n` + - `### From: ${parsed.from}\n` + - `**Date:** ${parsed.date}\n\n` + - `${parsed.body}\n\n---\n\n`; - subjectForLog = parsed.subject; - newestDate = tryParseDate(parsed.date); - } else { - const firstParsed = parseMessageData(messages[0]); - mdContent = `# ${firstParsed.subject}\n\n`; - mdContent += `**Thread ID:** ${threadId}\n`; - mdContent += `**Message Count:** ${messages.length}\n\n---\n\n`; - - for (const msg of messages) { - const parsed = parseMessageData(msg); - mdContent += `### From: ${parsed.from}\n`; - mdContent += `**Date:** ${parsed.date}\n\n`; - mdContent += `${parsed.body}\n\n`; - mdContent += `---\n\n`; - - const msgDate = tryParseDate(parsed.date); - if (msgDate && (!newestDate || msgDate > newestDate)) { - newestDate = msgDate; - } - } - subjectForLog = firstParsed.subject; - } - - fs.writeFileSync(path.join(syncDir, `${cleanFilename(threadId)}.md`), mdContent); - console.log(`[Gmail] Synced Thread: ${subjectForLog} (${threadId})`); - - const newestIsoPlusOne = newestDate ? new Date(newestDate.getTime() + 1000).toISOString() : null; - return { synced: { threadId, markdown: mdContent }, newestIsoPlusOne }; -} - -async function performSyncComposio() { - const ATTACHMENTS_DIR = path.join(SYNC_DIR, 'attachments'); - const STATE_FILE = path.join(SYNC_DIR, 'sync_state.json'); - - if (!fs.existsSync(SYNC_DIR)) fs.mkdirSync(SYNC_DIR, { recursive: true }); - if (!fs.existsSync(ATTACHMENTS_DIR)) fs.mkdirSync(ATTACHMENTS_DIR, { recursive: true }); - - const account = composioAccountsRepo.getAccount('gmail'); - if (!account || account.status !== 'ACTIVE') { - console.log('[Gmail] Gmail not connected via Composio. Skipping sync.'); - return; - } - - const connectedAccountId = account.id; - - const state = loadComposioState(STATE_FILE); - let afterEpochSeconds: number; - - if (state) { - afterEpochSeconds = Math.floor(new Date(state.last_sync).getTime() / 1000); - console.log(`[Gmail] Syncing messages since ${state.last_sync}...`); - } else { - const pastDate = new Date(); - pastDate.setDate(pastDate.getDate() - COMPOSIO_LOOKBACK_DAYS); - afterEpochSeconds = Math.floor(pastDate.getTime() / 1000); - console.log(`[Gmail] First sync - fetching last ${COMPOSIO_LOOKBACK_DAYS} days...`); - } - - let run: ServiceRunContext | null = null; - const ensureRun = async () => { - if (!run) { - run = await serviceLogger.startRun({ - service: 'gmail', - message: 'Syncing Gmail (Composio)', - trigger: 'timer', - }); - } - }; - - try { - const allThreadIds: string[] = []; - let pageToken: string | undefined; - - do { - const params: Record = { - query: `after:${afterEpochSeconds}`, - max_results: 20, - user_id: 'me', - }; - if (pageToken) { - params.page_token = pageToken; - } - - const result = await executeAction( - 'GMAIL_LIST_THREADS', - { - connected_account_id: connectedAccountId, - user_id: 'rowboat-user', - version: 'latest', - arguments: params, - } - ); - - if (!result.successful || !result.data) { - console.error('[Gmail] Failed to list threads:', result.error); - return; - } - - const data = result.data as Record; - const threads = data.threads as Array> | undefined; - - if (threads && threads.length > 0) { - for (const thread of threads) { - const threadId = thread.id as string | undefined; - if (threadId) { - allThreadIds.push(threadId); - } - } - } - - pageToken = data.nextPageToken as string | undefined; - } while (pageToken); - - if (allThreadIds.length === 0) { - console.log('[Gmail] No new threads.'); - return; - } - - console.log(`[Gmail] Found ${allThreadIds.length} threads to sync.`); - - await ensureRun(); - const limitedThreads = limitEventItems(allThreadIds); - await serviceLogger.log({ - type: 'changes_identified', - service: run!.service, - runId: run!.runId, - level: 'info', - message: `Found ${allThreadIds.length} thread${allThreadIds.length === 1 ? '' : 's'} to sync`, - counts: { threads: allThreadIds.length }, - items: limitedThreads.items, - truncated: limitedThreads.truncated, - }); - - // Process oldest first so high-water mark advances chronologically - allThreadIds.reverse(); - - let highWaterMark: string | null = state?.last_sync ?? null; - let processedCount = 0; - const synced: SyncedThread[] = []; - for (const threadId of allThreadIds) { - // Re-check connection in case user disconnected mid-sync - if (!composioAccountsRepo.isConnected('gmail')) { - console.log('[Gmail] Account disconnected during sync. Stopping.'); - break; - } - try { - const result = await processThreadComposio(connectedAccountId, threadId, SYNC_DIR); - processedCount++; - - if (result.synced) synced.push(result.synced); - - if (result.newestIsoPlusOne) { - if (!highWaterMark || new Date(result.newestIsoPlusOne) > new Date(highWaterMark)) { - highWaterMark = result.newestIsoPlusOne; - } - saveComposioState(STATE_FILE, highWaterMark); - } - } catch (error) { - console.error(`[Gmail] Error processing thread ${threadId}, skipping:`, error); - } - } - - await publishGmailSyncEvent(synced); - - await serviceLogger.log({ - type: 'run_complete', - service: run!.service, - runId: run!.runId, - level: 'info', - message: `Gmail sync complete: ${processedCount}/${allThreadIds.length} thread${allThreadIds.length === 1 ? '' : 's'}`, - durationMs: Date.now() - run!.startedAt, - outcome: 'ok', - summary: { threads: processedCount }, - }); - - console.log(`[Gmail] Sync completed. Processed ${processedCount}/${allThreadIds.length} threads.`); - } catch (error) { - console.error('[Gmail] Error during sync:', error); - await ensureRun(); - await serviceLogger.log({ - type: 'error', - service: run!.service, - runId: run!.runId, - level: 'error', - message: 'Gmail sync error', - error: error instanceof Error ? error.message : String(error), - }); - await serviceLogger.log({ - type: 'run_complete', - service: run!.service, - runId: run!.runId, - level: 'error', - message: 'Gmail sync failed', - durationMs: Date.now() - run!.startedAt, - outcome: 'error', - }); - } -} - export async function init() { console.log("Starting Gmail Sync (TS)..."); console.log(`Will sync every ${SYNC_INTERVAL_MS / 1000} seconds.`); while (true) { try { - const composioMode = await useComposioForGoogle(); - if (composioMode) { - const isConnected = composioAccountsRepo.isConnected('gmail'); - if (!isConnected) { - console.log('[Gmail] Gmail not connected via Composio. Sleeping...'); - } else { - await performSyncComposio(); - } + const hasCredentials = await GoogleClientFactory.hasValidCredentials(REQUIRED_SCOPE); + if (!hasCredentials) { + console.log("Google OAuth credentials not available or missing required Gmail scope. Sleeping..."); } else { - // Check if credentials are available with required scopes - const hasCredentials = await GoogleClientFactory.hasValidCredentials(REQUIRED_SCOPE); - - if (!hasCredentials) { - console.log("Google OAuth credentials not available or missing required Gmail scope. Sleeping..."); - } else { - // Perform one sync - await performSync(); - } + await performSync(); } } catch (error) { console.error("Error in main loop:", error); diff --git a/apps/x/packages/core/src/migrations/composio-google-migration.ts b/apps/x/packages/core/src/migrations/composio-google-migration.ts new file mode 100644 index 00000000..3e8f699d --- /dev/null +++ b/apps/x/packages/core/src/migrations/composio-google-migration.ts @@ -0,0 +1,132 @@ +import fs from 'fs'; +import path from 'path'; +import { z } from 'zod'; +import { WorkDir } from '../config/config.js'; +import { isSignedIn } from '../account/account.js'; +import { composioAccountsRepo } from '../composio/repo.js'; +import { deleteConnectedAccount } from '../composio/client.js'; +import container from '../di/container.js'; +import { IOAuthRepo } from '../auth/repo.js'; + +/** + * One-time migration that moves Composio-connected Gmail/Calendar users + * to the native rowboat-mode Google OAuth flow. + * + * Triggered by the renderer on app launch and after Rowboat sign-in. The + * single guard is `dismissed_at` in the migration state file — once set, + * none of the migration's side effects run again. This protects users who + * later re-add Composio Google for non-sync purposes (e.g. a tool that + * happens to use the Gmail toolkit) from having that connection blown + * away on a future launch. + */ + +const STATE_FILE = path.join(WorkDir, 'config', 'composio-google-migration.json'); + +const ZState = z.object({ + dismissed_at: z.string().min(1).optional(), +}); +type State = z.infer; + +function loadState(): State { + try { + if (fs.existsSync(STATE_FILE)) { + const raw = fs.readFileSync(STATE_FILE, 'utf-8'); + return ZState.parse(JSON.parse(raw)); + } + } catch (error) { + console.error('[composio-google-migration] failed to load state:', error); + } + return {}; +} + +function saveState(state: State): void { + const dir = path.dirname(STATE_FILE); + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2)); +} + +function markDismissed(): void { + saveState({ dismissed_at: new Date().toISOString() }); +} + +async function disconnectComposioGoogle(): Promise { + for (const slug of ['gmail', 'googlecalendar'] as const) { + const account = composioAccountsRepo.getAccount(slug); + if (!account?.id) continue; + + try { + await deleteConnectedAccount(account.id); + console.log(`[composio-google-migration] composio: deleted ${slug} (${account.id})`); + } catch (error) { + // Best-effort — logged but doesn't block the local cleanup. + console.warn(`[composio-google-migration] composio delete failed for ${slug}:`, error); + } + + try { + composioAccountsRepo.deleteAccount(slug); + } catch (error) { + console.warn(`[composio-google-migration] local delete failed for ${slug}:`, error); + } + } +} + +function cleanupCalendarComposioState(): void { + const file = path.join(WorkDir, 'calendar_sync', 'composio_state.json'); + try { + if (fs.existsSync(file)) { + fs.unlinkSync(file); + console.log('[composio-google-migration] removed stale calendar composio_state.json'); + } + } catch (error) { + console.warn('[composio-google-migration] failed to remove composio_state.json:', error); + } +} + +/** + * Check whether the user qualifies for the migration. If they do, atomically + * mark `dismissed_at`, fire-and-forget the Composio disconnect, and return + * `{shouldShow: true}` so the renderer can show the modal. + * + * Idempotent: subsequent calls return `{shouldShow: false}` once `dismissed_at` + * is set, regardless of whether the modal was actually shown or the user + * completed the OAuth flow. + */ +export async function qualifyAndDisconnectComposioGoogle(): Promise<{ shouldShow: boolean }> { + // Rule 4 — already processed + const state = loadState(); + if (state.dismissed_at) { + return { shouldShow: false }; + } + + // Rule 1 — must be signed in to Rowboat + if (!(await isSignedIn())) { + return { shouldShow: false }; + } + + // Rule 3 — already on native rowboat-mode Google → silently mark dismissed + // (so we stop re-checking) and bail before touching Composio state. + const oauthRepo = container.resolve('oauthRepo'); + const googleConnection = await oauthRepo.read('google'); + if (googleConnection.tokens && googleConnection.mode === 'rowboat') { + markDismissed(); + return { shouldShow: false }; + } + + // Rule 2 — must have at least one Composio Google toolkit connected + const hasGmail = composioAccountsRepo.isConnected('gmail'); + const hasCalendar = composioAccountsRepo.isConnected('googlecalendar'); + if (!hasGmail && !hasCalendar) { + return { shouldShow: false }; + } + + // All rules pass. Mark dismissed atomically before any side effects so + // a crash mid-migration leaves us in a deterministic post-migration state. + markDismissed(); + + // Fire-and-forget: disconnect Composio Google + clean up the stale + // calendar state file. Both are best-effort. + void disconnectComposioGoogle(); + cleanupCalendarComposioState(); + + return { shouldShow: true }; +} diff --git a/apps/x/packages/shared/src/ipc.ts b/apps/x/packages/shared/src/ipc.ts index ab7d7f73..605b26d9 100644 --- a/apps/x/packages/shared/src/ipc.ts +++ b/apps/x/packages/shared/src/ipc.ts @@ -429,16 +429,10 @@ const ipcSchemas = { toolkits: z.array(z.string()), }), }, - 'composio:use-composio-for-google': { + 'migration:check-composio-google': { req: z.null(), res: z.object({ - enabled: z.boolean(), - }), - }, - 'composio:use-composio-for-google-calendar': { - req: z.null(), - res: z.object({ - enabled: z.boolean(), + shouldShow: z.boolean(), }), }, 'composio:didConnect': { From c382e3ee8afa318a2dc53f7d615b65f804e86b92 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Tue, 5 May 2026 16:07:37 +0530 Subject: [PATCH 07/22] use gemini as default kg model --- apps/x/packages/core/src/models/defaults.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/x/packages/core/src/models/defaults.ts b/apps/x/packages/core/src/models/defaults.ts index 66dda9e0..dc690d66 100644 --- a/apps/x/packages/core/src/models/defaults.ts +++ b/apps/x/packages/core/src/models/defaults.ts @@ -6,7 +6,7 @@ import container from "../di/container.js"; const SIGNED_IN_DEFAULT_MODEL = "gpt-5.4"; const SIGNED_IN_DEFAULT_PROVIDER = "rowboat"; -const SIGNED_IN_KG_MODEL = "anthropic/claude-haiku-4.5"; +const SIGNED_IN_KG_MODEL = "google/gemini-3.1-flash-lite-preview"; const SIGNED_IN_TRACK_BLOCK_MODEL = "anthropic/claude-haiku-4.5"; /** From c6083de05438a893e5f324fe472b2a2ccba74baf Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Tue, 5 May 2026 19:21:32 +0530 Subject: [PATCH 08/22] show errors in activity tab for knowledge graph --- .../src/components/sidebar-content.tsx | 73 +++++++++++++++++-- apps/x/packages/core/src/agents/utils.ts | 62 +++++++++++++--- .../core/src/knowledge/agent_notes.ts | 15 +++- .../core/src/knowledge/build_graph.ts | 13 ++-- .../core/src/knowledge/label_emails.ts | 21 ++++-- .../packages/core/src/knowledge/tag_notes.ts | 21 ++++-- 6 files changed, 171 insertions(+), 34 deletions(-) diff --git a/apps/x/apps/renderer/src/components/sidebar-content.tsx b/apps/x/apps/renderer/src/components/sidebar-content.tsx index 7e204781..9c50c334 100644 --- a/apps/x/apps/renderer/src/components/sidebar-content.tsx +++ b/apps/x/apps/renderer/src/components/sidebar-content.tsx @@ -156,6 +156,28 @@ const SERVICE_LABELS: Record = { granola: "Syncing Granola", graph: "Updating knowledge", voice_memo: "Processing voice memo", + email_labeling: "Labeling emails", + note_tagging: "Tagging notes", + agent_notes: "Updating agent notes", +} + +function summarizeServiceError(error: string): string { + const firstLine = error.split("\n").find((line) => line.trim().length > 0) + return firstLine?.trim() || error.trim() +} + +function collectServiceErrors(events: ServiceEventType[]): Map { + const errors = new Map() + for (const event of events) { + if (event.type === "error") { + errors.set(event.service, summarizeServiceError(event.error)) + continue + } + if (event.type === "run_complete" && event.outcome !== "error") { + errors.delete(event.service) + } + } + return errors } type TasksActions = { @@ -227,6 +249,7 @@ function formatRunTime(ts: string): string { function SyncStatusBar() { const { state } = useSidebar() const [activeServices, setActiveServices] = useState>(new Map()) + const [serviceErrors, setServiceErrors] = useState>(new Map()) const [popoverOpen, setPopoverOpen] = useState(false) const [logEvents, setLogEvents] = useState([]) const [logLoading, setLogLoading] = useState(false) @@ -260,11 +283,25 @@ function SyncStatusBar() { next.delete(nextEvent.runId) return next }) + if (nextEvent.outcome !== 'error') { + setServiceErrors((prev) => { + if (!prev.has(nextEvent.service)) return prev + const next = new Map(prev) + next.delete(nextEvent.service) + return next + }) + } const existingTimeout = runTimeoutsRef.current.get(nextEvent.runId) if (existingTimeout) { clearTimeout(existingTimeout) runTimeoutsRef.current.delete(nextEvent.runId) } + } else if (nextEvent.type === 'error') { + setServiceErrors((prev) => { + const next = new Map(prev) + next.set(nextEvent.service, summarizeServiceError(nextEvent.error)) + return next + }) } }) return cleanup @@ -298,10 +335,14 @@ function SyncStatusBar() { // skip malformed lines } } + setServiceErrors(collectServiceErrors(parsed)) // Newest first, limit to 1000 setLogEvents(parsed.reverse().slice(0, MAX_SYNC_EVENTS)) } catch { - if (!cancelled) setLogEvents([]) + if (!cancelled) { + setLogEvents([]) + setServiceErrors(new Map()) + } } finally { if (!cancelled) setLogLoading(false) } @@ -312,12 +353,19 @@ function SyncStatusBar() { const isSyncing = activeServices.size > 0 const isCollapsed = state === "collapsed" + const errorEntries = Array.from(serviceErrors.entries()) + const primaryErrorService = errorEntries[0]?.[0] ?? null + const hasServiceErrors = errorEntries.length > 0 // Build status label from active services const activeServiceNames = [...new Set(activeServices.values())] const statusLabel = isSyncing ? activeServiceNames.map((s) => SERVICE_LABELS[s] || s).join(", ") - : "All caught up" + : hasServiceErrors + ? errorEntries.length === 1 + ? `${SERVICE_LABELS[primaryErrorService ?? ""] || primaryErrorService} failed` + : "Recent sync issues" + : "All caught up" return ( <> @@ -335,11 +383,16 @@ function SyncStatusBar() { + +

+ Notes that contain track blocks. Toggle a note inactive to pause every background agent in it. +

+ +
+ {loading ? ( +
+ +
+ ) : error ? ( +
+
+ +
+

{error}

+
+ ) : notes.length === 0 ? ( +
+
+ +
+

+ No notes with background agents yet. +

+
+ ) : ( +
+ + + + + + + + + + + {notes.map((note) => { + const isUpdating = updatingPaths.has(note.path) + return ( + + + + + + + ) + })} + +
NoteCreated dateLast ranState
+
+
+ + + {note.trackCount} {note.trackCount === 1 ? 'agent' : 'agents'} + +
+
+ {stripKnowledgePrefix(note.path)} +
+
+ 		+ {formatDateLabel(note.createdAt)} + + {formatDateTimeLabel(note.lastRunAt)} + +
+ {isUpdating ? ( + + ) : ( +
+
+
+ )} +
+ + ) +} diff --git a/apps/x/apps/renderer/src/components/sidebar-content.tsx b/apps/x/apps/renderer/src/components/sidebar-content.tsx index 9c50c334..dc49307c 100644 --- a/apps/x/apps/renderer/src/components/sidebar-content.tsx +++ b/apps/x/apps/renderer/src/components/sidebar-content.tsx @@ -214,6 +214,8 @@ type SidebarContentPanelProps = { onToggleBrowser?: () => void isSuggestedTopicsOpen?: boolean onOpenSuggestedTopics?: () => void + isBackgroundAgentsOpen?: boolean + onOpenBackgroundAgents?: () => void } & React.ComponentProps const sectionTabs: { id: ActiveSection; label: string }[] = [ @@ -491,6 +493,8 @@ export function SidebarContentPanel({ onToggleBrowser, isSuggestedTopicsOpen = false, onOpenSuggestedTopics, + isBackgroundAgentsOpen = false, + onOpenBackgroundAgents, ...props }: SidebarContentPanelProps) { const { activeSection, setActiveSection } = useSidebarSection() @@ -506,6 +510,7 @@ export function SidebarContentPanel({ const isMeetingQuickActionSelected = isMeetingActionActive const isBrowserQuickActionSelected = isBrowserOpen && !isSearchOpen && !isMeetingQuickActionSelected const isSuggestedTopicsQuickActionSelected = isSuggestedTopicsOpen && !isBrowserOpen + const isBackgroundAgentsQuickActionSelected = isBackgroundAgentsOpen && !isBrowserOpen const handleRowboatLogin = useCallback(async () => { try { @@ -679,6 +684,21 @@ export function SidebarContentPanel({ Suggested Topics )} + {onOpenBackgroundAgents && ( + + )} diff --git a/apps/x/packages/core/src/application/assistant/skills/background-agents/skill.ts b/apps/x/packages/core/src/application/assistant/skills/background-agents/skill.ts deleted file mode 100644 index b8e481b6..00000000 --- a/apps/x/packages/core/src/application/assistant/skills/background-agents/skill.ts +++ /dev/null @@ -1,555 +0,0 @@ -export const skill = String.raw` -# Background Agents - -Load this skill whenever a user wants to inspect, create, edit, or schedule background agents inside the Rowboat workspace. - -## Core Concepts - -**IMPORTANT**: In the CLI, there are NO separate "workflow" files. Everything is an agent. - -- **All definitions live in ` + "`agents/*.md`" + `** - Markdown files with YAML frontmatter -- Agents configure a model, tools (in frontmatter), and instructions (in the body) -- Tools can be: builtin (like ` + "`executeCommand`" + `), MCP integrations, or **other agents** -- **"Workflows" are just agents that orchestrate other agents** by having them as tools -- **Background agents run on schedules** defined in ` + "`config/agent-schedule.json`" + ` within the workspace root - -## How multi-agent workflows work - -1. **Create an orchestrator agent** that has other agents in its ` + "`tools`" + ` -2. **Schedule the orchestrator** in agent-schedule.json (see Scheduling section below) -3. The orchestrator calls other agents as tools when needed -4. Data flows through tool call parameters and responses - -## Scheduling Background Agents - -Background agents run automatically based on schedules defined in ` + "`config/agent-schedule.json`" + ` in the workspace root. - -### Schedule Configuration File - -` + "```json" + ` -{ - "agents": { - "agent_name": { - "schedule": { ... }, - "enabled": true - } - } -} -` + "```" + ` - -### Schedule Types - -**IMPORTANT: All times are in local time** (the timezone of the machine running Rowboat). - -**1. Cron Schedule** - Runs at exact times defined by cron expression -` + "```json" + ` -{ - "schedule": { - "type": "cron", - "expression": "0 8 * * *" - }, - "enabled": true -} -` + "```" + ` - -Common cron expressions: -- ` + "`*/5 * * * *`" + ` - Every 5 minutes -- ` + "`0 8 * * *`" + ` - Every day at 8am -- ` + "`0 9 * * 1`" + ` - Every Monday at 9am -- ` + "`0 0 1 * *`" + ` - First day of every month at midnight - -**2. Window Schedule** - Runs once during a time window -` + "```json" + ` -{ - "schedule": { - "type": "window", - "cron": "0 0 * * *", - "startTime": "08:00", - "endTime": "10:00" - }, - "enabled": true -} -` + "```" + ` - -The agent will run once at a random time within the window. Use this when you want flexibility (e.g., "sometime in the morning" rather than "exactly at 8am"). - -**3. Once Schedule** - Runs exactly once at a specific time -` + "```json" + ` -{ - "schedule": { - "type": "once", - "runAt": "2024-02-05T10:30:00" - }, - "enabled": true -} -` + "```" + ` - -Use this for one-time tasks like migrations or setup scripts. The ` + "`runAt`" + ` is in local time (no Z suffix). - -### Starting Message - -You can specify a ` + "`startingMessage`" + ` that gets sent to the agent when it starts. If not provided, defaults to ` + "`\"go\"`" + `. - -` + "```json" + ` -{ - "schedule": { "type": "cron", "expression": "0 8 * * *" }, - "enabled": true, - "startingMessage": "Please summarize my emails from the last 24 hours" -} -` + "```" + ` - -### Description - -You can add a ` + "`description`" + ` field to describe what the agent does. This is displayed in the UI. - -` + "```json" + ` -{ - "schedule": { "type": "cron", "expression": "0 8 * * *" }, - "enabled": true, - "description": "Summarizes emails and calendar events every morning" -} -` + "```" + ` - -### Complete Schedule Example - -` + "```json" + ` -{ - "agents": { - "daily_digest": { - "schedule": { - "type": "cron", - "expression": "0 8 * * *" - }, - "enabled": true, - "description": "Daily email and calendar summary", - "startingMessage": "Summarize my emails and calendar for today" - }, - "morning_briefing": { - "schedule": { - "type": "window", - "cron": "0 0 * * *", - "startTime": "07:00", - "endTime": "09:00" - }, - "enabled": true, - "description": "Morning news and updates briefing" - }, - "one_time_setup": { - "schedule": { - "type": "once", - "runAt": "2024-12-01T12:00:00" - }, - "enabled": true, - "description": "One-time data migration task" - } - } -} -` + "```" + ` - -### Schedule State (Read-Only) - -**IMPORTANT: Do NOT modify ` + "`agent-schedule-state.json`" + `** - it is managed automatically by the background runner. - -The runner automatically tracks execution state in ` + "`config/agent-schedule-state.json`" + ` in the workspace root: -- ` + "`status`" + `: scheduled, running, finished, failed, triggered (for once-schedules) -- ` + "`lastRunAt`" + `: When the agent last ran -- ` + "`nextRunAt`" + `: When the agent will run next -- ` + "`lastError`" + `: Error message if the last run failed -- ` + "`runCount`" + `: Total number of runs - -When you add an agent to ` + "`agent-schedule.json`" + `, the runner will automatically create and manage its state entry. You only need to edit ` + "`agent-schedule.json`" + `. - -## Agent File Format - -Agent files are **Markdown files with YAML frontmatter**. The frontmatter contains configuration (model, tools), and the body contains the instructions. - -### Basic Structure -` + "```markdown" + ` ---- -model: gpt-5.1 -tools: - tool_key: - type: builtin - name: tool_name ---- -# Instructions - -Your detailed instructions go here in Markdown format. -` + "```" + ` - -### Frontmatter Fields -- ` + "`model`" + `: (OPTIONAL) Model to use (e.g., 'gpt-5.1', 'claude-sonnet-4-5') -- ` + "`provider`" + `: (OPTIONAL) Provider alias from models.json -- ` + "`tools`" + `: (OPTIONAL) Object containing tool definitions - -### Instructions (Body) -The Markdown body after the frontmatter contains the agent's instructions. Use standard Markdown formatting. - -### Naming Rules -- Agent filename determines the agent name (without .md extension) -- Example: ` + "`summariser_agent.md`" + ` creates an agent named "summariser_agent" -- Use lowercase with underscores for multi-word names -- No spaces or special characters in names -- **The agent name in agent-schedule.json must match the filename** (without .md) - -### Agent Format Example -` + "```markdown" + ` ---- -model: gpt-5.1 -tools: - search: - type: mcp - name: firecrawl_search - description: Search the web - mcpServerName: firecrawl - inputSchema: - type: object - properties: - query: - type: string - description: Search query - required: - - query ---- -# Web Search Agent - -You are a web search agent. When asked a question: - -1. Use the search tool to find relevant information -2. Summarize the results clearly -3. Cite your sources - -Be concise and accurate. -` + "```" + ` - -## Tool Types & Schemas - -Tools in agents must follow one of three types. Each has specific required fields. - -### 1. Builtin Tools -Internal Rowboat tools (executeCommand, file operations, MCP queries, etc.) - -**YAML Schema:** -` + "```yaml" + ` -tool_key: - type: builtin - name: tool_name -` + "```" + ` - -**Required fields:** -- ` + "`type`" + `: Must be "builtin" -- ` + "`name`" + `: Builtin tool name (e.g., "executeCommand", "workspace-readFile") - -**Example:** -` + "```yaml" + ` -bash: - type: builtin - name: executeCommand -` + "```" + ` - -**Available builtin tools:** -- ` + "`executeCommand`" + ` - Execute shell commands -- ` + "`workspace-readFile`" + `, ` + "`workspace-writeFile`" + `, ` + "`workspace-remove`" + ` - File operations -- ` + "`workspace-readdir`" + `, ` + "`workspace-exists`" + `, ` + "`workspace-stat`" + ` - Directory operations -- ` + "`workspace-mkdir`" + `, ` + "`workspace-rename`" + `, ` + "`workspace-copy`" + ` - File/directory management -- ` + "`analyzeAgent`" + ` - Analyze agent structure -- ` + "`addMcpServer`" + `, ` + "`listMcpServers`" + `, ` + "`listMcpTools`" + ` - MCP management -- ` + "`loadSkill`" + ` - Load skill guidance - -### 2. MCP Tools -Tools from external MCP servers (APIs, databases, web scraping, etc.) - -**YAML Schema:** -` + "```yaml" + ` -tool_key: - type: mcp - name: tool_name_from_server - description: What the tool does - mcpServerName: server_name_from_config - inputSchema: - type: object - properties: - param: - type: string - description: Parameter description - required: - - param -` + "```" + ` - -**Required fields:** -- ` + "`type`" + `: Must be "mcp" -- ` + "`name`" + `: Exact tool name from MCP server -- ` + "`description`" + `: What the tool does (helps agent understand when to use it) -- ` + "`mcpServerName`" + `: Server name from config/mcp.json -- ` + "`inputSchema`" + `: Full JSON Schema object for tool parameters - -**Example:** -` + "```yaml" + ` -search: - type: mcp - name: firecrawl_search - description: Search the web - mcpServerName: firecrawl - inputSchema: - type: object - properties: - query: - type: string - description: Search query - required: - - query -` + "```" + ` - -**Important:** -- Use ` + "`listMcpTools`" + ` to get the exact inputSchema from the server -- Copy the schema exactly—don't modify property types or structure -- Only include ` + "`required`" + ` array if parameters are mandatory - -### 3. Agent Tools (for chaining agents) -Reference other agents as tools to build multi-agent workflows - -**YAML Schema:** -` + "```yaml" + ` -tool_key: - type: agent - name: target_agent_name -` + "```" + ` - -**Required fields:** -- ` + "`type`" + `: Must be "agent" -- ` + "`name`" + `: Name of the target agent (must exist in agents/ directory) - -**Example:** -` + "```yaml" + ` -summariser: - type: agent - name: summariser_agent -` + "```" + ` - -**How it works:** -- Use ` + "`type: agent`" + ` to call other agents as tools -- The target agent will be invoked with the parameters you pass -- Results are returned as tool output -- This is how you build multi-agent workflows -- The referenced agent file must exist (e.g., ` + "`agents/summariser_agent.md`" + `) - -## Complete Multi-Agent Workflow Example - -**Email digest workflow** - This is all done through agents calling other agents: - -**1. Task-specific agent** (` + "`agents/email_reader.md`" + `): -` + "```markdown" + ` ---- -model: gpt-5.1 -tools: - read_file: - type: builtin - name: workspace-readFile - list_dir: - type: builtin - name: workspace-readdir ---- -# Email Reader Agent - -Read emails from the gmail_sync folder and extract key information. -Look for unread or recent emails and summarize the sender, subject, and key points. -Don't ask for human input. -` + "```" + ` - -**2. Agent that delegates to other agents** (` + "`agents/daily_summary.md`" + `): -` + "```markdown" + ` ---- -model: gpt-5.1 -tools: - email_reader: - type: agent - name: email_reader - write_file: - type: builtin - name: workspace-writeFile ---- -# Daily Summary Agent - -1. Use the email_reader tool to get email summaries -2. Create a consolidated daily digest -3. Save the digest to ~/Desktop/daily_digest.md - -Don't ask for human input. -` + "```" + ` - -Note: The output path (` + "`~/Desktop/daily_digest.md`" + `) is hardcoded in the instructions. When creating agents that output files, always ask the user where they want files saved and include the full path in the agent instructions. - -**3. Orchestrator agent** (` + "`agents/morning_briefing.md`" + `): -` + "```markdown" + ` ---- -model: gpt-5.1 -tools: - daily_summary: - type: agent - name: daily_summary - search: - type: mcp - name: search - mcpServerName: exa - description: Search the web for news - inputSchema: - type: object - properties: - query: - type: string - description: Search query ---- -# Morning Briefing Workflow - -Create a morning briefing: - -1. Get email digest using daily_summary -2. Search for relevant news using the search tool -3. Compile a comprehensive morning briefing - -Execute these steps in sequence. Don't ask for human input. -` + "```" + ` - -**4. Schedule the workflow** in ` + "`config/agent-schedule.json`" + `: -` + "```json" + ` -{ - "agents": { - "morning_briefing": { - "schedule": { - "type": "cron", - "expression": "0 7 * * *" - }, - "enabled": true, - "startingMessage": "Create my morning briefing for today" - } - } -} -` + "```" + ` - -This schedules the morning briefing workflow to run every day at 7am local time. - -## Naming and organization rules -- **All agents live in ` + "`agents/*.md`" + `** - Markdown files with YAML frontmatter -- Agent filename (without .md) becomes the agent name -- When referencing an agent as a tool, use its filename without extension -- When scheduling an agent, use its filename without extension in agent-schedule.json -- Use relative paths (no \${BASE_DIR} prefixes) when giving examples to users - -## Best practices for background agents -1. **Single responsibility**: Each agent should do one specific thing well -2. **Clear delegation**: Agent instructions should explicitly say when to call other agents -3. **Autonomous operation**: Add "Don't ask for human input" for background agents -4. **Data passing**: Make it clear what data to extract and pass between agents -5. **Tool naming**: Use descriptive tool keys (e.g., "summariser", "fetch_data", "analyze") -6. **Orchestration**: Create a top-level agent that coordinates the workflow -7. **Scheduling**: Use appropriate schedule types - cron for recurring, window for flexible timing, once for one-time tasks -8. **Error handling**: Background agents should handle errors gracefully since there's no human to intervene -9. **Avoid executeCommand**: Do NOT attach ` + "`executeCommand`" + ` to background agents as it poses security risks when running unattended. Instead, use the specific builtin tools needed (` + "`workspace-readFile`" + `, ` + "`workspace-writeFile`" + `, etc.) or MCP tools for external integrations -10. **File output paths**: When creating an agent that outputs files, ASK the user where the file should be stored (default to Desktop: ` + "`~/Desktop`" + `). Then hardcode the full output path in the agent's instructions so it knows exactly where to write files. Example instruction: "Save the output to /Users/username/Desktop/daily_report.md" - -## Validation & Best Practices - -### CRITICAL: Schema Compliance -- Agent files MUST be valid Markdown with YAML frontmatter -- Agent filename (without .md) becomes the agent name -- Tools in frontmatter MUST have valid ` + "`type`" + ` ("builtin", "mcp", or "agent") -- MCP tools MUST have all required fields: name, description, mcpServerName, inputSchema -- Agent tools MUST reference existing agent files -- Invalid agents will fail to load and prevent workflow execution - -### File Creation/Update Process -1. When creating an agent, use ` + "`workspace-writeFile`" + ` with valid Markdown + YAML frontmatter -2. When updating an agent, read it first with ` + "`workspace-readFile`" + `, modify, then use ` + "`workspace-writeFile`" + ` -3. Validate YAML syntax in frontmatter before writing—malformed YAML breaks the agent -4. **Quote strings containing colons** (e.g., ` + "`description: \"Default: 8\"`" + ` not ` + "`description: Default: 8`" + `) -5. Test agent loading after creation/update by using ` + "`analyzeAgent`" + ` - -### Common Validation Errors to Avoid - -❌ **WRONG - Missing frontmatter delimiters:** -` + "```markdown" + ` -model: gpt-5.1 -# My Agent -Instructions here -` + "```" + ` - -❌ **WRONG - Invalid YAML indentation:** -` + "```markdown" + ` ---- -tools: -bash: - type: builtin ---- -` + "```" + ` -(bash should be indented under tools) - -❌ **WRONG - Invalid tool type:** -` + "```yaml" + ` -tools: - tool1: - type: custom - name: something -` + "```" + ` -(type must be builtin, mcp, or agent) - -❌ **WRONG - Unquoted strings containing colons:** -` + "```yaml" + ` -tools: - search: - description: Number of results (default: 8) -` + "```" + ` -(Strings with colons must be quoted: ` + "`description: \"Number of results (default: 8)\"`" + `) - -❌ **WRONG - MCP tool missing required fields:** -` + "```yaml" + ` -tools: - search: - type: mcp - name: firecrawl_search -` + "```" + ` -(Missing: description, mcpServerName, inputSchema) - -✅ **CORRECT - Minimal valid agent** (` + "`agents/simple_agent.md`" + `): -` + "```markdown" + ` ---- -model: gpt-5.1 ---- -# Simple Agent - -Do simple tasks as instructed. -` + "```" + ` - -✅ **CORRECT - Agent with MCP tool** (` + "`agents/search_agent.md`" + `): -` + "```markdown" + ` ---- -model: gpt-5.1 -tools: - search: - type: mcp - name: firecrawl_search - description: Search the web - mcpServerName: firecrawl - inputSchema: - type: object - properties: - query: - type: string ---- -# Search Agent - -Use the search tool to find information on the web. -` + "```" + ` - -## Capabilities checklist -1. Explore ` + "`agents/`" + ` directory to understand existing agents before editing -2. Read existing agents with ` + "`workspace-readFile`" + ` before making changes -3. Validate YAML frontmatter syntax before creating/updating agents -4. Use ` + "`analyzeAgent`" + ` to verify agent structure after creation/update -5. When creating multi-agent workflows, create an orchestrator agent -6. Add other agents as tools with ` + "`type: agent`" + ` for chaining -7. Use ` + "`listMcpServers`" + ` and ` + "`listMcpTools`" + ` when adding MCP integrations -8. Configure schedules in ` + "`config/agent-schedule.json`" + ` (ONLY edit this file, NOT the state file) -9. Confirm work done and outline next steps once changes are complete -`; - -export default skill; diff --git a/apps/x/packages/core/src/application/assistant/skills/index.ts b/apps/x/packages/core/src/application/assistant/skills/index.ts index 6d3cdc5b..f4ba9b1d 100644 --- a/apps/x/packages/core/src/application/assistant/skills/index.ts +++ b/apps/x/packages/core/src/application/assistant/skills/index.ts @@ -7,7 +7,6 @@ import draftEmailsSkill from "./draft-emails/skill.js"; import mcpIntegrationSkill from "./mcp-integration/skill.js"; import meetingPrepSkill from "./meeting-prep/skill.js"; import organizeFilesSkill from "./organize-files/skill.js"; -import backgroundAgentsSkill from "./background-agents/skill.js"; import createPresentationsSkill from "./create-presentations/skill.js"; import appNavigationSkill from "./app-navigation/skill.js"; @@ -65,12 +64,6 @@ const definitions: SkillDefinition[] = [ summary: "Find, organize, and tidy up files on the user's machine. Move files to folders, clean up Desktop/Downloads, locate specific files.", content: organizeFilesSkill, }, - { - id: "background-agents", - title: "Background Agents", - summary: "Creating, editing, and scheduling background agents. Configure schedules in agent-schedule.json and build multi-agent workflows.", - content: backgroundAgentsSkill, - }, { id: "builtin-tools", title: "Builtin Tools Reference", diff --git a/apps/x/packages/core/src/application/assistant/skills/tracks/skill.ts b/apps/x/packages/core/src/application/assistant/skills/tracks/skill.ts index 17521806..c9624c66 100644 --- a/apps/x/packages/core/src/application/assistant/skills/tracks/skill.ts +++ b/apps/x/packages/core/src/application/assistant/skills/tracks/skill.ts @@ -349,6 +349,20 @@ In that flow: 6. Keep the surrounding note scaffolding minimal but useful. The track block should be the core of the note. 7. If the target folder is one of the structured knowledge folders (` + "`" + `knowledge/People/` + "`" + `, ` + "`" + `knowledge/Organizations/` + "`" + `, ` + "`" + `knowledge/Projects/` + "`" + `, ` + "`" + `knowledge/Topics/` + "`" + `), mirror the local note style by quickly checking a nearby note or config before writing if needed. +### Background agent setup flow + +Sometimes the user arrives from the Background agents panel and wants help creating a new background agent without naming a note yet. + +In this flow, treat "background agent" and "track block" as the same feature. The user-facing term can stay "background agent", but the implementation is a track block inside a note. Do **not** claim these are different systems, and do **not** redirect the user toward standalone agent files or ` + "`" + `agent-schedule.json` + "`" + ` unless they explicitly ask for that architecture. + +In that flow: +1. On the first turn, **do not create or modify anything yet**. Briefly explain what you can set up, say you will put it in ` + "`" + `knowledge/Tasks/` + "`" + ` by default, and ask what it should monitor plus how often it should run. +2. **Do not** ask the user where the results should live unless they explicitly said they want a different folder or there is a real ambiguity you cannot resolve. +3. If the user clearly confirms later, treat ` + "`" + `knowledge/Tasks/` + "`" + ` as the default target folder. +4. Before creating a new note there, search ` + "`" + `knowledge/Tasks/` + "`" + ` for an existing matching note and update it if one already exists. +5. If ` + "`" + `knowledge/Tasks/` + "`" + ` does not exist, create it as part of setup instead of bouncing back to ask. +6. Keep the surrounding note scaffolding minimal but useful. The track block should be the core of the note. + ## The Exact Text to Insert Write it verbatim like this (including the blank line between fence and target): diff --git a/apps/x/packages/core/src/knowledge/track/fileops.ts b/apps/x/packages/core/src/knowledge/track/fileops.ts index bd731823..b7ade8de 100644 --- a/apps/x/packages/core/src/knowledge/track/fileops.ts +++ b/apps/x/packages/core/src/knowledge/track/fileops.ts @@ -70,6 +70,122 @@ export async function fetch(filePath: string, trackId: string): Promise b.track.trackId === trackId) ?? null; } +type TrackNoteSummary = { + path: string; + trackCount: number; + createdAt: string | null; + lastRunAt: string | null; + isActive: boolean; +}; + +async function summarizeTrackNote( + filePath: string, + tracks: z.infer[], +): Promise { + if (tracks.length === 0) return null; + + const stats = await fs.stat(absPath(filePath)); + const createdMs = stats.birthtimeMs > 0 ? stats.birthtimeMs : stats.ctimeMs; + + let latestRunAt: string | null = null; + let latestRunMs = -1; + for (const { track } of tracks) { + if (!track.lastRunAt) continue; + const candidateMs = Date.parse(track.lastRunAt); + if (Number.isNaN(candidateMs) || candidateMs <= latestRunMs) continue; + latestRunMs = candidateMs; + latestRunAt = track.lastRunAt; + } + + return { + path: `knowledge/${filePath}`, + trackCount: tracks.length, + createdAt: createdMs > 0 ? new Date(createdMs).toISOString() : null, + lastRunAt: latestRunAt, + isActive: tracks.every(({ track }) => track.active !== false), + }; +} + +export async function listNotesWithTracks(): Promise { + async function walk(relativeDir = ''): Promise { + const dirPath = absPath(relativeDir); + try { + const entries = await fs.readdir(dirPath, { withFileTypes: true }); + const files: string[] = []; + + for (const entry of entries) { + if (entry.name.startsWith('.')) continue; + + const childRelPath = relativeDir + ? path.posix.join(relativeDir, entry.name) + : entry.name; + + if (entry.isDirectory()) { + files.push(...await walk(childRelPath)); + continue; + } + + if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) { + files.push(childRelPath); + } + } + + return files; + } catch { + return []; + } + } + + const markdownFiles = await walk(); + const notes = await Promise.all(markdownFiles.map(async (relativePath) => { + try { + const tracks = await fetchAll(relativePath); + return await summarizeTrackNote(relativePath, tracks); + } catch { + return null; + } + })); + + return notes + .filter((note): note is TrackNoteSummary => note !== null) + .sort((a, b) => { + const aName = path.basename(a.path, '.md').toLowerCase(); + const bName = path.basename(b.path, '.md').toLowerCase(); + if (aName !== bName) return aName.localeCompare(bName); + return a.path.localeCompare(b.path); + }); +} + +export async function setNoteTracksActive(filePath: string, active: boolean): Promise { + return withFileLock(absPath(filePath), async () => { + const blocks = await fetchAll(filePath); + if (blocks.length === 0) return null; + + const alreadyMatches = blocks.every(({ track }) => (track.active !== false) === active); + if (alreadyMatches) { + return summarizeTrackNote(filePath, blocks); + } + + const content = await fs.readFile(absPath(filePath), 'utf-8'); + const lines = content.split('\n'); + const updatedBlocks = blocks + .map((block) => ({ + ...block, + track: { ...block.track, active }, + })) + .sort((a, b) => b.fenceStart - a.fenceStart); + + for (const block of updatedBlocks) { + const yaml = stringifyYaml(block.track).trimEnd(); + const yamlLines = yaml ? yaml.split('\n') : []; + lines.splice(block.fenceStart, block.fenceEnd - block.fenceStart + 1, '```track', ...yamlLines, '```'); + } + + await fs.writeFile(absPath(filePath), lines.join('\n'), 'utf-8'); + return summarizeTrackNote(filePath, updatedBlocks); + }); +} + /** * Fetch a track block and return its canonical YAML string (or null if not found). * Useful for IPC handlers that need to return the fresh YAML without taking a @@ -196,4 +312,4 @@ export async function deleteTrackBlock(filePath: string, trackId: string): Promi await fs.writeFile(absPath(filePath), lines.join('\n'), 'utf-8'); }); -} \ No newline at end of file +} diff --git a/apps/x/packages/shared/src/ipc.ts b/apps/x/packages/shared/src/ipc.ts index 605b26d9..9e62f3d9 100644 --- a/apps/x/packages/shared/src/ipc.ts +++ b/apps/x/packages/shared/src/ipc.ts @@ -662,6 +662,35 @@ const ipcSchemas = { error: z.string().optional(), }), }, + 'track:setNoteActive': { + req: z.object({ + path: RelPath, + active: z.boolean(), + }), + res: z.object({ + success: z.boolean(), + note: z.object({ + path: RelPath, + trackCount: z.number().int().positive(), + createdAt: z.string().nullable(), + lastRunAt: z.string().nullable(), + isActive: z.boolean(), + }).optional(), + error: z.string().optional(), + }), + }, + 'track:listNotes': { + req: z.null(), + res: z.object({ + notes: z.array(z.object({ + path: RelPath, + trackCount: z.number().int().positive(), + createdAt: z.string().nullable(), + lastRunAt: z.string().nullable(), + isActive: z.boolean(), + })), + }), + }, // Embedded browser (WebContentsView) channels 'browser:setBounds': { req: z.object({ From 72ed4bd6d9f7caa3850c3105f7e128bf40c5e19f Mon Sep 17 00:00:00 2001 From: arkml <6592213+arkml@users.noreply.github.com> Date: Wed, 6 May 2026 12:25:10 +0530 Subject: [PATCH 11/22] pull browser-harness skills (#519) use browser-harness skill without eval or http-fetch --- .../apps/main/src/browser/control-service.ts | 30 ++- .../assistant/skills/browser-control/skill.ts | 17 +- .../src/application/browser-skills/index.ts | 3 + .../src/application/browser-skills/loader.ts | 215 ++++++++++++++++++ .../src/application/browser-skills/matcher.ts | 56 +++++ .../core/src/application/lib/builtin-tools.ts | 66 ++++++ apps/x/packages/shared/src/browser-control.ts | 8 + 7 files changed, 389 insertions(+), 6 deletions(-) create mode 100644 apps/x/packages/core/src/application/browser-skills/index.ts create mode 100644 apps/x/packages/core/src/application/browser-skills/loader.ts create mode 100644 apps/x/packages/core/src/application/browser-skills/matcher.ts diff --git a/apps/x/apps/main/src/browser/control-service.ts b/apps/x/apps/main/src/browser/control-service.ts index b83ea7cb..7c97ea7a 100644 --- a/apps/x/apps/main/src/browser/control-service.ts +++ b/apps/x/apps/main/src/browser/control-service.ts @@ -1,8 +1,24 @@ import type { IBrowserControlService } from '@x/core/dist/application/browser-control/service.js'; -import type { BrowserControlAction, BrowserControlInput, BrowserControlResult } from '@x/shared/dist/browser-control.js'; +import type { BrowserControlAction, BrowserControlInput, BrowserControlResult, SuggestedBrowserSkill } from '@x/shared/dist/browser-control.js'; +import { ensureLoaded, matchSkillsForUrl } from '@x/core/dist/application/browser-skills/index.js'; import { browserViewManager } from './view.js'; import { normalizeNavigationTarget } from './navigation.js'; +async function getSuggestedSkills(url: string | undefined): Promise { + if (!url) return undefined; + try { + const status = await ensureLoaded(); + if (status.status === 'ready' || status.status === 'stale') { + const matched = matchSkillsForUrl(status.index, url); + if (matched.length === 0) return undefined; + return matched.map((e) => ({ id: e.id, title: e.title, path: e.path })); + } + } catch (err) { + console.warn('[browser-control] suggestedSkills lookup failed:', err); + } + return undefined; +} + function buildSuccessResult( action: BrowserControlAction, message: string, @@ -52,11 +68,13 @@ export class ElectronBrowserControlService implements IBrowserControlService { } await browserViewManager.ensureActiveTabReady(signal); const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; - return buildSuccessResult( + const suggestedSkills = await getSuggestedSkills(page?.url); + const success = buildSuccessResult( 'new-tab', target ? `Opened a new tab for ${target}.` : 'Opened a new tab.', page, ); + return suggestedSkills ? { ...success, suggestedSkills } : success; } case 'switch-tab': { @@ -99,7 +117,9 @@ export class ElectronBrowserControlService implements IBrowserControlService { } await browserViewManager.ensureActiveTabReady(signal); const page = await browserViewManager.readPageSummary(signal, { waitForReady: false }) ?? undefined; - return buildSuccessResult('navigate', `Navigated to ${target}.`, page); + const suggestedSkills = await getSuggestedSkills(page?.url); + const success = buildSuccessResult('navigate', `Navigated to ${target}.`, page); + return suggestedSkills ? { ...success, suggestedSkills } : success; } case 'back': { @@ -140,7 +160,9 @@ export class ElectronBrowserControlService implements IBrowserControlService { if (!result.ok || !result.page) { return buildErrorResult('read-page', result.error ?? 'Failed to read the current page.'); } - return buildSuccessResult('read-page', 'Read the current page.', result.page); + const suggestedSkills = await getSuggestedSkills(result.page.url); + const success = buildSuccessResult('read-page', 'Read the current page.', result.page); + return suggestedSkills ? { ...success, suggestedSkills } : success; } case 'click': { diff --git a/apps/x/packages/core/src/application/assistant/skills/browser-control/skill.ts b/apps/x/packages/core/src/application/assistant/skills/browser-control/skill.ts index f1c06f0c..868ce8e8 100644 --- a/apps/x/packages/core/src/application/assistant/skills/browser-control/skill.ts +++ b/apps/x/packages/core/src/application/assistant/skills/browser-control/skill.ts @@ -14,8 +14,10 @@ Use this skill when the user asks you to open a website, browse in-app, search t - page ` + "`url`" + ` and ` + "`title`" + ` - visible page text - interactable elements with numbered ` + "`index`" + ` values -4. Prefer acting on those numbered indices with ` + "`click`" + ` / ` + "`type`" + ` / ` + "`press`" + `. -5. After each action, read the returned page snapshot before deciding the next step. + - ` + "`suggestedSkills`" + ` — site-specific and interaction-specific skill hints for the current page +4. **Always inspect ` + "`suggestedSkills`" + ` before acting.** If any skill in the list matches what the user asked for (site or task), call ` + "`load-browser-skill({ id: \"\" })`" + ` *first*, read it in full, then plan your actions. These skills encode selectors, timing, and gotchas that would otherwise cost you several failed attempts to rediscover. If no skill matches, proceed — but do not skip this check. +5. Prefer acting on those numbered indices with ` + "`click`" + ` / ` + "`type`" + ` / ` + "`press`" + `. +6. After each action, read the returned page snapshot before deciding the next step — including re-checking ` + "`suggestedSkills`" + ` if the navigation landed you on a new domain. ## Actions @@ -92,12 +94,23 @@ Wait for the page to settle, useful after async UI changes. Parameters: - ` + "`ms`" + `: milliseconds to wait (optional) +## Companion Tools + +### load-browser-skill +Rowboat caches a library of browser skills (from ` + "`browser-use/browser-harness`" + `) indexed by both **domain** (github, linkedin, amazon, booking, …) and **interaction type** within a domain (e.g. ` + "`github/repo-actions`" + `, ` + "`github/scraping`" + `, ` + "`arxiv-bulk/*`" + `). Whenever ` + "`browser-control`" + ` returns a ` + "`suggestedSkills`" + ` array — which it does on ` + "`navigate`" + `, ` + "`new-tab`" + `, and ` + "`read-page`" + ` — treat it as a required reading step, not optional. Pick the entry that matches the current task (domain match first, then the interaction-specific variant if one exists) and call ` + "`load-browser-skill({ id: \"\" })`" + ` before attempting the action. + +You can also proactively call ` + "`load-browser-skill({ action: \"list\", site: \"\" })`" + ` when you know you're about to work on a site, to see what skills exist even if ` + "`suggestedSkills`" + ` is empty (e.g. before navigating). + +These skills are written against a Python harness, so treat them as **reference knowledge**. Reuse the selectors, timing, and sequencing, but adapt them to Rowboat's structured browser actions. **Do not look for or call ` + "`http-fetch`" + `.** If a browser-harness recipe suggests ` + "`js(...)`" + ` or ` + "`http_get(...)`" + ` style shortcuts, treat those as non-portable and fall back to reading and interacting with the page itself. + ## Important Rules - Prefer ` + "`read-page`" + ` before interacting. - Prefer element ` + "`index`" + ` over CSS selectors. - If the tool says the snapshot is stale, call ` + "`read-page`" + ` again. - After navigation, clicking, typing, pressing, or scrolling, use the returned page snapshot instead of assuming the page state. +- **Always check ` + "`suggestedSkills`" + ` after ` + "`navigate`" + `, ` + "`new-tab`" + `, or ` + "`read-page`" + `, and load the matching domain or interaction skill before acting.** Skipping this step is the single most common way to waste a dozen failed clicks on a site whose quirks are already documented. If the array is empty, proceed normally — but don't skip the check. +- Do not try to use ` + "`http-fetch`" + `. If a browser-harness recipe mentions ` + "`http_get(...)`" + ` or a public API shortcut, adapt it to DOM-based browsing instead. - Use Rowboat's browser for live interaction. Use web search tools for research where a live session is unnecessary. - Do not wrap browser URLs or browser pages in ` + "```filepath" + ` blocks. Filepath cards are only for real files on disk, not web pages or browser tabs. - If you mention a page the browser opened, use plain text for the URL/title instead of trying to create a clickable file card. diff --git a/apps/x/packages/core/src/application/browser-skills/index.ts b/apps/x/packages/core/src/application/browser-skills/index.ts new file mode 100644 index 00000000..2040c963 --- /dev/null +++ b/apps/x/packages/core/src/application/browser-skills/index.ts @@ -0,0 +1,3 @@ +export { ensureLoaded, readSkillContent, refreshFromRemote } from './loader.js'; +export type { SkillEntry, SkillsIndex, LoaderStatus } from './loader.js'; +export { matchSkillsForUrl } from './matcher.js'; diff --git a/apps/x/packages/core/src/application/browser-skills/loader.ts b/apps/x/packages/core/src/application/browser-skills/loader.ts new file mode 100644 index 00000000..3e68d7ca --- /dev/null +++ b/apps/x/packages/core/src/application/browser-skills/loader.ts @@ -0,0 +1,215 @@ +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import { WorkDir } from '../../config/config.js'; + +const REPO_OWNER = 'browser-use'; +const REPO_NAME = 'browser-harness'; +const REPO_BRANCH = 'main'; +const DOMAIN_SKILLS_PREFIX = 'domain-skills/'; + +const MANIFEST_TTL_MS = 24 * 60 * 60 * 1000; +const FETCH_TIMEOUT_MS = 20_000; + +export type SkillEntry = { + id: string; // e.g. "github/repo-actions" + site: string; // e.g. "github" + fileName: string; // e.g. "repo-actions.md" + title: string; // first H1 from the markdown, or a derived title + path: string; // relative repo path, e.g. "domain-skills/github/repo-actions.md" + localPath: string; // absolute path on disk +}; + +export type SkillsIndex = { + fetchedAt: number; + treeSha: string; + entries: SkillEntry[]; +}; + +export type LoaderStatus = + | { status: 'ready'; index: SkillsIndex } + | { status: 'stale'; index: SkillsIndex; refreshing: boolean } + | { status: 'empty' } + | { status: 'error'; error: string }; + +const cacheRoot = () => path.join(WorkDir, 'cache', 'browser-skills'); +const skillsDir = () => path.join(cacheRoot(), 'domain-skills'); +const manifestPath = () => path.join(cacheRoot(), 'manifest.json'); + +async function ensureCacheDir(): Promise { + await fs.mkdir(skillsDir(), { recursive: true }); +} + +async function readManifest(): Promise { + try { + const raw = await fs.readFile(manifestPath(), 'utf8'); + const parsed = JSON.parse(raw) as SkillsIndex; + if (!parsed.entries || !Array.isArray(parsed.entries)) return null; + return parsed; + } catch { + return null; + } +} + +async function writeManifest(index: SkillsIndex): Promise { + await ensureCacheDir(); + await fs.writeFile(manifestPath(), JSON.stringify(index, null, 2), 'utf8'); +} + +function extractTitle(markdown: string, fallback: string): string { + const match = markdown.match(/^#\s+(.+?)\s*$/m); + if (match?.[1]) return match[1].trim(); + return fallback; +} + +async function fetchWithTimeout(url: string, init?: RequestInit): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + return await fetch(url, { + ...init, + signal: controller.signal, + headers: { + 'User-Agent': 'rowboat-browser-skills', + Accept: 'application/vnd.github+json', + ...(init?.headers ?? {}), + }, + }); + } finally { + clearTimeout(timer); + } +} + +type GithubTreeNode = { path: string; type: string; sha: string }; + +async function fetchRepoTree(): Promise<{ treeSha: string; skillPaths: string[] }> { + const branchUrl = `https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/branches/${REPO_BRANCH}`; + const branchRes = await fetchWithTimeout(branchUrl); + if (!branchRes.ok) { + throw new Error(`GitHub branch fetch failed: ${branchRes.status} ${branchRes.statusText}`); + } + const branch = (await branchRes.json()) as { commit: { commit: { tree: { sha: string } } } }; + const treeSha = branch.commit?.commit?.tree?.sha; + if (!treeSha) throw new Error('Could not resolve tree SHA from branch response.'); + + const treeUrl = `https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/git/trees/${treeSha}?recursive=1`; + const treeRes = await fetchWithTimeout(treeUrl); + if (!treeRes.ok) { + throw new Error(`GitHub tree fetch failed: ${treeRes.status} ${treeRes.statusText}`); + } + const tree = (await treeRes.json()) as { tree: GithubTreeNode[]; truncated: boolean }; + + const skillPaths = tree.tree + .filter((n) => n.type === 'blob' && n.path.startsWith(DOMAIN_SKILLS_PREFIX) && n.path.endsWith('.md')) + .map((n) => n.path); + + return { treeSha, skillPaths }; +} + +async function fetchRawFile(repoPath: string): Promise { + const url = `https://raw.githubusercontent.com/${REPO_OWNER}/${REPO_NAME}/${REPO_BRANCH}/${repoPath}`; + const res = await fetchWithTimeout(url, { headers: { Accept: 'text/plain' } }); + if (!res.ok) { + throw new Error(`Raw file fetch failed for ${repoPath}: ${res.status} ${res.statusText}`); + } + return res.text(); +} + +function parseRepoPath(repoPath: string): { id: string; site: string; fileName: string } | null { + const rel = repoPath.slice(DOMAIN_SKILLS_PREFIX.length); + const parts = rel.split('/'); + if (parts.length < 2) return null; + const site = parts[0]; + const fileName = parts.slice(1).join('/'); + const id = rel.replace(/\.md$/, ''); + return { id, site, fileName }; +} + +export async function refreshFromRemote(): Promise { + await ensureCacheDir(); + const { treeSha, skillPaths } = await fetchRepoTree(); + + const entries: SkillEntry[] = []; + await Promise.all(skillPaths.map(async (repoPath) => { + const parsed = parseRepoPath(repoPath); + if (!parsed) return; + try { + const content = await fetchRawFile(repoPath); + const localRel = path.join(parsed.site, parsed.fileName); + const localPath = path.join(skillsDir(), localRel); + await fs.mkdir(path.dirname(localPath), { recursive: true }); + await fs.writeFile(localPath, content, 'utf8'); + entries.push({ + id: parsed.id, + site: parsed.site, + fileName: parsed.fileName, + title: extractTitle(content, parsed.id), + path: repoPath, + localPath, + }); + } catch (err) { + console.warn(`[browser-skills] Failed to fetch ${repoPath}:`, err); + } + })); + + entries.sort((a, b) => a.id.localeCompare(b.id)); + + const index: SkillsIndex = { + fetchedAt: Date.now(), + treeSha, + entries, + }; + await writeManifest(index); + return index; +} + +let inFlightRefresh: Promise | null = null; + +export async function ensureLoaded(options?: { forceRefresh?: boolean }): Promise { + try { + const existing = await readManifest(); + const fresh = existing && Date.now() - existing.fetchedAt < MANIFEST_TTL_MS; + + if (existing && fresh && !options?.forceRefresh) { + return { status: 'ready', index: existing }; + } + + if (existing && !options?.forceRefresh) { + if (!inFlightRefresh) { + inFlightRefresh = refreshFromRemote() + .catch((err) => { + console.warn('[browser-skills] Background refresh failed:', err); + return existing; + }) + .finally(() => { inFlightRefresh = null; }); + } + return { status: 'stale', index: existing, refreshing: true }; + } + + if (!inFlightRefresh) { + inFlightRefresh = refreshFromRemote().finally(() => { inFlightRefresh = null; }); + } + try { + const index = await inFlightRefresh; + return { status: 'ready', index }; + } catch (err) { + return { status: 'error', error: err instanceof Error ? err.message : 'Failed to load skills.' }; + } + } catch (err) { + return { status: 'error', error: err instanceof Error ? err.message : 'Skill loader failed.' }; + } +} + +export async function readSkillContent(id: string): Promise<{ ok: true; content: string; entry: SkillEntry } | { ok: false; error: string }> { + const status = await ensureLoaded(); + if (status.status === 'error' || status.status === 'empty') { + return { ok: false, error: status.status === 'error' ? status.error : 'No skills cached yet.' }; + } + const entry = status.index.entries.find((e) => e.id === id); + if (!entry) return { ok: false, error: `Skill '${id}' not found.` }; + try { + const content = await fs.readFile(entry.localPath, 'utf8'); + return { ok: true, content, entry }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : 'Failed to read skill file.' }; + } +} diff --git a/apps/x/packages/core/src/application/browser-skills/matcher.ts b/apps/x/packages/core/src/application/browser-skills/matcher.ts new file mode 100644 index 00000000..a4aabde8 --- /dev/null +++ b/apps/x/packages/core/src/application/browser-skills/matcher.ts @@ -0,0 +1,56 @@ +import type { SkillEntry, SkillsIndex } from './loader.js'; + +/** + * Map browser-harness `domain-skills//` folder names to hostname tokens we + * match against the current tab's URL. + * + * Heuristic: for each site folder we generate candidate hostnames like + * "booking-com" -> ["booking-com", "bookingcom", "booking.com"] + * "github" -> ["github", "github.com"] + * "dev-to" -> ["dev-to", "devto", "dev.to"] + * Then we check whether any candidate is a substring of the tab hostname. + */ +function siteCandidates(site: string): string[] { + const candidates = new Set(); + candidates.add(site); + candidates.add(site.replace(/-/g, '')); + candidates.add(site.replace(/-/g, '.')); + if (site.endsWith('-com')) { + candidates.add(`${site.slice(0, -4)}.com`); + } + if (site.endsWith('-org')) { + candidates.add(`${site.slice(0, -4)}.org`); + } + if (site.endsWith('-io')) { + candidates.add(`${site.slice(0, -3)}.io`); + } + return Array.from(candidates); +} + +function extractHostname(url: string): string | null { + try { + return new URL(url).hostname.toLowerCase(); + } catch { + return null; + } +} + +export function matchSkillsForUrl(index: SkillsIndex, url: string, limit = 5): SkillEntry[] { + const hostname = extractHostname(url); + if (!hostname) return []; + + const bySite = new Map(); + for (const entry of index.entries) { + if (!bySite.has(entry.site)) bySite.set(entry.site, []); + bySite.get(entry.site)!.push(entry); + } + + const matched: SkillEntry[] = []; + for (const [site, entries] of bySite) { + const candidates = siteCandidates(site); + const hit = candidates.some((c) => hostname === c || hostname.endsWith(`.${c}`) || hostname.includes(c)); + if (hit) matched.push(...entries); + } + + return matched.slice(0, limit); +} diff --git a/apps/x/packages/core/src/application/lib/builtin-tools.ts b/apps/x/packages/core/src/application/lib/builtin-tools.ts index 65b398a1..7dd06dd2 100644 --- a/apps/x/packages/core/src/application/lib/builtin-tools.ts +++ b/apps/x/packages/core/src/application/lib/builtin-tools.ts @@ -18,6 +18,7 @@ import { composioAccountsRepo } from "../../composio/repo.js"; import { executeAction as executeComposioAction, isConfigured as isComposioConfigured, searchTools as searchComposioTools } from "../../composio/client.js"; import { CURATED_TOOLKITS, CURATED_TOOLKIT_SLUGS } from "@x/shared/dist/composio.js"; import { BrowserControlInputSchema, type BrowserControlInput } from "@x/shared/dist/browser-control.js"; +import { ensureLoaded as ensureBrowserSkillsLoaded, readSkillContent as readBrowserSkillContent, refreshFromRemote as refreshBrowserSkills } from "../browser-skills/index.js"; import type { ToolContext } from "./exec-tool.js"; import { generateText } from "ai"; import { createProvider } from "../../models/models.js"; @@ -1007,6 +1008,71 @@ export const BuiltinTools: z.infer = { }, }, + // ============================================================================ + // Browser Skills (browser-use/browser-harness domain-skills cache) + // ============================================================================ + + 'load-browser-skill': { + description: 'Load a site-specific browser skill (from the browser-use/browser-harness domain-skills library) by id. Returns the full markdown content with selectors, gotchas, and recipes for the target site. Call this after browser-control responses surface a matching skill in suggestedSkills. Pass action="list" to see all available skills. Skills are fetched on first use and cached locally; pass action="refresh" to force an update from upstream.', + inputSchema: z.object({ + action: z.enum(['load', 'list', 'refresh']).optional().describe('load: fetch a skill by id (default). list: list all cached skills. refresh: re-fetch the library from upstream.'), + id: z.string().optional().describe('Skill id (e.g., "github/repo-actions") — required for load.'), + site: z.string().optional().describe('Filter list results to a single site (e.g., "github").'), + }), + execute: async (input: { action?: 'load' | 'list' | 'refresh'; id?: string; site?: string }) => { + const action = input.action ?? 'load'; + try { + if (action === 'refresh') { + const index = await refreshBrowserSkills(); + return { + success: true, + message: `Refreshed ${index.entries.length} skill${index.entries.length === 1 ? '' : 's'} from upstream.`, + count: index.entries.length, + treeSha: index.treeSha, + }; + } + + if (action === 'list') { + const status = await ensureBrowserSkillsLoaded(); + if (status.status === 'error') { + return { success: false, error: status.error }; + } + if (status.status === 'empty') { + return { success: false, error: 'No browser skills cached yet.' }; + } + const entries = status.index.entries + .filter((e) => !input.site || e.site === input.site) + .map((e) => ({ id: e.id, title: e.title, site: e.site })); + return { + success: true, + count: entries.length, + skills: entries, + cacheAgeMs: Date.now() - status.index.fetchedAt, + refreshing: status.status === 'stale' ? status.refreshing : false, + }; + } + + if (!input.id) { + return { success: false, error: 'id is required for load.' }; + } + const result = await readBrowserSkillContent(input.id); + if (!result.ok) { + return { success: false, error: result.error }; + } + return { + success: true, + id: result.entry.id, + title: result.entry.title, + site: result.entry.site, + path: result.entry.path, + content: result.content, + }; + } catch (err) { + return { success: false, error: err instanceof Error ? err.message : 'Failed to load browser skill.' }; + } + }, + }, + // ============================================================================ // Browser Control // ============================================================================ diff --git a/apps/x/packages/shared/src/browser-control.ts b/apps/x/packages/shared/src/browser-control.ts index e1418a5e..e4eb112d 100644 --- a/apps/x/packages/shared/src/browser-control.ts +++ b/apps/x/packages/shared/src/browser-control.ts @@ -116,6 +116,12 @@ export const BrowserControlInputSchema = z.object({ } }); +export const SuggestedBrowserSkillSchema = z.object({ + id: z.string(), + title: z.string(), + path: z.string(), +}); + export const BrowserControlResultSchema = z.object({ success: z.boolean(), action: BrowserControlActionSchema, @@ -123,6 +129,7 @@ export const BrowserControlResultSchema = z.object({ error: z.string().optional(), browser: BrowserStateSchema, page: BrowserPageSnapshotSchema.optional(), + suggestedSkills: z.array(SuggestedBrowserSkillSchema).optional(), }); export type BrowserTabState = z.infer; @@ -132,3 +139,4 @@ export type BrowserPageSnapshot = z.infer; export type BrowserControlAction = z.infer; export type BrowserControlInput = z.infer; export type BrowserControlResult = z.infer; +export type SuggestedBrowserSkill = z.infer; From 5e47bd430942f736a564d26e8eb0b3f55d2ab0e4 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Wed, 6 May 2026 13:02:01 +0530 Subject: [PATCH 12/22] fix shell path issue on mac --- apps/x/apps/main/src/main.ts | 4 +++- .../core/src/application/assistant/runtime-context.ts | 10 +++++++++- .../core/src/application/lib/command-executor.ts | 10 ++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/apps/x/apps/main/src/main.ts b/apps/x/apps/main/src/main.ts index c3618000..b7d0a491 100644 --- a/apps/x/apps/main/src/main.ts +++ b/apps/x/apps/main/src/main.ts @@ -112,7 +112,9 @@ function initializeExecutionEnvironment(): void { ).trim(); const env = JSON.parse(stdout) as Record; - process.env = { ...env, ...process.env }; + // Let the user's shell environment win for overlapping keys like PATH. + // Finder/launched GUI apps on macOS often start with a stripped PATH. + process.env = { ...process.env, ...env }; } catch (error) { console.error('Failed to load shell environment', error); } diff --git a/apps/x/packages/core/src/application/assistant/runtime-context.ts b/apps/x/packages/core/src/application/assistant/runtime-context.ts index f1011c2c..a9baffc2 100644 --- a/apps/x/packages/core/src/application/assistant/runtime-context.ts +++ b/apps/x/packages/core/src/application/assistant/runtime-context.ts @@ -9,7 +9,15 @@ export interface RuntimeContext { } export function getExecutionShell(platform: NodeJS.Platform = process.platform): string { - return platform === 'win32' ? (process.env.ComSpec || 'cmd.exe') : '/bin/sh'; + if (platform === 'win32') { + return process.env.ComSpec || 'cmd.exe'; + } + + if (process.env.SHELL) { + return process.env.SHELL; + } + + return platform === 'darwin' ? '/bin/zsh' : '/bin/sh'; } export function getRuntimeContext(platform: NodeJS.Platform = process.platform): RuntimeContext { diff --git a/apps/x/packages/core/src/application/lib/command-executor.ts b/apps/x/packages/core/src/application/lib/command-executor.ts index 611bde45..11b15d90 100644 --- a/apps/x/packages/core/src/application/lib/command-executor.ts +++ b/apps/x/packages/core/src/application/lib/command-executor.ts @@ -8,7 +8,6 @@ const execPromise = promisify(exec); const COMMAND_SPLIT_REGEX = /(?:\|\||&&|;|\||\n|`|\$\(|\(|\))/; const ENV_ASSIGNMENT_REGEX = /^[A-Za-z_][A-Za-z0-9_]*=.*/; const WRAPPER_COMMANDS = new Set(['sudo', 'env', 'time', 'command']); -const EXECUTION_SHELL = getExecutionShell(); function sanitizeToken(token: string): string { return token.trim().replace(/^['"()]+|['"()]+$/g, ''); @@ -84,11 +83,12 @@ export async function executeCommand( } ): Promise { try { + const shell = getExecutionShell(); const { stdout, stderr } = await execPromise(command, { cwd: options?.cwd, timeout: options?.timeout, maxBuffer: options?.maxBuffer || 1024 * 1024, // default 1MB - shell: EXECUTION_SHELL, + shell, }); return { @@ -161,8 +161,9 @@ export function executeCommandAbortable( }; } + const shell = getExecutionShell(); const proc = spawn(command, [], { - shell: EXECUTION_SHELL, + shell, cwd: options?.cwd, detached: process.platform !== 'win32', // Create process group on Unix stdio: ['ignore', 'pipe', 'pipe'], @@ -272,11 +273,12 @@ export function executeCommandSync( } ): CommandResult { try { + const shell = getExecutionShell(); const stdout = execSync(command, { cwd: options?.cwd, timeout: options?.timeout, encoding: 'utf-8', - shell: EXECUTION_SHELL, + shell, }); return { From f26d57e8eb15af350f65c7bc916df82ceb4167b7 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Wed, 6 May 2026 13:10:20 +0530 Subject: [PATCH 13/22] fix sync resume modal copy --- .../composio-google-migration-modal.tsx | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/apps/x/apps/renderer/src/components/composio-google-migration-modal.tsx b/apps/x/apps/renderer/src/components/composio-google-migration-modal.tsx index 8afea839..97ef9321 100644 --- a/apps/x/apps/renderer/src/components/composio-google-migration-modal.tsx +++ b/apps/x/apps/renderer/src/components/composio-google-migration-modal.tsx @@ -37,21 +37,14 @@ export function ComposioGoogleMigrationModal({
- Reconnect Google to keep syncing + Reconnect Google to resume syncing

- Rowboat used to sync your Gmail and Calendar through{" "} - Composio, a - third-party connector. We've now built a direct connection to - Google — it's faster, more private, and doesn't rely on a - middleman. -

-

- We've disconnected the Composio connection. Reconnect Google - directly to resume syncing — your existing emails and calendar - events stay exactly where they are. + Knowledge graph syncing for Gmail and Calendar now uses a + direct Google connection. Reconnect to resume. Your existing + emails and events stay where they are.

From 0bb58e55ac6f8f91927298f96e0719579a89bed1 Mon Sep 17 00:00:00 2001 From: gagan Date: Wed, 6 May 2026 14:34:53 +0530 Subject: [PATCH 14/22] feat: minimal Today.md UI polish - no emoji headings, better track chip (#528) * feat: remove emoji headings and polish track block chip styling - Strip emojis from Today.md section headings (new + existing files via migration) - Track chip: full-width card style matching email blocks, colored icons per track type - Larger, taller chip with muted gray background for light/dark mode * feat: increase track chip icon and text size * feat: make track block icons configurable via yaml --- .../renderer/src/extensions/track-block.tsx | 36 ++++++++++++++----- apps/x/apps/renderer/src/styles/editor.css | 34 ++++++++++-------- .../core/src/knowledge/ensure_daily_note.ts | 36 ++++++++++++++++--- apps/x/packages/shared/src/track-block.ts | 1 + 4 files changed, 79 insertions(+), 28 deletions(-) diff --git a/apps/x/apps/renderer/src/extensions/track-block.tsx b/apps/x/apps/renderer/src/extensions/track-block.tsx index a87decc8..4f2a1f0a 100644 --- a/apps/x/apps/renderer/src/extensions/track-block.tsx +++ b/apps/x/apps/renderer/src/extensions/track-block.tsx @@ -1,12 +1,31 @@ import { z } from 'zod' -import { useMemo } from 'react' +import { useMemo, type ComponentType } from 'react' import { mergeAttributes, Node } from '@tiptap/react' import { ReactNodeViewRenderer, NodeViewWrapper } from '@tiptap/react' -import { Radio, Loader2 } from 'lucide-react' +import { Radio, Loader2, type LucideProps } from 'lucide-react' +import * as LucideIcons from 'lucide-react' import { parse as parseYaml } from 'yaml' import { TrackBlockSchema } from '@x/shared/dist/track-block.js' import { useTrackStatus } from '@/hooks/use-track-status' +function resolveIcon(iconName: string): ComponentType | null { + const key = iconName + .split('-') + .map(w => w.charAt(0).toUpperCase() + w.slice(1)) + .join('') + const component = (LucideIcons as Record)[key] + if (component != null) return component as ComponentType + return null +} + +function TrackIcon({ icon, size }: { icon?: string; size: number }) { + if (icon) { + const Icon = resolveIcon(icon) + if (Icon) return + } + return +} + function truncate(text: string, maxLen: number): string { const clean = text.replace(/\s+/g, ' ').trim() if (clean.length <= maxLen) return clean @@ -87,6 +106,7 @@ function TrackBlockView({ node, deleteNode, extension }: { data-type="track-block" data-trigger={triggerType} data-active={active ? 'true' : 'false'} + data-trackid={trackId} > + )} + +
+ )} + + {hasDraft && ( +
+
+ Reply + {config.from && {config.from}} +
+