diff --git a/apps/x/apps/main/src/ipc.ts b/apps/x/apps/main/src/ipc.ts index ec1a0aaa..a9de9572 100644 --- a/apps/x/apps/main/src/ipc.ts +++ b/apps/x/apps/main/src/ipc.ts @@ -455,7 +455,7 @@ export function setupIpcHandlers() { return runsCore.createRun(args); }, 'runs:createMessage': async (_event, args) => { - return { messageId: await runsCore.createMessage(args.runId, args.message, args.voiceInput, args.voiceOutput, args.searchEnabled) }; + return { messageId: await runsCore.createMessage(args.runId, args.message, args.voiceInput, args.voiceOutput, args.searchEnabled, args.middlePaneContext) }; }, 'runs:authorizePermission': async (_event, args) => { await runsCore.authorizePermission(args.runId, args.authorization); diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index eaf411e1..14cc1140 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -2070,6 +2070,34 @@ function App() { return cleanup }, [handleRunEvent]) + type MiddlePaneContextPayload = + | { kind: 'note'; path: string; content: string } + | { kind: 'browser'; url: string; title: string } + const buildMiddlePaneContext = async (): Promise => { + // Nothing visible in the middle pane when the right pane is maximized. + if (isRightPaneMaximized) return undefined + + // Browser is an overlay on top of any note — when it's open, it's what the user is looking at. + if (isBrowserOpen) { + try { + const state = await window.ipc.invoke('browser:getState', null) + const activeTab = state.tabs.find((t) => t.id === state.activeTabId) + if (activeTab) { + return { kind: 'browser', url: activeTab.url, title: activeTab.title } + } + } catch { + // fall through to no-context if browser state is unavailable + } + return undefined + } + + // Note case: only markdown files are meaningfully readable as context. + const path = selectedPathRef.current + if (!path || !path.endsWith('.md')) return undefined + const content = editorContentRef.current ?? '' + return { kind: 'note', path, content } + } + const handlePromptSubmit = async ( message: PromptInputMessage, mentions?: FileMention[], @@ -2173,12 +2201,14 @@ function App() { // Shared IPC payload types can lag until package rebuilds; runtime validation still enforces schema. const attachmentPayload = contentParts as unknown as string + const middlePaneContext = await buildMiddlePaneContext() await window.ipc.invoke('runs:createMessage', { runId: currentRunId, message: attachmentPayload, voiceInput: pendingVoiceInputRef.current || undefined, voiceOutput: ttsEnabledRef.current ? ttsModeRef.current : undefined, searchEnabled: searchEnabled || undefined, + middlePaneContext, }) analytics.chatMessageSent({ voiceInput: pendingVoiceInputRef.current || undefined, @@ -2186,12 +2216,14 @@ function App() { searchEnabled: searchEnabled || undefined, }) } else { + const middlePaneContext = await buildMiddlePaneContext() await window.ipc.invoke('runs:createMessage', { runId: currentRunId, message: userMessage, voiceInput: pendingVoiceInputRef.current || undefined, voiceOutput: ttsEnabledRef.current ? ttsModeRef.current : undefined, searchEnabled: searchEnabled || undefined, + middlePaneContext, }) analytics.chatMessageSent({ voiceInput: pendingVoiceInputRef.current || undefined, diff --git a/apps/x/packages/core/src/agents/runtime.ts b/apps/x/packages/core/src/agents/runtime.ts index 507bd0c7..f978449b 100644 --- a/apps/x/packages/core/src/agents/runtime.ts +++ b/apps/x/packages/core/src/agents/runtime.ts @@ -878,6 +878,10 @@ export async function* streamAgent({ let voiceInput = false; let voiceOutput: 'summary' | 'full' | null = null; let searchEnabled = false; + let middlePaneContext: + | { kind: 'note'; path: string; content: string } + | { kind: 'browser'; url: string; title: string } + | null = null; while (true) { // Check abort at the top of each iteration signal.throwIfAborted(); @@ -1005,6 +1009,9 @@ export async function* streamAgent({ if (msg.voiceOutput) { voiceOutput = msg.voiceOutput; } + // Middle pane is NOT sticky — it should reflect the state at the moment of the + // latest user message. If the user closed the pane between messages, clear it. + middlePaneContext = msg.middlePaneContext ?? null; loopLogger.log('dequeued user message', msg.messageId); yield* processEvent({ runId, @@ -1051,6 +1058,19 @@ export async function* streamAgent({ if (agentNotesContext) { instructionsWithDateTime += `\n\n${agentNotesContext}`; } + // Always inject a Middle Pane section so the LLM has a clear, up-to-date signal + // that supersedes any earlier middle-pane mention in the conversation history. + const middlePaneHeader = `\n\n# Middle Pane (Current State)\nThis section reflects what the user has open in the middle pane RIGHT NOW, at the time of their latest message. **This is authoritative and overrides any earlier mention of a note or web page in this conversation** — if the conversation history references a different note or browser page, the user has since closed or navigated away from it. Do not treat earlier context as current.\n\n`; + if (!middlePaneContext) { + loopLogger.log('injecting middle pane context (empty)'); + instructionsWithDateTime += `${middlePaneHeader}**Nothing relevant is open in the middle pane right now.** The user is not looking at any note or web page. If earlier in this conversation you referenced a note or browser page as "what the user is viewing", that is no longer accurate — do not refer to it as currently open. Answer the user's latest message on its own merits.`; + } else if (middlePaneContext.kind === 'note') { + loopLogger.log('injecting middle pane context (note)', middlePaneContext.path); + instructionsWithDateTime += `${middlePaneHeader}The user has a note open. Its path and full content are provided below so you can reference it when relevant.\n\n**How to use this context:**\n- The user may or may not be talking about this note. Do NOT assume every message is about it.\n- Only reference or act on this note when the user's message clearly relates to it (e.g. "this note", "what I'm looking at", "here", "above", "below", or questions whose subject is plainly this note's content).\n- For unrelated questions (general chat, questions about other notes, tasks, emails, calendar, etc.), ignore this context entirely and answer normally.\n- Do not mention that you can see this note unless it is relevant to the answer.\n\n## Open note path\n${middlePaneContext.path}\n\n## Open note content\n\`\`\`\n${middlePaneContext.content}\n\`\`\``; + } else if (middlePaneContext.kind === 'browser') { + loopLogger.log('injecting middle pane context (browser)', middlePaneContext.url); + instructionsWithDateTime += `${middlePaneHeader}The user has the embedded browser open and is viewing a web page. Only the URL and page title are shown below — the page content itself is NOT included here. If you need the page content to answer, use the browser tools available to you to read the page.\n\n**How to use this context:**\n- The user may or may not be talking about this page. Do NOT assume every message is about it.\n- Only reference or act on this page when the user's message clearly relates to it (e.g. "this page", "this article", "what I'm looking at", "this site", "summarize this").\n- For unrelated questions (general chat, questions about other notes, tasks, emails, calendar, etc.), ignore this context entirely and answer normally.\n- Do not mention that you can see the browser unless it is relevant to the answer.\n\n## Current page\nURL: ${middlePaneContext.url}\nTitle: ${middlePaneContext.title}`; + } } if (voiceInput) { loopLogger.log('voice input enabled, injecting voice input prompt'); diff --git a/apps/x/packages/core/src/application/lib/message-queue.ts b/apps/x/packages/core/src/application/lib/message-queue.ts index d60b51b1..b3d2affa 100644 --- a/apps/x/packages/core/src/application/lib/message-queue.ts +++ b/apps/x/packages/core/src/application/lib/message-queue.ts @@ -4,6 +4,9 @@ import z from "zod"; export type UserMessageContentType = z.infer; export type VoiceOutputMode = 'summary' | 'full'; +export type MiddlePaneContext = + | { kind: 'note'; path: string; content: string } + | { kind: 'browser'; url: string; title: string }; type EnqueuedMessage = { messageId: string; @@ -11,10 +14,11 @@ type EnqueuedMessage = { voiceInput?: boolean; voiceOutput?: VoiceOutputMode; searchEnabled?: boolean; + middlePaneContext?: MiddlePaneContext; }; export interface IMessageQueue { - enqueue(runId: string, message: UserMessageContentType, voiceInput?: boolean, voiceOutput?: VoiceOutputMode, searchEnabled?: boolean): Promise; + enqueue(runId: string, message: UserMessageContentType, voiceInput?: boolean, voiceOutput?: VoiceOutputMode, searchEnabled?: boolean, middlePaneContext?: MiddlePaneContext): Promise; dequeue(runId: string): Promise; } @@ -30,7 +34,7 @@ export class InMemoryMessageQueue implements IMessageQueue { this.idGenerator = idGenerator; } - async enqueue(runId: string, message: UserMessageContentType, voiceInput?: boolean, voiceOutput?: VoiceOutputMode, searchEnabled?: boolean): Promise { + async enqueue(runId: string, message: UserMessageContentType, voiceInput?: boolean, voiceOutput?: VoiceOutputMode, searchEnabled?: boolean, middlePaneContext?: MiddlePaneContext): Promise { if (!this.store[runId]) { this.store[runId] = []; } @@ -41,6 +45,7 @@ export class InMemoryMessageQueue implements IMessageQueue { voiceInput, voiceOutput, searchEnabled, + middlePaneContext, }); return id; } diff --git a/apps/x/packages/core/src/runs/runs.ts b/apps/x/packages/core/src/runs/runs.ts index 30c9bd67..8ea4688b 100644 --- a/apps/x/packages/core/src/runs/runs.ts +++ b/apps/x/packages/core/src/runs/runs.ts @@ -1,6 +1,6 @@ import z from "zod"; import container from "../di/container.js"; -import { IMessageQueue, UserMessageContentType, VoiceOutputMode } from "../application/lib/message-queue.js"; +import { IMessageQueue, UserMessageContentType, VoiceOutputMode, MiddlePaneContext } from "../application/lib/message-queue.js"; import { AskHumanResponseEvent, ToolPermissionRequestEvent, ToolPermissionResponseEvent, CreateRunOptions, Run, ListRunsResponse, ToolPermissionAuthorizePayload, AskHumanResponsePayload } from "@x/shared/dist/runs.js"; import { IRunsRepo } from "./repo.js"; import { IAgentRuntime } from "../agents/runtime.js"; @@ -19,9 +19,9 @@ export async function createRun(opts: z.infer): Promise return run; } -export async function createMessage(runId: string, message: UserMessageContentType, voiceInput?: boolean, voiceOutput?: VoiceOutputMode, searchEnabled?: boolean): Promise { +export async function createMessage(runId: string, message: UserMessageContentType, voiceInput?: boolean, voiceOutput?: VoiceOutputMode, searchEnabled?: boolean, middlePaneContext?: MiddlePaneContext): Promise { const queue = container.resolve('messageQueue'); - const id = await queue.enqueue(runId, message, voiceInput, voiceOutput, searchEnabled); + const id = await queue.enqueue(runId, message, voiceInput, voiceOutput, searchEnabled, middlePaneContext); const runtime = container.resolve('agentRuntime'); runtime.trigger(runId); return id; diff --git a/apps/x/packages/shared/src/ipc.ts b/apps/x/packages/shared/src/ipc.ts index f0de64af..cc98f4f1 100644 --- a/apps/x/packages/shared/src/ipc.ts +++ b/apps/x/packages/shared/src/ipc.ts @@ -137,6 +137,18 @@ const ipcSchemas = { voiceInput: z.boolean().optional(), voiceOutput: z.enum(['summary', 'full']).optional(), searchEnabled: z.boolean().optional(), + middlePaneContext: z.discriminatedUnion('kind', [ + z.object({ + kind: z.literal('note'), + path: z.string(), + content: z.string(), + }), + z.object({ + kind: z.literal('browser'), + url: z.string(), + title: z.string(), + }), + ]).optional(), }), res: z.object({ messageId: z.string(),