From b52a287c374bcea0cc96c5760ec00d603817abcc Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Sat, 21 Mar 2026 19:43:40 +0530 Subject: [PATCH] user.md --- .../src/application/assistant/instructions.ts | 30 ++ .../core/src/application/lib/builtin-tools.ts | 22 + .../core/src/knowledge/agent_notes.ts | 375 +++++++++++++----- .../core/src/knowledge/agent_notes_state.ts | 7 +- 4 files changed, 328 insertions(+), 106 deletions(-) diff --git a/apps/x/packages/core/src/application/assistant/instructions.ts b/apps/x/packages/core/src/application/assistant/instructions.ts index 978e6c40..a6eff725 100644 --- a/apps/x/packages/core/src/application/assistant/instructions.ts +++ b/apps/x/packages/core/src/application/assistant/instructions.ts @@ -37,6 +37,35 @@ Rowboat is an agentic assistant for everyday work - emails, meetings, projects, **Slack:** When users ask about Slack messages, want to send messages to teammates, check channel conversations, or find someone on Slack, load the \`slack\` skill. You can send messages, view channel history, search conversations, and find users. Always show message drafts to the user before sending. +## Learning About the User (save-to-memory) + +Use the \`save-to-memory\` tool to note things worth remembering about the user. This builds a persistent profile that helps you serve them better over time. Call it proactively — don't ask permission. + +**When to save:** +- User states a preference: "I prefer bullet points" → save as preference +- User corrects your style: "too formal, keep it casual" → save as style +- You learn about their relationships: "Monica is my co-founder" → save as people +- You notice workflow patterns: "no meetings before 11am" → save as routine +- User gives explicit instructions: "never use em-dashes" → save as preference + +**Capture context, not blanket rules:** +- BAD: "User prefers casual tone" — this loses important context +- GOOD: "User prefers casual tone with internal team (Ramnique, Monica) but formal/polished with investors (Brad, Dalton)" +- BAD: "User likes short emails" — too vague +- GOOD: "User sends very terse 1-2 line emails to co-founder Ramnique, but writes structured 2-3 paragraph emails to investors with proper greetings" +- Always note WHO or WHAT CONTEXT a preference applies to. Most preferences are situational, not universal. + +**When NOT to save:** +- Ephemeral task details ("draft an email about X") +- Things already in the knowledge graph +- Information you can derive from reading their notes + +**Categories:** +- \`preference\` — rules about how they want things done +- \`style\` — writing and communication patterns (always note the context: who, what type of communication) +- \`people\` — relationship context and per-person tone +- \`routine\` — scheduling, workflow, recurring patterns + ## Memory That Compounds Unlike other AI assistants that start cold every session, you have access to a live knowledge graph that updates itself from Gmail, calendar, and meeting notes (Google Meet, Granola, Fireflies). This isn't just summaries - it's structured extraction of decisions, commitments, open questions, and context, routed to long-lived notes for each person, project, and topic. @@ -187,6 +216,7 @@ ${runtimeContextPrompt} - \`slack-checkConnection\`, \`slack-listAvailableTools\`, \`slack-executeAction\` - Slack integration (requires Slack to be connected via Composio). Use \`slack-listAvailableTools\` first to discover available tool slugs, then \`slack-executeAction\` to execute them. - \`web-search\` and \`research-search\` - Web and research search tools (available when configured). **You MUST load the \`web-search\` skill before using either of these tools.** It tells you which tool to pick and how many searches to do. - \`app-navigation\` - Control the app UI: open notes, switch views, filter/search the knowledge base, manage saved views. **Load the \`app-navigation\` skill before using this tool.** +- \`save-to-memory\` - Save observations about the user to the agent memory system. Use this proactively during conversations. **Prefer these tools whenever possible** — they work instantly with zero friction. For file operations inside \`~/.rowboat/\`, always use these instead of \`executeCommand\`. diff --git a/apps/x/packages/core/src/application/lib/builtin-tools.ts b/apps/x/packages/core/src/application/lib/builtin-tools.ts index 92742668..2fd0b4a5 100644 --- a/apps/x/packages/core/src/application/lib/builtin-tools.ts +++ b/apps/x/packages/core/src/application/lib/builtin-tools.ts @@ -1260,4 +1260,26 @@ export const BuiltinTools: z.infer = { } }, }, + 'save-to-memory': { + description: "Save a note about user preferences, style, people, or routines to the agent memory inbox. Use this when you observe something worth remembering about the user — their preferences, communication patterns, relationship context, scheduling habits, or explicit instructions about how they want things done.", + inputSchema: z.object({ + note: z.string().describe("The observation or preference to remember. Be specific and concise."), + category: z.enum(['preference', 'style', 'people', 'routine']).describe("Category: 'preference' for rules/preferences, 'style' for writing/communication patterns, 'people' for relationship context, 'routine' for scheduling/workflow patterns"), + }), + execute: async ({ note, category }: { note: string; category: string }) => { + const inboxPath = path.join(WorkDir, 'knowledge', 'agent-notes', 'inbox.md'); + const dir = path.dirname(inboxPath); + await fs.mkdir(dir, { recursive: true }); + + const timestamp = new Date().toISOString(); + const entry = `\n- [${timestamp}] [${category}] ${note}\n`; + + await fs.appendFile(inboxPath, entry, 'utf-8'); + + return { + success: true, + message: `Saved to memory inbox: [${category}] ${note}`, + }; + }, + }, }; diff --git a/apps/x/packages/core/src/knowledge/agent_notes.ts b/apps/x/packages/core/src/knowledge/agent_notes.ts index 2d783537..2733e776 100644 --- a/apps/x/packages/core/src/knowledge/agent_notes.ts +++ b/apps/x/packages/core/src/knowledge/agent_notes.ts @@ -17,12 +17,14 @@ import { type AgentNotesState, } from './agent_notes_state.js'; -const SYNC_INTERVAL_MS = 30 * 60 * 1000; // 30 minutes +const SYNC_INTERVAL_MS = 10 * 1000; // 10 seconds (for testing) const EMAIL_BATCH_SIZE = 5; +const RUNS_BATCH_SIZE = 5; const GMAIL_SYNC_DIR = path.join(WorkDir, 'gmail_sync'); const RUNS_DIR = path.join(WorkDir, 'runs'); const AGENT_NOTES_DIR = path.join(WorkDir, 'knowledge', 'agent-notes'); const STYLE_DIR = path.join(AGENT_NOTES_DIR, 'style'); +const INBOX_FILE = path.join(AGENT_NOTES_DIR, 'inbox.md'); const NOTE_FILES = { preferences: path.join(AGENT_NOTES_DIR, 'preferences.md'), @@ -32,6 +34,14 @@ const NOTE_FILES = { documentsStyle: path.join(STYLE_DIR, 'documents.md'), people: path.join(AGENT_NOTES_DIR, 'people.md'), routines: path.join(AGENT_NOTES_DIR, 'routines.md'), + user: path.join(AGENT_NOTES_DIR, 'user.md'), +}; + +const CATEGORY_TO_FILE: Record = { + preference: [NOTE_FILES.preferences], + style: [NOTE_FILES.writingStyle], + people: [NOTE_FILES.people], + routine: [NOTE_FILES.routines], }; // --- LLM helpers --- @@ -103,7 +113,6 @@ function findUserSentEmails( try { const content = fs.readFileSync(fullPath, 'utf-8'); - // Check if any From header contains the user's email const fromLines = content.match(/^### From:.*$/gm); if (fromLines?.some(line => line.toLowerCase().includes(userEmailLower))) { results.push({ path: fullPath, mtime: stat.mtimeMs }); @@ -117,14 +126,12 @@ function findUserSentEmails( traverse(GMAIL_SYNC_DIR); - // Sort by mtime descending (newest first), return up to limit results.sort((a, b) => b.mtime - a.mtime); return results.slice(0, limit).map(r => r.path); } function extractUserPartsFromEmail(content: string, userEmail: string): string | null { const userEmailLower = userEmail.toLowerCase(); - // Split by message separator const sections = content.split(/^---$/m); const userSections: string[] = []; @@ -138,75 +145,41 @@ function extractUserPartsFromEmail(content: string, userEmail: string): string | return userSections.length > 0 ? userSections.join('\n\n---\n\n') : null; } -// --- Run scanning --- +// --- Inbox processing --- -function findNewCopilotRuns(state: AgentNotesState): string[] { - if (!fs.existsSync(RUNS_DIR)) { +interface InboxEntry { + timestamp: string; + category: string; + note: string; +} + +function readInbox(): InboxEntry[] { + const content = readNoteFile(INBOX_FILE); + if (!content.trim()) { return []; } - const results: string[] = []; - const files = fs.readdirSync(RUNS_DIR).filter(f => f.endsWith('.jsonl')); + const entries: InboxEntry[] = []; + const lines = content.split('\n').filter(l => l.trim()); - for (const file of files) { - if (state.processedRuns[file]) { - continue; - } - - try { - const fullPath = path.join(RUNS_DIR, file); - const fd = fs.openSync(fullPath, 'r'); - const buf = Buffer.alloc(512); - const bytesRead = fs.readSync(fd, buf, 0, 512, 0); - fs.closeSync(fd); - - const firstLine = buf.subarray(0, bytesRead).toString('utf-8').split('\n')[0]; - const event = JSON.parse(firstLine); - if (event.agentName === 'copilot') { - results.push(file); - } - } catch { - continue; + for (const line of lines) { + const match = line.match(/^- \[([^\]]+)\] \[([^\]]+)\] (.+)$/); + if (match) { + entries.push({ + timestamp: match[1], + category: match[2], + note: match[3], + }); } } - // Sort chronologically (filenames are timestamps) - results.sort(); - return results; + return entries; } -function extractUserMessages(runFilePath: string): string[] { - const messages: string[] = []; - try { - const content = fs.readFileSync(runFilePath, 'utf-8'); - const lines = content.split('\n').filter(l => l.trim()); - - for (const line of lines) { - try { - const event = JSON.parse(line); - if (event.type === 'message' && event.message?.role === 'user') { - const msgContent = event.message.content; - if (typeof msgContent === 'string' && msgContent.trim()) { - messages.push(msgContent.trim()); - } else if (Array.isArray(msgContent)) { - // Handle array content format (text parts) - const text = msgContent - .filter((p: { type: string }) => p.type === 'text') - .map((p: { text: string }) => p.text) - .join('\n'); - if (text.trim()) { - messages.push(text.trim()); - } - } - } - } catch { - continue; - } - } - } catch { - // ignore +function clearInbox(): void { + if (fs.existsSync(INBOX_FILE)) { + fs.writeFileSync(INBOX_FILE, ''); } - return messages; } // --- Note file updates (single LLM call per file) --- @@ -251,7 +224,6 @@ async function updateEmailStyle( userName: string, userEmail: string, ): Promise { - // Build source content from user-sent email parts let sourceContent = `Emails written by ${userName}:\n\n`; for (const file of emailFiles) { const userParts = extractUserPartsFromEmail(file.content, userEmail); @@ -273,47 +245,208 @@ async function updateEmailStyle( ); } -// --- Copilot run processing --- +// --- Inbox processing --- -async function updateFromCopilotRuns(runFiles: string[]): Promise { - // Collect user messages from all new runs - let allUserMessages: string[] = []; - for (const runFile of runFiles) { - const msgs = extractUserMessages(path.join(RUNS_DIR, runFile)); - allUserMessages.push(...msgs); +async function processInbox(entries: InboxEntry[]): Promise { + if (entries.length === 0) { + return 0; } - if (allUserMessages.length === 0) { - return; + // Group entries by category + const grouped = new Map(); + for (const entry of entries) { + const category = entry.category; + if (!grouped.has(category)) { + grouped.set(category, []); + } + grouped.get(category)!.push(entry); } - // Cap to avoid massive prompts - if (allUserMessages.length > 20) { - allUserMessages = allUserMessages.slice(-20); + // Update each relevant note file + for (const [category, categoryEntries] of grouped) { + const targetFiles = CATEGORY_TO_FILE[category]; + if (!targetFiles) { + console.log(`[AgentNotes] Unknown category: ${category}, skipping`); + continue; + } + + const sourceContent = `Observations from conversations:\n\n${categoryEntries.map(e => `- ${e.note}`).join('\n')}`; + + for (const targetFile of targetFiles) { + const description = targetFile === NOTE_FILES.preferences + ? 'Hard rules and explicit preferences — always loaded for context' + : targetFile === NOTE_FILES.writingStyle + ? 'General voice and tone patterns across all writing' + : targetFile === NOTE_FILES.people + ? 'Per-person relationship context, tone preferences, and interaction notes' + : 'Scheduling patterns, workflow habits, recurring tasks'; + + await updateNoteFile(targetFile, description, sourceContent); + } } - const sourceContent = `User messages from recent AI assistant conversations:\n\n${allUserMessages.map((m, i) => `${i + 1}. ${m}`).join('\n\n')}`; + return entries.length; +} - // Update preferences - await updateNoteFile( - NOTE_FILES.preferences, - 'Hard rules and explicit preferences the user has stated — always loaded for context', - sourceContent, - ); +// --- Copilot run scanning --- - // Update people context - await updateNoteFile( - NOTE_FILES.people, - 'Per-person relationship context, tone preferences, and interaction notes', - sourceContent, - ); +function findNewCopilotRuns(state: AgentNotesState): string[] { + if (!fs.existsSync(RUNS_DIR)) { + return []; + } - // Update routines - await updateNoteFile( - NOTE_FILES.routines, - 'Scheduling patterns, workflow habits, recurring tasks', - sourceContent, - ); + const results: string[] = []; + const files = fs.readdirSync(RUNS_DIR).filter(f => f.endsWith('.jsonl')); + + for (const file of files) { + if (state.processedRuns[file]) { + continue; + } + + try { + const fullPath = path.join(RUNS_DIR, file); + const fd = fs.openSync(fullPath, 'r'); + const buf = Buffer.alloc(512); + const bytesRead = fs.readSync(fd, buf, 0, 512, 0); + fs.closeSync(fd); + + const firstLine = buf.subarray(0, bytesRead).toString('utf-8').split('\n')[0]; + const event = JSON.parse(firstLine); + if (event.agentName === 'copilot') { + results.push(file); + } + } catch { + continue; + } + } + + // Sort chronologically (filenames are timestamps), newest last + results.sort(); + return results; +} + +/** + * Extract only user and assistant text messages from a run file. + * Skips tool calls, tool results, system messages, and any non-text content. + */ +function extractConversationMessages(runFilePath: string): { role: string; text: string }[] { + const messages: { role: string; text: string }[] = []; + try { + const content = fs.readFileSync(runFilePath, 'utf-8'); + const lines = content.split('\n').filter(l => l.trim()); + + for (const line of lines) { + try { + const event = JSON.parse(line); + if (event.type !== 'message') continue; + + const msg = event.message; + if (!msg || (msg.role !== 'user' && msg.role !== 'assistant')) continue; + + let text = ''; + if (typeof msg.content === 'string') { + text = msg.content.trim(); + } else if (Array.isArray(msg.content)) { + // Only extract text parts, skip tool-call parts + text = msg.content + .filter((p: { type: string }) => p.type === 'text') + .map((p: { text: string }) => p.text) + .join('\n') + .trim(); + } + + if (text) { + messages.push({ role: msg.role, text }); + } + } catch { + continue; + } + } + } catch { + // ignore + } + return messages; +} + +/** + * Process copilot runs and append new facts to user.md. + * Each fact is a timestamped line. The LLM decides what's new vs already known. + */ +async function updateUserNotes(runFiles: string[]): Promise { + if (runFiles.length === 0) { + return 0; + } + + // Collect conversations from runs (limit to RUNS_BATCH_SIZE) + const runsToProcess = runFiles.slice(-RUNS_BATCH_SIZE); + let conversationText = ''; + + for (const runFile of runsToProcess) { + const messages = extractConversationMessages(path.join(RUNS_DIR, runFile)); + if (messages.length === 0) continue; + + conversationText += `\n--- Conversation ---\n`; + for (const msg of messages) { + conversationText += `${msg.role}: ${msg.text}\n\n`; + } + } + + if (!conversationText.trim()) { + return 0; + } + + const model = await getModel(); + const existing = readNoteFile(NOTE_FILES.user); + const timestamp = new Date().toISOString(); + + const system = `You analyze conversations between a user and their AI assistant to learn facts about the user. + +Your job: extract any new, non-trivial facts about the user that are worth remembering long-term. + +Examples of good facts: +- Working on Project X, an AI assistant product +- Team is 4 people, co-founder is Ramnique +- Preparing for Series A fundraise +- Based in Bangalore, India +- Prefers to work late evenings +- Has a meeting with Brad from Smash Capital next week + +Examples of things NOT to extract: +- Ephemeral task details ("user asked to draft an email") +- Facts the assistant already knows from tools/knowledge graph +- Obvious or trivial observations ("user uses a computer") + +Output format: Return ONLY new facts as a bullet list, one per line. Each line should be: +- [${timestamp}] The fact + +If there are no new facts worth noting, return exactly: NO_NEW_FACTS + +IMPORTANT: Check the existing user notes below. Do NOT repeat facts that are already captured there (even if worded differently).`; + + const prompt = `## Existing user notes: +${existing || '(none yet)'} + +## Recent conversations to analyze: +${conversationText} + +Extract new facts (or return NO_NEW_FACTS):`; + + const result = await generateText({ model, system, prompt }); + const text = stripCodeFences(result.text).trim(); + + if (text === 'NO_NEW_FACTS' || !text) { + return 0; + } + + // Append new facts to user.md + const header = existing ? '' : '# User\n\n'; + const newContent = existing + ? existing.trimEnd() + '\n' + text + '\n' + : header + text + '\n'; + fs.writeFileSync(NOTE_FILES.user, newContent); + + // Count lines added + return text.split('\n').filter(l => l.trim().startsWith('-')).length; } // --- Main processing --- @@ -336,7 +469,8 @@ async function processAgentNotes(): Promise { let hadError = false; let emailsProcessed = 0; - let runsProcessed = 0; + let inboxProcessed = 0; + let userFactsAdded = 0; // --- Email Style Learning --- try { @@ -351,7 +485,7 @@ async function processAgentNotes(): Promise { message: `Analyzing ${emailPaths.length} emails for style`, step: 'email_style', current: 1, - total: 2, + total: 3, }); const emailFiles = emailPaths.map(p => ({ @@ -380,7 +514,39 @@ async function processAgentNotes(): Promise { }); } - // --- Chat Run Learning --- + // --- Inbox Processing --- + try { + const entries = readInbox(); + if (entries.length > 0) { + console.log(`[AgentNotes] Found ${entries.length} inbox entries`); + await serviceLogger.log({ + type: 'progress', + service: run.service, + runId: run.runId, + level: 'info', + message: `Processing ${entries.length} inbox entries`, + step: 'inbox', + current: 2, + total: 3, + }); + + inboxProcessed = await processInbox(entries); + clearInbox(); + } + } catch (error) { + hadError = true; + console.error('[AgentNotes] Error processing inbox:', error); + await serviceLogger.log({ + type: 'error', + service: run.service, + runId: run.runId, + level: 'error', + message: 'Error processing inbox', + error: error instanceof Error ? error.message : String(error), + }); + } + + // --- Copilot Run Learning (user.md) --- try { const newRuns = findNewCopilotRuns(state); if (newRuns.length > 0) { @@ -390,23 +556,22 @@ async function processAgentNotes(): Promise { service: run.service, runId: run.runId, level: 'info', - message: `Analyzing ${newRuns.length} copilot runs`, - step: 'chat_runs', - current: 2, - total: 2, + message: `Analyzing ${newRuns.length} copilot runs for user facts`, + step: 'copilot_runs', + current: 3, + total: 3, }); - await updateFromCopilotRuns(newRuns); + userFactsAdded = await updateUserNotes(newRuns); for (const r of newRuns) { markRunProcessed(r, state); } saveAgentNotesState(state); - runsProcessed = newRuns.length; } } catch (error) { hadError = true; - console.error('[AgentNotes] Error processing runs:', error); + console.error('[AgentNotes] Error processing copilot runs:', error); await serviceLogger.log({ type: 'error', service: run.service, @@ -428,7 +593,7 @@ async function processAgentNotes(): Promise { message: 'Agent notes processing complete', durationMs: Date.now() - run.startedAt, outcome: hadError ? 'error' : 'ok', - summary: { emailsProcessed, runsProcessed }, + summary: { emailsProcessed, inboxProcessed, userFactsAdded }, }); } diff --git a/apps/x/packages/core/src/knowledge/agent_notes_state.ts b/apps/x/packages/core/src/knowledge/agent_notes_state.ts index 93d1bae1..d500e155 100644 --- a/apps/x/packages/core/src/knowledge/agent_notes_state.ts +++ b/apps/x/packages/core/src/knowledge/agent_notes_state.ts @@ -13,7 +13,12 @@ export interface AgentNotesState { export function loadAgentNotesState(): AgentNotesState { if (fs.existsSync(STATE_FILE)) { try { - return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); + const parsed = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); + // Handle migration from older state without processedRuns + if (!parsed.processedRuns) { + parsed.processedRuns = {}; + } + return parsed; } catch (error) { console.error('Error loading agent notes state:', error); }