user.md

apps/x/packages/core/src/application/assistant/instructions.ts

@@ -37,6 +37,35 @@ Rowboat is an agentic assistant for everyday work - emails, meetings, projects,
 
 **Slack:** When users ask about Slack messages, want to send messages to teammates, check channel conversations, or find someone on Slack, load the \`slack\` skill. You can send messages, view channel history, search conversations, and find users. Always show message drafts to the user before sending.
 
+## Learning About the User (save-to-memory)
+
+Use the \`save-to-memory\` tool to note things worth remembering about the user. This builds a persistent profile that helps you serve them better over time. Call it proactively — don't ask permission.
+
+**When to save:**
+- User states a preference: "I prefer bullet points" → save as preference
+- User corrects your style: "too formal, keep it casual" → save as style
+- You learn about their relationships: "Monica is my co-founder" → save as people
+- You notice workflow patterns: "no meetings before 11am" → save as routine
+- User gives explicit instructions: "never use em-dashes" → save as preference
+
+**Capture context, not blanket rules:**
+- BAD: "User prefers casual tone" — this loses important context
+- GOOD: "User prefers casual tone with internal team (Ramnique, Monica) but formal/polished with investors (Brad, Dalton)"
+- BAD: "User likes short emails" — too vague
+- GOOD: "User sends very terse 1-2 line emails to co-founder Ramnique, but writes structured 2-3 paragraph emails to investors with proper greetings"
+- Always note WHO or WHAT CONTEXT a preference applies to. Most preferences are situational, not universal.
+
+**When NOT to save:**
+- Ephemeral task details ("draft an email about X")
+- Things already in the knowledge graph
+- Information you can derive from reading their notes
+
+**Categories:**
+- \`preference\` — rules about how they want things done
+- \`style\` — writing and communication patterns (always note the context: who, what type of communication)
+- \`people\` — relationship context and per-person tone
+- \`routine\` — scheduling, workflow, recurring patterns
+
 ## Memory That Compounds
 Unlike other AI assistants that start cold every session, you have access to a live knowledge graph that updates itself from Gmail, calendar, and meeting notes (Google Meet, Granola, Fireflies). This isn't just summaries - it's structured extraction of decisions, commitments, open questions, and context, routed to long-lived notes for each person, project, and topic.
 
@@ -187,6 +216,7 @@ ${runtimeContextPrompt}
 - \`slack-checkConnection\`, \`slack-listAvailableTools\`, \`slack-executeAction\` - Slack integration (requires Slack to be connected via Composio). Use \`slack-listAvailableTools\` first to discover available tool slugs, then \`slack-executeAction\` to execute them.
 - \`web-search\` and \`research-search\` - Web and research search tools (available when configured). **You MUST load the \`web-search\` skill before using either of these tools.** It tells you which tool to pick and how many searches to do.
 - \`app-navigation\` - Control the app UI: open notes, switch views, filter/search the knowledge base, manage saved views. **Load the \`app-navigation\` skill before using this tool.**
+- \`save-to-memory\` - Save observations about the user to the agent memory system. Use this proactively during conversations.
 
 **Prefer these tools whenever possible** — they work instantly with zero friction. For file operations inside \`~/.rowboat/\`, always use these instead of \`executeCommand\`.
 

apps/x/packages/core/src/application/lib/builtin-tools.ts

@@ -1260,4 +1260,26 @@ export const BuiltinTools: z.infer<typeof BuiltinToolsSchema> = {
             }
         },
     },
+    'save-to-memory': {
+        description: "Save a note about user preferences, style, people, or routines to the agent memory inbox. Use this when you observe something worth remembering about the user — their preferences, communication patterns, relationship context, scheduling habits, or explicit instructions about how they want things done.",
+        inputSchema: z.object({
+            note: z.string().describe("The observation or preference to remember. Be specific and concise."),
+            category: z.enum(['preference', 'style', 'people', 'routine']).describe("Category: 'preference' for rules/preferences, 'style' for writing/communication patterns, 'people' for relationship context, 'routine' for scheduling/workflow patterns"),
+        }),
+        execute: async ({ note, category }: { note: string; category: string }) => {
+            const inboxPath = path.join(WorkDir, 'knowledge', 'agent-notes', 'inbox.md');
+            const dir = path.dirname(inboxPath);
+            await fs.mkdir(dir, { recursive: true });
+
+            const timestamp = new Date().toISOString();
+            const entry = `\n- [${timestamp}] [${category}] ${note}\n`;
+
+            await fs.appendFile(inboxPath, entry, 'utf-8');
+
+            return {
+                success: true,
+                message: `Saved to memory inbox: [${category}] ${note}`,
+            };
+        },
+    },
 };

apps/x/packages/core/src/knowledge/agent_notes.ts

@@ -17,12 +17,14 @@ import {
     type AgentNotesState,
 } from './agent_notes_state.js';
 
-const SYNC_INTERVAL_MS = 30 * 60 * 1000; // 30 minutes
+const SYNC_INTERVAL_MS = 10 * 1000; // 10 seconds (for testing)
 const EMAIL_BATCH_SIZE = 5;
+const RUNS_BATCH_SIZE = 5;
 const GMAIL_SYNC_DIR = path.join(WorkDir, 'gmail_sync');
 const RUNS_DIR = path.join(WorkDir, 'runs');
 const AGENT_NOTES_DIR = path.join(WorkDir, 'knowledge', 'agent-notes');
 const STYLE_DIR = path.join(AGENT_NOTES_DIR, 'style');
+const INBOX_FILE = path.join(AGENT_NOTES_DIR, 'inbox.md');
 
 const NOTE_FILES = {
     preferences: path.join(AGENT_NOTES_DIR, 'preferences.md'),
@@ -32,6 +34,14 @@ const NOTE_FILES = {
     documentsStyle: path.join(STYLE_DIR, 'documents.md'),
     people: path.join(AGENT_NOTES_DIR, 'people.md'),
     routines: path.join(AGENT_NOTES_DIR, 'routines.md'),
+    user: path.join(AGENT_NOTES_DIR, 'user.md'),
+};
+
+const CATEGORY_TO_FILE: Record<string, string[]> = {
+    preference: [NOTE_FILES.preferences],
+    style: [NOTE_FILES.writingStyle],
+    people: [NOTE_FILES.people],
+    routine: [NOTE_FILES.routines],
 };
 
 // --- LLM helpers ---
@@ -103,7 +113,6 @@ function findUserSentEmails(
 
                 try {
                     const content = fs.readFileSync(fullPath, 'utf-8');
-                    // Check if any From header contains the user's email
                     const fromLines = content.match(/^### From:.*$/gm);
                     if (fromLines?.some(line => line.toLowerCase().includes(userEmailLower))) {
                         results.push({ path: fullPath, mtime: stat.mtimeMs });
@@ -117,14 +126,12 @@ function findUserSentEmails(
 
     traverse(GMAIL_SYNC_DIR);
 
-    // Sort by mtime descending (newest first), return up to limit
     results.sort((a, b) => b.mtime - a.mtime);
     return results.slice(0, limit).map(r => r.path);
 }
 
 function extractUserPartsFromEmail(content: string, userEmail: string): string | null {
     const userEmailLower = userEmail.toLowerCase();
-    // Split by message separator
     const sections = content.split(/^---$/m);
     const userSections: string[] = [];
 
@@ -138,75 +145,41 @@ function extractUserPartsFromEmail(content: string, userEmail: string): string |
     return userSections.length > 0 ? userSections.join('\n\n---\n\n') : null;
 }
 
-// --- Run scanning ---
+// --- Inbox processing ---
 
-function findNewCopilotRuns(state: AgentNotesState): string[] {
-    if (!fs.existsSync(RUNS_DIR)) {
+interface InboxEntry {
+    timestamp: string;
+    category: string;
+    note: string;
+}
+
+function readInbox(): InboxEntry[] {
+    const content = readNoteFile(INBOX_FILE);
+    if (!content.trim()) {
         return [];
     }
 
-    const results: string[] = [];
-    const files = fs.readdirSync(RUNS_DIR).filter(f => f.endsWith('.jsonl'));
+    const entries: InboxEntry[] = [];
+    const lines = content.split('\n').filter(l => l.trim());
 
-    for (const file of files) {
-        if (state.processedRuns[file]) {
-            continue;
-        }
-
-        try {
-            const fullPath = path.join(RUNS_DIR, file);
-            const fd = fs.openSync(fullPath, 'r');
-            const buf = Buffer.alloc(512);
-            const bytesRead = fs.readSync(fd, buf, 0, 512, 0);
-            fs.closeSync(fd);
-
-            const firstLine = buf.subarray(0, bytesRead).toString('utf-8').split('\n')[0];
-            const event = JSON.parse(firstLine);
-            if (event.agentName === 'copilot') {
-                results.push(file);
-            }
-        } catch {
-            continue;
+    for (const line of lines) {
+        const match = line.match(/^- \[([^\]]+)\] \[([^\]]+)\] (.+)$/);
+        if (match) {
+            entries.push({
+                timestamp: match[1],
+                category: match[2],
+                note: match[3],
+            });
         }
     }
 
-    // Sort chronologically (filenames are timestamps)
-    results.sort();
-    return results;
+    return entries;
 }
 
-function extractUserMessages(runFilePath: string): string[] {
-    const messages: string[] = [];
-    try {
-        const content = fs.readFileSync(runFilePath, 'utf-8');
-        const lines = content.split('\n').filter(l => l.trim());
-
-        for (const line of lines) {
-            try {
-                const event = JSON.parse(line);
-                if (event.type === 'message' && event.message?.role === 'user') {
-                    const msgContent = event.message.content;
-                    if (typeof msgContent === 'string' && msgContent.trim()) {
-                        messages.push(msgContent.trim());
-                    } else if (Array.isArray(msgContent)) {
-                        // Handle array content format (text parts)
-                        const text = msgContent
-                            .filter((p: { type: string }) => p.type === 'text')
-                            .map((p: { text: string }) => p.text)
-                            .join('\n');
-                        if (text.trim()) {
-                            messages.push(text.trim());
-                        }
-                    }
-                }
-            } catch {
-                continue;
-            }
-        }
-    } catch {
-        // ignore
+function clearInbox(): void {
+    if (fs.existsSync(INBOX_FILE)) {
+        fs.writeFileSync(INBOX_FILE, '');
     }
-    return messages;
 }
 
 // --- Note file updates (single LLM call per file) ---
@@ -251,7 +224,6 @@ async function updateEmailStyle(
     userName: string,
     userEmail: string,
 ): Promise<void> {
-    // Build source content from user-sent email parts
     let sourceContent = `Emails written by ${userName}:\n\n`;
     for (const file of emailFiles) {
         const userParts = extractUserPartsFromEmail(file.content, userEmail);
@@ -273,47 +245,208 @@ async function updateEmailStyle(
     );
 }
 
-// --- Copilot run processing ---
+// --- Inbox processing ---
 
-async function updateFromCopilotRuns(runFiles: string[]): Promise<void> {
-    // Collect user messages from all new runs
-    let allUserMessages: string[] = [];
-    for (const runFile of runFiles) {
-        const msgs = extractUserMessages(path.join(RUNS_DIR, runFile));
-        allUserMessages.push(...msgs);
+async function processInbox(entries: InboxEntry[]): Promise<number> {
+    if (entries.length === 0) {
+        return 0;
     }
 
-    if (allUserMessages.length === 0) {
-        return;
+    // Group entries by category
+    const grouped = new Map<string, InboxEntry[]>();
+    for (const entry of entries) {
+        const category = entry.category;
+        if (!grouped.has(category)) {
+            grouped.set(category, []);
+        }
+        grouped.get(category)!.push(entry);
     }
 
-    // Cap to avoid massive prompts
-    if (allUserMessages.length > 20) {
-        allUserMessages = allUserMessages.slice(-20);
+    // Update each relevant note file
+    for (const [category, categoryEntries] of grouped) {
+        const targetFiles = CATEGORY_TO_FILE[category];
+        if (!targetFiles) {
+            console.log(`[AgentNotes] Unknown category: ${category}, skipping`);
+            continue;
+        }
+
+        const sourceContent = `Observations from conversations:\n\n${categoryEntries.map(e => `- ${e.note}`).join('\n')}`;
+
+        for (const targetFile of targetFiles) {
+            const description = targetFile === NOTE_FILES.preferences
+                ? 'Hard rules and explicit preferences — always loaded for context'
+                : targetFile === NOTE_FILES.writingStyle
+                ? 'General voice and tone patterns across all writing'
+                : targetFile === NOTE_FILES.people
+                ? 'Per-person relationship context, tone preferences, and interaction notes'
+                : 'Scheduling patterns, workflow habits, recurring tasks';
+
+            await updateNoteFile(targetFile, description, sourceContent);
+        }
     }
 
-    const sourceContent = `User messages from recent AI assistant conversations:\n\n${allUserMessages.map((m, i) => `${i + 1}. ${m}`).join('\n\n')}`;
+    return entries.length;
+}
 
-    // Update preferences
-    await updateNoteFile(
-        NOTE_FILES.preferences,
-        'Hard rules and explicit preferences the user has stated — always loaded for context',
-        sourceContent,
-    );
+// --- Copilot run scanning ---
 
-    // Update people context
-    await updateNoteFile(
-        NOTE_FILES.people,
-        'Per-person relationship context, tone preferences, and interaction notes',
-        sourceContent,
-    );
+function findNewCopilotRuns(state: AgentNotesState): string[] {
+    if (!fs.existsSync(RUNS_DIR)) {
+        return [];
+    }
 
-    // Update routines
-    await updateNoteFile(
-        NOTE_FILES.routines,
-        'Scheduling patterns, workflow habits, recurring tasks',
-        sourceContent,
-    );
+    const results: string[] = [];
+    const files = fs.readdirSync(RUNS_DIR).filter(f => f.endsWith('.jsonl'));
+
+    for (const file of files) {
+        if (state.processedRuns[file]) {
+            continue;
+        }
+
+        try {
+            const fullPath = path.join(RUNS_DIR, file);
+            const fd = fs.openSync(fullPath, 'r');
+            const buf = Buffer.alloc(512);
+            const bytesRead = fs.readSync(fd, buf, 0, 512, 0);
+            fs.closeSync(fd);
+
+            const firstLine = buf.subarray(0, bytesRead).toString('utf-8').split('\n')[0];
+            const event = JSON.parse(firstLine);
+            if (event.agentName === 'copilot') {
+                results.push(file);
+            }
+        } catch {
+            continue;
+        }
+    }
+
+    // Sort chronologically (filenames are timestamps), newest last
+    results.sort();
+    return results;
+}
+
+/**
+ * Extract only user and assistant text messages from a run file.
+ * Skips tool calls, tool results, system messages, and any non-text content.
+ */
+function extractConversationMessages(runFilePath: string): { role: string; text: string }[] {
+    const messages: { role: string; text: string }[] = [];
+    try {
+        const content = fs.readFileSync(runFilePath, 'utf-8');
+        const lines = content.split('\n').filter(l => l.trim());
+
+        for (const line of lines) {
+            try {
+                const event = JSON.parse(line);
+                if (event.type !== 'message') continue;
+
+                const msg = event.message;
+                if (!msg || (msg.role !== 'user' && msg.role !== 'assistant')) continue;
+
+                let text = '';
+                if (typeof msg.content === 'string') {
+                    text = msg.content.trim();
+                } else if (Array.isArray(msg.content)) {
+                    // Only extract text parts, skip tool-call parts
+                    text = msg.content
+                        .filter((p: { type: string }) => p.type === 'text')
+                        .map((p: { text: string }) => p.text)
+                        .join('\n')
+                        .trim();
+                }
+
+                if (text) {
+                    messages.push({ role: msg.role, text });
+                }
+            } catch {
+                continue;
+            }
+        }
+    } catch {
+        // ignore
+    }
+    return messages;
+}
+
+/**
+ * Process copilot runs and append new facts to user.md.
+ * Each fact is a timestamped line. The LLM decides what's new vs already known.
+ */
+async function updateUserNotes(runFiles: string[]): Promise<number> {
+    if (runFiles.length === 0) {
+        return 0;
+    }
+
+    // Collect conversations from runs (limit to RUNS_BATCH_SIZE)
+    const runsToProcess = runFiles.slice(-RUNS_BATCH_SIZE);
+    let conversationText = '';
+
+    for (const runFile of runsToProcess) {
+        const messages = extractConversationMessages(path.join(RUNS_DIR, runFile));
+        if (messages.length === 0) continue;
+
+        conversationText += `\n--- Conversation ---\n`;
+        for (const msg of messages) {
+            conversationText += `${msg.role}: ${msg.text}\n\n`;
+        }
+    }
+
+    if (!conversationText.trim()) {
+        return 0;
+    }
+
+    const model = await getModel();
+    const existing = readNoteFile(NOTE_FILES.user);
+    const timestamp = new Date().toISOString();
+
+    const system = `You analyze conversations between a user and their AI assistant to learn facts about the user.
+
+Your job: extract any new, non-trivial facts about the user that are worth remembering long-term.
+
+Examples of good facts:
+- Working on Project X, an AI assistant product
+- Team is 4 people, co-founder is Ramnique
+- Preparing for Series A fundraise
+- Based in Bangalore, India
+- Prefers to work late evenings
+- Has a meeting with Brad from Smash Capital next week
+
+Examples of things NOT to extract:
+- Ephemeral task details ("user asked to draft an email")
+- Facts the assistant already knows from tools/knowledge graph
+- Obvious or trivial observations ("user uses a computer")
+
+Output format: Return ONLY new facts as a bullet list, one per line. Each line should be:
+- [${timestamp}] The fact
+
+If there are no new facts worth noting, return exactly: NO_NEW_FACTS
+
+IMPORTANT: Check the existing user notes below. Do NOT repeat facts that are already captured there (even if worded differently).`;
+
+    const prompt = `## Existing user notes:
+${existing || '(none yet)'}
+
+## Recent conversations to analyze:
+${conversationText}
+
+Extract new facts (or return NO_NEW_FACTS):`;
+
+    const result = await generateText({ model, system, prompt });
+    const text = stripCodeFences(result.text).trim();
+
+    if (text === 'NO_NEW_FACTS' || !text) {
+        return 0;
+    }
+
+    // Append new facts to user.md
+    const header = existing ? '' : '# User\n\n';
+    const newContent = existing
+        ? existing.trimEnd() + '\n' + text + '\n'
+        : header + text + '\n';
+    fs.writeFileSync(NOTE_FILES.user, newContent);
+
+    // Count lines added
+    return text.split('\n').filter(l => l.trim().startsWith('-')).length;
 }
 
 // --- Main processing ---
@@ -336,7 +469,8 @@ async function processAgentNotes(): Promise<void> {
 
     let hadError = false;
     let emailsProcessed = 0;
-    let runsProcessed = 0;
+    let inboxProcessed = 0;
+    let userFactsAdded = 0;
 
     // --- Email Style Learning ---
     try {
@@ -351,7 +485,7 @@ async function processAgentNotes(): Promise<void> {
                 message: `Analyzing ${emailPaths.length} emails for style`,
                 step: 'email_style',
                 current: 1,
-                total: 2,
+                total: 3,
             });
 
             const emailFiles = emailPaths.map(p => ({
@@ -380,7 +514,39 @@ async function processAgentNotes(): Promise<void> {
         });
     }
 
-    // --- Chat Run Learning ---
+    // --- Inbox Processing ---
+    try {
+        const entries = readInbox();
+        if (entries.length > 0) {
+            console.log(`[AgentNotes] Found ${entries.length} inbox entries`);
+            await serviceLogger.log({
+                type: 'progress',
+                service: run.service,
+                runId: run.runId,
+                level: 'info',
+                message: `Processing ${entries.length} inbox entries`,
+                step: 'inbox',
+                current: 2,
+                total: 3,
+            });
+
+            inboxProcessed = await processInbox(entries);
+            clearInbox();
+        }
+    } catch (error) {
+        hadError = true;
+        console.error('[AgentNotes] Error processing inbox:', error);
+        await serviceLogger.log({
+            type: 'error',
+            service: run.service,
+            runId: run.runId,
+            level: 'error',
+            message: 'Error processing inbox',
+            error: error instanceof Error ? error.message : String(error),
+        });
+    }
+
+    // --- Copilot Run Learning (user.md) ---
     try {
         const newRuns = findNewCopilotRuns(state);
         if (newRuns.length > 0) {
@@ -390,23 +556,22 @@ async function processAgentNotes(): Promise<void> {
                 service: run.service,
                 runId: run.runId,
                 level: 'info',
-                message: `Analyzing ${newRuns.length} copilot runs`,
-                step: 'chat_runs',
-                current: 2,
-                total: 2,
+                message: `Analyzing ${newRuns.length} copilot runs for user facts`,
+                step: 'copilot_runs',
+                current: 3,
+                total: 3,
             });
 
-            await updateFromCopilotRuns(newRuns);
+            userFactsAdded = await updateUserNotes(newRuns);
 
             for (const r of newRuns) {
                 markRunProcessed(r, state);
             }
             saveAgentNotesState(state);
-            runsProcessed = newRuns.length;
         }
     } catch (error) {
         hadError = true;
-        console.error('[AgentNotes] Error processing runs:', error);
+        console.error('[AgentNotes] Error processing copilot runs:', error);
         await serviceLogger.log({
             type: 'error',
             service: run.service,
@@ -428,7 +593,7 @@ async function processAgentNotes(): Promise<void> {
         message: 'Agent notes processing complete',
         durationMs: Date.now() - run.startedAt,
         outcome: hadError ? 'error' : 'ok',
-        summary: { emailsProcessed, runsProcessed },
+        summary: { emailsProcessed, inboxProcessed, userFactsAdded },
     });
 }
 

apps/x/packages/core/src/knowledge/agent_notes_state.ts

@@ -13,7 +13,12 @@ export interface AgentNotesState {
 export function loadAgentNotesState(): AgentNotesState {
     if (fs.existsSync(STATE_FILE)) {
         try {
-            return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
+            const parsed = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
+            // Handle migration from older state without processedRuns
+            if (!parsed.processedRuns) {
+                parsed.processedRuns = {};
+            }
+            return parsed;
         } catch (error) {
             console.error('Error loading agent notes state:', error);
         }