diff --git a/.github/workflows/electron-build.yml b/.github/workflows/electron-build.yml index e68ce4f4..6566f105 100644 --- a/.github/workflows/electron-build.yml +++ b/.github/workflows/electron-build.yml @@ -16,14 +16,14 @@ jobs: uses: actions/checkout@v6 - name: Setup pnpm - uses: pnpm/action-setup@v6 + uses: pnpm/action-setup@v4 with: - version: 10 + version: 9 - name: Setup Node.js uses: actions/setup-node@v6 with: - node-version: 24.15.0 + node-version: 24 cache: 'pnpm' cache-dependency-path: 'apps/x/pnpm-lock.yaml' @@ -39,17 +39,17 @@ jobs: node -e " const fs = require('fs'); const version = '${{ steps.version.outputs.version }}'; - + // Update apps/x/package.json const rootPackage = JSON.parse(fs.readFileSync('apps/x/package.json', 'utf8')); rootPackage.version = version; fs.writeFileSync('apps/x/package.json', JSON.stringify(rootPackage, null, 2) + '\n'); - + // Update apps/x/apps/main/package.json const mainPackage = JSON.parse(fs.readFileSync('apps/x/apps/main/package.json', 'utf8')); mainPackage.version = version; fs.writeFileSync('apps/x/apps/main/package.json', JSON.stringify(mainPackage, null, 2) + '\n'); - + console.log('Updated version to:', version); " @@ -61,25 +61,25 @@ jobs: # Create a temporary keychain KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db KEYCHAIN_PASSWORD=$(openssl rand -base64 32) - + # Create keychain security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH" security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH" security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH" - + # Decode and import certificate echo "$APPLE_CERTIFICATE" | base64 --decode > $RUNNER_TEMP/certificate.p12 security import $RUNNER_TEMP/certificate.p12 -P "$APPLE_CERTIFICATE_PASSWORD" -A -t cert -f pkcs12 -k "$KEYCHAIN_PATH" - + # Allow codesign to access the keychain security set-key-partition-list -S apple-tool:,apple: -s -k "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH" - + # Add keychain to search list security list-keychains -d user -s "$KEYCHAIN_PATH" login.keychain - + # Verify certificate was imported security find-identity -v "$KEYCHAIN_PATH" - + # Clean up certificate file rm -f $RUNNER_TEMP/certificate.p12 @@ -111,7 +111,6 @@ jobs: with: name: distributables path: apps/x/apps/main/out/make/* - if-no-files-found: error retention-days: 30 build-linux: @@ -122,14 +121,14 @@ jobs: uses: actions/checkout@v6 - name: Setup pnpm - uses: pnpm/action-setup@v6 + uses: pnpm/action-setup@v4 with: - version: 10 + version: 9 - name: Setup Node.js uses: actions/setup-node@v6 with: - node-version: 24.15.0 + node-version: 24 cache: 'pnpm' cache-dependency-path: 'apps/x/pnpm-lock.yaml' @@ -145,17 +144,17 @@ jobs: node -e " const fs = require('fs'); const version = '${{ steps.version.outputs.version }}'; - + // Update apps/x/package.json const rootPackage = JSON.parse(fs.readFileSync('apps/x/package.json', 'utf8')); rootPackage.version = version; fs.writeFileSync('apps/x/package.json', JSON.stringify(rootPackage, null, 2) + '\n'); - + // Update apps/x/apps/main/package.json const mainPackage = JSON.parse(fs.readFileSync('apps/x/apps/main/package.json', 'utf8')); mainPackage.version = version; fs.writeFileSync('apps/x/apps/main/package.json', JSON.stringify(mainPackage, null, 2) + '\n'); - + console.log('Updated version to:', version); " @@ -163,25 +162,12 @@ jobs: run: pnpm install --frozen-lockfile working-directory: apps/x - - name: Build node-pty native binary for Linux - working-directory: apps/x - run: | - # node-pty ships prebuilt binaries only for darwin/win32; compile the - # linux-x64 binary so bundle.mjs can stage it into the package. Without - # this the Linux app crashes on launch (missing prebuilds/linux-x64/pty.node). - PTY="node_modules/.pnpm/node-pty@1.1.0/node_modules/node-pty" - cd "$PTY" - npx node-gyp rebuild - mkdir -p prebuilds/linux-x64 - cp build/Release/pty.node prebuilds/linux-x64/ - - name: Build electron app env: VITE_PUBLIC_POSTHOG_KEY: ${{ secrets.VITE_PUBLIC_POSTHOG_KEY }} VITE_PUBLIC_POSTHOG_HOST: ${{ secrets.VITE_PUBLIC_POSTHOG_HOST }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ROWBOAT_SKIP_PACMAN: '1' # Arch Linux package is built locally only, never in CI - run: npx electron-forge publish --arch=x64 --platform=linux + run: npx electron-forge publish --arch=x64,arm64 --platform=linux working-directory: apps/x/apps/main - name: Upload workflow artifacts @@ -189,7 +175,6 @@ jobs: with: name: distributables-linux path: apps/x/apps/main/out/make/* - if-no-files-found: error retention-days: 30 build-windows: @@ -200,14 +185,14 @@ jobs: uses: actions/checkout@v6 - name: Setup pnpm - uses: pnpm/action-setup@v6 + uses: pnpm/action-setup@v4 with: - version: 10 + version: 9 - name: Setup Node.js uses: actions/setup-node@v6 with: - node-version: 24.15.0 + node-version: 24 cache: 'pnpm' cache-dependency-path: 'apps/x/pnpm-lock.yaml' @@ -225,17 +210,17 @@ jobs: node -e " const fs = require('fs'); const version = '${{ steps.version.outputs.version }}'; - + // Update apps/x/package.json const rootPackage = JSON.parse(fs.readFileSync('apps/x/package.json', 'utf8')); rootPackage.version = version; fs.writeFileSync('apps/x/package.json', JSON.stringify(rootPackage, null, 2) + '\n'); - + // Update apps/x/apps/main/package.json const mainPackage = JSON.parse(fs.readFileSync('apps/x/apps/main/package.json', 'utf8')); mainPackage.version = version; fs.writeFileSync('apps/x/apps/main/package.json', JSON.stringify(mainPackage, null, 2) + '\n'); - + console.log('Updated version to:', version); " @@ -256,5 +241,4 @@ jobs: with: name: distributables-windows path: apps/x/apps/main/out/make/* - if-no-files-found: error retention-days: 30 diff --git a/.github/workflows/rowboat-build.yml b/.github/workflows/rowboat-build.yml new file mode 100644 index 00000000..270e6263 --- /dev/null +++ b/.github/workflows/rowboat-build.yml @@ -0,0 +1,47 @@ +name: Rowboat Build + +on: + pull_request: + +jobs: + build-rowboat-nextjs: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + cache-dependency-path: 'apps/rowboat/package-lock.json' + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + working-directory: apps/rowboat + + - name: Build Rowboat + run: npm run build + working-directory: apps/rowboat + + build-rowboatx: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + cache-dependency-path: 'apps/rowboat/package-lock.json' + node-version: '24' + cache: 'npm' + + - name: Install dependencies + run: npm ci + working-directory: apps/cli + + - name: Build Rowboat + run: npm run build + working-directory: apps/cli \ No newline at end of file diff --git a/.github/workflows/x-publish.yml b/.github/workflows/x-publish.yml new file mode 100644 index 00000000..c411ab68 --- /dev/null +++ b/.github/workflows/x-publish.yml @@ -0,0 +1,43 @@ +name: Publish to npm + +on: workflow_dispatch + +permissions: + id-token: write # Required for OIDC + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v6 + + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version: 24 + registry-url: https://registry.npmjs.org/ + + - name: Update npm + run: npm install -g npm@latest + + - name: Install deps + run: npm ci + working-directory: apps/cli + + # optional: run tests + # - run: npm test + + - name: Build + run: npm run build + working-directory: apps/cli + + - name: Pack + run: npm pack + working-directory: apps/cli + + - name: Publish to npm + run: npm publish --access public + working-directory: apps/cli \ No newline at end of file diff --git a/.github/workflows/x-tests.yml b/.github/workflows/x-tests.yml deleted file mode 100644 index c08c4095..00000000 --- a/.github/workflows/x-tests.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: rowboat - -on: - pull_request: - push: - branches: [main] - workflow_dispatch: - -jobs: - apps-x-vitest: - name: apps/x Vitest suites - runs-on: ubuntu-latest - defaults: - run: - working-directory: apps/x - steps: - - uses: actions/checkout@v4 - - - uses: pnpm/action-setup@v4 - with: - version: 10 - - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - cache-dependency-path: apps/x/pnpm-lock.yaml - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - # Builds @x/shared (core/renderer tests import its dist), then runs - # the shared, core, and renderer vitest suites in order. - - name: Run apps/x Vitest suites - run: npm test - - # Typechecks with the dev tsconfigs, which include test files — - # the build tsconfigs exclude them and vitest strips types without - # checking, so this is the only gate that sees type errors in tests. - - name: Typecheck apps/x packages - run: npm run typecheck - - apps-x-electron-package: - name: apps/x Electron package smoke test - runs-on: ubuntu-latest - defaults: - run: - working-directory: apps/x - steps: - - uses: actions/checkout@v4 - - - uses: pnpm/action-setup@v4 - with: - version: 10 - - - uses: actions/setup-node@v4 - with: - node-version: 24.15.0 - cache: pnpm - cache-dependency-path: apps/x/pnpm-lock.yaml - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Package Electron app without signing - env: - ROWBOAT_SKIP_CODE_SIGNING: "1" - run: npm run package - working-directory: apps/x/apps/main diff --git a/.gitignore b/.gitignore index c9eee447..086ea0b5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,3 @@ data/ .venv/ .claude/ - -# Local-only meeting-prep planning doc -/PLAN.md diff --git a/CLAUDE.md b/CLAUDE.md index a5fe24c8..75a0f6a5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,9 +109,7 @@ Long-form docs for specific features. Read the relevant file before making chang | Feature | Doc | |---------|-----| | Live Notes — single `live:` frontmatter block (one objective + optional cron / windows / eventMatchCriteria) that turns a note into a self-updating artifact, panel UI, Copilot skill, prompts catalog | `apps/x/LIVE_NOTE.md` | -| Calls (video mode) — one hands-free call engine with four presets (voice / video / share screen / practice coaching), device-derived surfaces (full-screen ⇄ floating popout), frame pipeline, prompts catalog | `apps/x/VIDEO_MODE.md` | | Analytics — PostHog event catalog, person properties, use-case taxonomy, how to add a new event | `apps/x/ANALYTICS.md` | -| Turn/session runtime — event-sourced storage, reference model, the `npm run inspect` debugger | `apps/x/packages/core/docs/turn-runtime-design.md`, `session-design.md` | ## Common Tasks @@ -139,7 +137,6 @@ Long-form docs for specific features. Read the relevant file before making chang ### Verify compilation ```bash cd apps/x && npm run deps && npm run lint -cd apps/x && npm run typecheck # dev tsconfigs — the only gate that typechecks *.test.ts ``` ## Tech Stack diff --git a/README.md b/README.md index 9e002cd1..361b87a0 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,9 @@ - rowboat-github-2 + rowboat-github-2
-

Rowboat

-

A desktop AI coworker with a memory of your work and built-in surfaces to act on it.

-

rowboatlabs/rowboat | Trendshift @@ -28,105 +25,28 @@

+# Rowboat +**Open-source AI coworker that turns work into a knowledge graph and acts on it** + -Rowboat indexes your work into a living knowledge graph and uses that to get work done on your machine. It includes work surfaces for collaborating with AI: email client, notes, browser, code mode, meeting note taker, and workspaces for different projects. +Rowboat connects to your email and meeting notes, builds a long-lived knowledge graph, and uses that context to help you get work done - privately, on your machine. +You can do things like: +- `Build me a deck about our next quarter roadmap` → generates a PDF using context from your knowledge graph +- `Prep me for my meeting with Alex` → pulls past decisions, open questions, and relevant threads into a crisp brief (or a voice note) +- Track a person, company or topic through live notes +- Visualize, edit, and update your knowledge graph anytime (it’s just Markdown) +- Record voice memos that automatically capture and update key takeaways in the graph Download latest for Mac/Windows/Linux: [Download](https://www.rowboatlabs.com/downloads) -

- -Rowboat Apps to Code demo - -

- -

- Demo - apps to code · Demo - knowledge graph -

- - ⭐ If you find Rowboat useful, please star the repo. It helps more people find it. ---- -## Overview +## Demo +[![Demo](https://github.com/user-attachments/assets/8b9a859b-d4f1-47ca-9d1d-9d26d982e15d)](https://www.youtube.com/watch?v=7xTpciZCfpw) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-

Brain

-Rowboat indexes email, meetings, slack and assistant conversations into a living Obsidian-style backlinked knowledge graph. -
-Brain graph screenshot -
-

Email

-The built-in email client sorts emails into important and everything else. Rowboat automatically drafts responses for important email using all the work context. -
-Email screenshot -
-

Background agents

-You can set up background agents that run on events like new email or on schedule like every day at 8am. They can connect to tools, search the web, use the browser and write code using Claude Code or Codex. -
-Background agents screenshot - -
-

Built-in Browser

-Rowboat includes a browser that lets you and assistant collaborate on web tasks. Because it's isolated from your main browser, you can log in only to the accounts that want the assistant to access. -
-Browser screenshot -
-

Meeting Notes

-A local meeting note-taker that taps into mic & speaker, produces live transcript and summarizes the meeting in a markdown file and updates the knowledge graph. -
-Meeting notes screenshot -
-

Code Mode

-Code mode lets you spin up parallel coding agents with Claude Code or Codex, and have Rowboat drive them with all the work context where needed. -
-Code mode screenshot -
-

Apps

-You can build your own work surfaces inside Rowboat — they get access to all the tools and integrations, and you can share them with other people. -
-Apps screenshot -
-

Integrations

-Includes one-click integrations to most popular products. -
-Integrations screenshot -
+[Watch the full video](https://www.youtube.com/watch?v=7xTpciZCfpw) --- @@ -161,6 +81,23 @@ All API key files use the same format: } ``` +## What it does + +Rowboat is a **local-first AI coworker** that can: +- **Remember** the important context you don’t want to re-explain (people, projects, decisions, commitments) +- **Understand** what’s relevant right now (before a meeting, while replying to an email, when writing a doc) +- **Help you act** by drafting, summarizing, planning, and producing real artifacts (briefs, emails, docs, PDF slides) + +Under the hood, Rowboat maintains an **Obsidian-compatible vault** of plain Markdown notes with backlinks — a transparent “working memory” you can inspect and edit. + +## Integrations + +Rowboat builds memory from the work you already do, including: +- **Gmail** (email) +- **Google Calendar** +- **Rowboat meeting notes** or **Fireflies** + +It also contains a library of product integrations through Composio.dev ## How it’s different @@ -174,6 +111,24 @@ Rowboat maintains **long-lived knowledge** instead: The result is memory that compounds, rather than retrieval that starts cold every time. +## What you can do with it + +- **Meeting prep** from prior decisions, threads, and open questions +- **Email drafting** grounded in history and commitments +- **Docs & decks** generated from your ongoing context (including PDF slides) +- **Follow-ups**: capture decisions, action items, and owners so nothing gets dropped +- **On-your-machine help**: create files, summarize into notes, and run workflows using local tools (with explicit, reviewable actions) + +## Live notes + +Live notes are notes that stay updated automatically. You can create one by typing '@rowboat' on a note. + +- Track a competitor or market topic across X, Reddit, and the news +- Monitor a person, project, or deal across web or your communications +- Keep a running summary of any subject you care about + +Everything is written back into your local Markdown vault. You control what runs and when. + ## Bring your own model Rowboat works with the model setup you prefer: diff --git a/apps/cli/src/application/lib/command-executor.ts b/apps/cli/src/application/lib/command-executor.ts index b2a5a94f..cd16f05e 100644 --- a/apps/cli/src/application/lib/command-executor.ts +++ b/apps/cli/src/application/lib/command-executor.ts @@ -4,13 +4,7 @@ import { getSecurityAllowList, SECURITY_CONFIG_PATH } from '../../config/securit import { getExecutionShell } from '../assistant/runtime-context.js'; const execPromise = promisify(exec); -// Order matters: longer separators (`||`, `&&`) must precede their single-char -// prefixes (`|`, `&`) so the leftmost-longest match consumes the right token. -// `&` (background), backtick / `$(` (command substitution), and `(` `)` -// (subshell) are also command separators — without them, `echo hi & rm /x`, -// `echo \`rm /x\``, and `echo $(rm /x)` slip past isBlocked() with only -// `echo` in the allowlist. -const COMMAND_SPLIT_REGEX = /(?:\|\||&&|&|;|\||\n|`|\$\(|\(|\))/; +const COMMAND_SPLIT_REGEX = /(?:\|\||&&|;|\||\n)/; const ENV_ASSIGNMENT_REGEX = /^[A-Za-z_][A-Za-z0-9_]*=.*/; const WRAPPER_COMMANDS = new Set(['sudo', 'env', 'time', 'command']); const EXECUTION_SHELL = getExecutionShell(); diff --git a/apps/cli/todo.md b/apps/cli/todo.md index 3ca4d45c..9c97afd3 100644 --- a/apps/cli/todo.md +++ b/apps/cli/todo.md @@ -12,4 +12,4 @@ o file logging rowboat agent --- -- create agent +- create agent \ No newline at end of file diff --git a/apps/docs/docs.json b/apps/docs/docs.json index 3178f741..8442b158 100644 --- a/apps/docs/docs.json +++ b/apps/docs/docs.json @@ -55,6 +55,6 @@ "chatgpt", "claude" ] - } + } } - + \ No newline at end of file diff --git a/apps/docs/docs/development/contribution-guide.mdx b/apps/docs/docs/development/contribution-guide.mdx index 8fbf2a1d..1089d7e0 100644 --- a/apps/docs/docs/development/contribution-guide.mdx +++ b/apps/docs/docs/development/contribution-guide.mdx @@ -52,4 +52,4 @@ Rowboat is open-source and we welcome contributions of all kinds — bug reports ## Getting Help -If you're stuck or unsure about anything, drop a message in our [Discord](https://discord.gg/wajrgmJQ6b). We're happy to help you get unblocked. +If you're stuck or unsure about anything, drop a message in our [Discord](https://discord.gg/wajrgmJQ6b). We're happy to help you get unblocked. \ No newline at end of file diff --git a/apps/docs/docs/development/roadmap.mdx b/apps/docs/docs/development/roadmap.mdx index 8202b2b2..6248e567 100644 --- a/apps/docs/docs/development/roadmap.mdx +++ b/apps/docs/docs/development/roadmap.mdx @@ -4,4 +4,4 @@ icon: "road" # Roadmap -Explore the future development plans and upcoming features for Rowboat. +Explore the future development plans and upcoming features for Rowboat. \ No newline at end of file diff --git a/apps/x/.gitignore b/apps/x/.gitignore index 2c5f63fd..db195fb4 100644 --- a/apps/x/.gitignore +++ b/apps/x/.gitignore @@ -1,3 +1,2 @@ node_modules/ test-fixtures/ -*.tsbuildinfo diff --git a/apps/x/.pnpm-store/v11/index.db b/apps/x/.pnpm-store/v11/index.db deleted file mode 100644 index 7f9770bd..00000000 Binary files a/apps/x/.pnpm-store/v11/index.db and /dev/null differ diff --git a/apps/x/ANALYTICS.md b/apps/x/ANALYTICS.md index 5bc0f456..572e9a6f 100644 --- a/apps/x/ANALYTICS.md +++ b/apps/x/ANALYTICS.md @@ -16,15 +16,13 @@ ## Event catalog -All PostHog events include `app_version` and `platform: 'desktop'` automatically. Main-process events add them in `packages/core/src/analytics/posthog.ts`; renderer events get them from the `analytics:bootstrap` IPC payload via `posthog.register` (plus an initialization-time `before_send` hook for `app_version`). `platform` guards against the legacy web dashboard's autocapture (`apps/rowboat`, unidentified by design) muddying desktop dashboards if it ever shares the project. - ### `llm_usage` Emitted whenever ai-sdk returns token usage (one event per LLM call, not per run). | Property | Type | Notes | |---|---|---| -| `use_case` | enum | `copilot_chat` / `live_note_agent` / `meeting_note` / `knowledge_sync` / `code_session` | +| `use_case` | enum | `copilot_chat` / `live_note_agent` / `meeting_note` / `knowledge_sync` | | `sub_use_case` | string? | Refines `use_case` — see taxonomy table below | | `agent_name` | string? | Present when the call goes through an agent run (`createRun`); omitted for direct `generateText`/`generateObject` | | `model` | string | e.g. `claude-sonnet-4-6` | @@ -41,9 +39,9 @@ Every `llm_usage` emit point in the codebase: | `use_case` | `sub_use_case` | `agent_name`? | Where | File:line | |---|---|---|---|---| -| `copilot_chat` | (none) | yes | User chat in renderer (turn runtime; the ALS default when no caller set a use case) | `packages/core/src/runtime/turns/bridges/real-usage-reporter.ts` (`reportModelUsage`); legacy runs (code-mode carve-out) still emit from `packages/core/src/runtime/legacy/engine.ts` (`streamLlm` finish-step) | +| `copilot_chat` | (none) | yes | User chat in renderer (default for any `createRun` without `useCase`) | `packages/core/src/agents/runtime.ts:1313` (finish-step in `streamLlm`) | | `copilot_chat` | `scheduled` | yes | Background scheduled agent runner | `packages/core/src/agent-schedule/runner.ts:167` | -| `copilot_chat` | `file_parse` | inherits | `parseFile` builtin tool inside any chat | `packages/core/src/runtime/tools/domains/parsing.ts:179` | +| `copilot_chat` | `file_parse` | inherits | `parseFile` builtin tool inside any chat | `packages/core/src/application/lib/builtin-tools.ts:770` | | `live_note_agent` | `routing` | no | Pass 1 routing classifier (`generateObject`) | `packages/core/src/knowledge/live-note/routing.ts:93` | | `live_note_agent` | `manual` | yes | Pass 2 agent run — user clicked Run / called the `run-live-note-agent` tool | `packages/core/src/knowledge/live-note/runner.ts:140` (createRun, `subUseCase: trigger`) | | `live_note_agent` | `cron` | yes | Pass 2 agent run — cron expression matched | same call site | @@ -53,10 +51,10 @@ Every `llm_usage` emit point in the codebase: | `knowledge_sync` | `agent_notes` | yes | Agent notes learning service | `packages/core/src/knowledge/agent_notes.ts:309` (createRun) | | `knowledge_sync` | `tag_notes` | yes | Note tagging | `packages/core/src/knowledge/tag_notes.ts:86` (createRun) | | `knowledge_sync` | `build_graph` | yes | Knowledge graph note creation | `packages/core/src/knowledge/build_graph.ts:253` (createRun) | +| `knowledge_sync` | `label_emails` | yes | Email labeling | `packages/core/src/knowledge/label_emails.ts:73` (createRun) | | `knowledge_sync` | `inline_task_run` | yes | Inline `@rowboat` task execution (two call sites) | `packages/core/src/knowledge/inline_tasks.ts:471, 552` (createRun) | | `knowledge_sync` | `inline_task_classify` | no | Inline task scheduling classifier (`generateText`) | `packages/core/src/knowledge/inline_tasks.ts:673` | | `knowledge_sync` | `pre_built` | yes | Pre-built scheduled agents | `packages/core/src/pre_built/runner.ts:43` (createRun) | -| `code_session` | (none) | yes | Code-section coding session in Rowboat mode (direct mode talks to the on-device coding agent and emits no `llm_usage`) | `packages/core/src/code-mode/sessions/service.ts` (createRun) | ##### `live_note_agent` sub-use-case shape @@ -91,106 +89,9 @@ All in `apps/renderer/src/lib/analytics.ts`: - `chat_message_sent` — `{ voice_input, voice_output, search_enabled }` - `oauth_connected` / `oauth_disconnected` — `{ provider }` - `voice_input_started` — no properties -- `call_started` — `{ preset: 'voice' | 'video' | 'share' | 'practice' }` — a hands-free call began (see `apps/x/VIDEO_MODE.md`) -- `call_turn_latency` — `{ endpoint_to_submit_ms, submit_to_speak_ms, speak_to_audio_ms, total_ms }` — voice-to-voice latency breakdown for one call turn (utterance accepted → submitted → first TTS speak → audio playing) - `search_executed` — `{ types: string[] }` - `note_exported` — `{ format }` -### Client auto-update funnel - -The desktop client's own updates — distinct from the in-app apps feature, which owns `app_updated`: - -- `update_prompted` — renderer (`apps/renderer/src/lib/analytics.ts`): the "Update available" card was shown for a staged update -- `update_restarted` — main (`apps/main/src/updater.ts`), `{ from, to? }`: the user clicked restart-to-update (`to` may be missing when the update feed doesn't report the release name) -- `update_failed` — main (`apps/main/src/updater.ts`), `{ message }`: the auto-updater errored (includes network errors for now) -- `client_updated` — main (`apps/main/src/ipc.ts`), `{ from, to }`: first launch on a newer version (fires once per update, whatever the restart path; downgrades restamp silently and don't fire) -### `view_opened` — feature-importance funnel - -One event per view the user lands on, fired centrally from the `currentViewState` effect in `apps/renderer/src/App.tsx`. `view` is one of: `chat`, `file`, `graph`, `task`, `suggested-topics`, `meetings`, `live-notes`, `email`, `workspace`, `knowledge-view`, `chat-history`, `home`, `code`, `bg-tasks`, `apps`. Keyed on the view *type*, so switching files or threads inside a view doesn't re-fire. - -This is the top of every feature funnel: unique users on `view = 'email'` ÷ all users = how many people even open email. First visit to a key view also sets a one-shot person property (`has_used_email`, `has_used_meetings`, `has_used_live_notes`, `has_used_bg_agents`, `has_used_apps`, `has_used_code`) for cohort building. - -### Feature action events - -All renderer events live in `apps/renderer/src/lib/analytics.ts` (typed wrappers); the emit sites are in the components named below. Events marked **(main)** are captured in `apps/main/src/ipc.ts` via `capture()` because the operation runs there. - -**Email** (`components/email-view.tsx`): - -- `email_thread_opened` — a thread was expanded in the list -- `email_compose_opened` — `{ mode: 'new' | 'reply' | 'replyAll' | 'forward' | 'draft' }` — a composer was opened -- `email_sent` — `{ mode, has_attachments, ai_assisted }` — `ai_assisted` is true when Write-with-AI produced a draft in that composer -- `email_ai_draft_generated` — `{ mode: 'generate' | 'rewrite' }` — the Write/Edit-with-AI bar completed -- `email_archived` / `email_trashed` — one thread archived / moved to trash -- `email_marked_unread` — explicit mark-as-unread (marking *read* fires automatically on open, so it's deliberately not tracked) -- `email_importance_changed` — `{ importance: 'important' | 'other' }` — user corrected the importance verdict -- `email_category_changed` — `{ category }` — user re-filed a thread -- `email_category_archived` — `{ category }` — bulk "archive all in category" -- `email_searched` — a search query executed (debounced, one per settled query) -- `email_instructions_saved` — standing email-agent instructions saved -- `email_sync_triggered` — manual refresh button - -**Meetings** (`App.tsx`, `components/meetings-view.tsx`): - -- `meeting_recording_started` — `{ has_calendar_event }` — transcription actually began (all entry points: meetings view, home, sidebar, popup funnel through one call site) -- `meeting_recording_stopped` — `{ duration_seconds }` -- `meeting_popup_action` — `{ action: 'take-notes' | 'dismiss' }` **(main)** — the "meeting detected" popup window runs without PostHog, so the action is captured in its IPC handler -- `meeting_note_opened` — a past meeting note opened from the meetings list - -**Calls** (`App.tsx`): - -- `call_started` — (pre-existing, above) fires on every call-button press that starts a call -- `call_ended` — `{ duration_seconds }` - -**Background agents** (`components/bg-tasks-view.tsx`, `components/apps/app-detail.tsx`): - -- `bg_agent_created` — `{ method: 'manual' | 'coding' | 'copilot', has_triggers }` — `copilot` means the user submitted the "describe it" form (the agent is then created by Copilot in chat) -- `bg_agent_updated` — instructions/triggers/model saved on an existing agent -- `bg_agent_toggled` — `{ active }` -- `bg_agent_run_clicked` — manual Run now -- `bg_agent_stopped` — manual stop of a run -- `bg_agent_deleted` - -**Live notes** (`components/live-note-sidebar.tsx`, `components/live-notes-view.tsx`): - -- `live_note_saved` — live config created or edited via the panel -- `live_note_toggled` — `{ active }` -- `live_note_run_clicked` — manual Run -- `live_note_stopped` — in-flight run stopped -- `live_note_deleted` — live config removed from the note -- `live_note_edit_with_copilot_clicked` - -**Search** (`components/search-dialog.tsx`): - -- `search_opened` — the palette opened -- `search_executed` — (pre-existing, above) -- `search_result_selected` — `{ type: 'knowledge' | 'chat' }` - -**Apps** — all **(main)**, in `apps/main/src/ipc.ts` (pre-existing except `app_rolled_back`): `app_created`, `app_installed`, `app_uninstalled`, `app_updated`, `app_rolled_back`, `app_published`, `app_starred`, `app_deleted`. Plus renderer-side `app_opened` — `{ folder }` — an installed app's UI was opened (`components/apps/app-frame.tsx`). - -**Code mode** — both **(main/core)**: - -- `code_session_created` — `{ mode: 'direct' | 'rowboat', agent }` — captured in the `codeSession:create` IPC handler. This is the direct-vs-rowboat session split. -- `code_session_message_sent` — `{ mode, agent }` — one per direct-drive message (`packages/core/src/code-mode/sessions/service.ts`). Direct turns bypass the agent runtime and emit no `llm_usage`, so this is the only usage-depth signal for direct mode; Rowboat-mode depth comes from `llm_usage where use_case = code_session`. - -**Billing** (`components/billing-error-dialog.tsx`): - -- `billing_error_shown` — `{ kind: 'subscription_required' | 'out_of_credits' | 'subscription_inactive' }` — the paywall dialog appeared -- `billing_upgrade_clicked` — `{ kind }` — the upgrade CTA was clicked (shown → clicked = paywall conversion) - -**Failures** — success events all have a failure sibling where the operation can fail after the click: - -- `email_send_failed` — send returned an error or threw (`components/email-view.tsx`) -- `meeting_summarize_failed` — post-recording notes generation threw (`App.tsx`) -- `bg_agent_run_failed` / `bg_agent_run_completed` — `{ trigger: 'manual' | 'cron' | 'window' | 'event' }` **(core)** — every background-agent run settles as exactly one of these (`packages/core/src/background-tasks/runner.ts`), giving a failure *rate* across all trigger sources, not just manual clicks - -**Misc**: - -- `note_created` — new note from the sidebar/knowledge actions (`App.tsx`) -- `note_edited` — a note's autosave wrote changed content; deduped to one event per note per app session (so it counts "notes touched", not keystroke bursts) -- `settings_opened` — `{ tab }` — settings dialog opened (tab = the initial tab) -- `settings_tab_changed` — `{ tab }` -- `onboarding_completed` — the onboarding flow finished (`App.tsx`) - ## Person properties Persistent across sessions for the same user. Set via `posthog.people.set` or as the `properties` arg to `identify`. @@ -200,14 +101,10 @@ Persistent across sessions for the same user. Set via `posthog.people.set` or as | `email` | main on identify | From `/v1/me`; powers PostHog cohort match + integrations | | `plan`, `status` | main on identify | Subscription state | | `api_url` | both processes (init + identify) | Distinguishes prod / staging / custom — assign meaning in PostHog dashboard. `https://api.x.rowboatlabs.com` = production | -| `platform` | both processes (init + identify) | Always `desktop` from this app; segments desktop users from any other surface | -| `app_version` | both processes (init + identify) | Electron app version; also included automatically on every event | | `signed_in` | renderer | `true` while rowboat OAuth is connected | | `{provider}_connected` | renderer | One of `gmail`, `calendar`, `slack`, `rowboat` | | `total_notes` | renderer (init) | Workspace size signal | | `has_used_search`, `has_used_voice` | renderer | One-shot first-use flags | -| `has_used_email`, `has_used_meetings`, `has_used_live_notes`, `has_used_bg_agents`, `has_used_apps`, `has_used_code` | renderer (`view_opened`) | One-shot first-use flags per feature view | -| `has_created_bg_agent` | renderer | One-shot: user set up a background agent | ## How to add a new event diff --git a/apps/x/CODE_MODE_ENGINES_PLAN.md b/apps/x/CODE_MODE_ENGINES_PLAN.md deleted file mode 100644 index ce7a4958..00000000 --- a/apps/x/CODE_MODE_ENGINES_PLAN.md +++ /dev/null @@ -1,271 +0,0 @@ -# Code Mode — Managed Engine Provisioning Plan - -Branch: `feat/code-mode-managed-engines` (off `dev` @ `8ce24ebb`) - -## 1. Problem & Goal - -Code mode runs two coding agents — **Claude Code** and **Codex** — by spawning their -ACP adapters, which in turn spawn a heavy **native engine binary** (~205 MB claude, -~194 MB codex). We need code mode to work in **packaged releases with ~99% reliability -for both agents**, without shipping a ~400 MB installer. - -### Current state (HEAD = revert of #614) -- Packaged builds **do not stage** the ACP adapters, and `forge.config.cjs` - `ignore: /^\/node_modules\//` strips them. At runtime `agents.ts` resolves the - adapter via `require.resolve(...)` then spawns it — which **throws - `Cannot find module '@agentclientprotocol/...'`**. -- **Net: packaged code mode is broken in every release.** It only works in `dev` - because pnpm symlinks exist. There is no 400 MB bloat today — but no function either. - -### Why the two prior approaches are insufficient -- **Bundle engines (A):** +~400 MB per installer (one claude + one codex native binary - per OS/arch). Works offline, but installer is huge. -- **Drive from user's local install (B)** — what #614 settled on and was reverted: - requires the user to have **both** CLIs installed **and** logged in, correct version, - on the right PATH. Depends on the user's machine → **structurally cannot hit 99%** - (version skew, GUI-launch PATH stripping, missing installs). This is why #614 was - reverted. - -## 2. Chosen Architecture — Managed Engine Provisioning (validated against Conductor) - -Split the problem into **engine** vs **auth**, treat them differently — exactly what -Conductor (conductor.build) does: - -1. **Engine = owned by the app.** We provision **version-pinned** engine binaries into - **app-support** (`~/.rowboat/engines///`), download-on-demand on first - use, sha256-verified, symlink/path-pinned. Not the user's global npm, not their PATH. - → no version skew, no PATH quirks → this is what delivers the 99%. -2. **Auth = reused from the user.** The engines read existing credentials - (`~/.claude` API key / Pro / Max, `~/.codex` auth.json). No second login. `status.ts` - already inspects these. - -### Empirical proof from Conductor on this machine -- DMG is **123 MB** — far smaller than the ~400 MB of engines → engines are **not** in - the installer; they're downloaded after install. -- Layout observed at `~/Library/Application Support/com.conductor.app/`: - ``` - agent-binaries/claude/2.1.170/claude (222 MB, single Mach-O arm64) - agent-binaries/codex/0.138.0/codex (205 MB, single Mach-O arm64) - bin/claude -> agent-binaries/claude/2.1.170/claude (symlink to active version) - bin/codex -> agent-binaries/codex/0.138.0/codex - agent-binaries/.meta/claude-2.1.170.json = {sha256, size, downloaded_at_unix_ms, ...} - ``` -- So: versioned dirs + stable symlink + sha256 `.meta` ledger + download. We mirror this. - -## 3. Concrete facts that make this clean (verified) - -### Adapters honor an external engine via env var (no code change in adapters needed) -- **Claude** — `@agentclientprotocol/claude-agent-acp@0.39.0`, - `dist/acp-agent.js:39`: `if (process.env.CLAUDE_CODE_EXECUTABLE) return it;` - and line 1552 `pathToClaudeCodeExecutable: process.env.CLAUDE_CODE_EXECUTABLE ?? ...`. - If unset and no bundled native dep → it throws "set CLAUDE_CODE_EXECUTABLE". -- **Codex** — `@agentclientprotocol/codex-acp@0.0.44`, - `dist/index.js:20900`: `const codexPath = process.env["CODEX_PATH"] ?? "codex";` - → `spawn(codexPath, ["app-server"])`. -- **Implication:** provisioning + setting these two env vars is the *entire* engine story. - -### Our engine packages are already single self-contained binaries -- `@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.156` → contains one `claude` - executable (+ LICENSE/README). -- `@openai/codex@0.128.0-darwin-arm64` → `vendor//codex/codex` native binary - **plus a bundled `rg` (ripgrep) at `vendor//path/rg`** — see Risk R1. - -### Pinned versions + platform package names (read from the installed adapter trees) -- **Claude:** adapter `claude-agent-acp@0.39.0` → engine `@anthropic-ai/claude-agent-sdk@0.3.156`. - Platform optional deps `@anthropic-ai/claude-agent-sdk-@0.3.156`: - `darwin-arm64`, `darwin-x64`, `linux-x64`, `linux-arm64`, `linux-x64-musl`, - `linux-arm64-musl`, `win32-x64`, `win32-arm64`. -- **Codex:** adapter `codex-acp@0.0.44` → `@openai/codex@^0.128.0` (pnpm-**patched**, see R2). - Platform deps aliased `@openai/codex-` → `npm:@openai/codex@0.128.0-`: - `darwin-arm64`, `darwin-x64`, `linux-x64`, `linux-arm64`, `win32-x64`, `win32-arm64`. - -## 4. Distribution source — npm platform packages, adapter-pinned (DECIDED) - -We fetch the **per-platform engine packages from the npm registry at the exact versions -our ACP adapters depend on**, extract the native binary, and provision it into -`~/.rowboat/engines/...`. No self-host, no curl installer, no fallback. - -- **Tarball URL:** `https://registry.npmjs.org//-/-.tgz` - - claude: `@anthropic-ai/claude-agent-sdk-@0.3.156` → extract the `claude` binary. - - codex: `@openai/codex@0.128.0-` → extract `vendor//codex/codex` - **and keep `vendor//path/rg`** (see R1). -- **Why adapter-pinned npm (not curl installer, not release bucket, not self-host):** - - The binary is the **exact version our adapter was built/tested against** → ACP - handshake guaranteed → the key to ~99% reliability. - - npm registry is highly available, immutable per-version, and the packument provides - `dist.integrity` (sha512) + `dist.shasum` (sha1) → integrity verification with **zero - infra**. - - The official `curl | bash` installer is for end-users: it installs globally to - `~/.local/bin` and **auto-updates in the background** — exactly what we must avoid. We - want an isolated, pinned, app-managed copy that never moves under the adapter. - (Conductor likewise fetches the raw binary rather than running the installer.) -- **Versions are read from our lockfile at build time** and embedded in - `engine-manifest.json`, so the manifest can never drift from the adapters we ship. - -### Reference: how this relates to Conductor / official installers -- Conductor self-hosts the same kind of native binaries on `storage.conductor.build` - (raw/`.gz`/`.zst` + `{url,gzipUrl,zstdUrl,sha256}` manifest), pairing the adapters with - recent **standalone CLI** versions (claude 2.1.x, codex 0.138). That proves recent - versions work — but we deliberately pin to the **adapter's own** engine version for - determinism. -- Official Claude releases also publish a GPG-signed `manifest.json` (SHA256/platform) at - `downloads.claude.ai/claude-code-releases//`. We don't use it now (npm is simpler - and adapter-matched), but it's the upgrade path if we ever want the latest CLI line. -- If we later want control/availability/compression, we can mirror these npm tarballs to - our own bucket — **runtime stays identical, only manifest URLs change.** - -### Locked decisions -- **Source: npm platform packages, adapter-pinned versions (user).** -- **No self-host (user).** -- **Always provision; no fallback** to a user's pre-installed `claude`/`codex` (user). - Reverted path B is fully retired. -- **Codex pnpm patch — IRRELEVANT (user):** it targets the JS launcher; we point - `CODEX_PATH` at the native binary, so it does not apply. (Risk R2 removed.) - -## 5. Design - -### 5.1 Build-time: generate an engine manifest + stage adapter JS - -**(a) Engine manifest (`engine-manifest.json`, embedded in the app).** -A build script reads the installed adapter dependency trees and emits, per agent: -```jsonc -{ - "claude": { - "version": "0.3.156", - "platforms": { - "darwin-arm64": { "pkg": "@anthropic-ai/claude-agent-sdk-darwin-arm64", - "tarball": "https://registry.npmjs.org/.../-/...-0.3.156.tgz", - "integrity": "sha512-...", - "binRelPath": "claude" }, - "...": {} - } - }, - "codex": { - "version": "0.128.0", - "platforms": { - "darwin-arm64": { "pkg": "@openai/codex-darwin-arm64", - "tarball": "https://registry.npmjs.org/...", - "integrity": "sha512-...", - "binRelPath": "vendor/aarch64-apple-darwin/codex/codex", - "extraPaths": ["vendor/aarch64-apple-darwin/path/rg"] } - } - } -} -``` -- Versions + tarball URLs + integrity are pulled from the lockfile / npm packument so the - manifest is always in sync with the adapters we ship. (Generating it at build time - sidesteps hardcoding fragile package names.) -- `binRelPath` / `extraPaths` capture where the executable (and codex's `rg`) live inside - each tarball. - -**(b) Stage the ACP adapter JS into the package (the part of #614 we KEEP).** -The adapters themselves (tiny, ~15 MB total incl. their non-native JS deps) must exist on -disk in packaged builds so `agents.ts` can resolve + spawn them. In `forge.config.cjs` -`generateAssets`, stage the two adapters and their **non-native** production dependency -closure into `.package/acp/node_modules` (npm-style nested layout), and **exempt -`.package` from the `node_modules` ignore rule**. We **drop** #614's native-engine staging -entirely — engines come from provisioning, not the bundle. - -### 5.2 Runtime: engine provisioner (new module in `packages/core`) - -New file: `packages/core/src/code-mode/acp/engine-provisioner.ts`. - -``` -ensureEngine(agent): Promise<{ executablePath: string }> - 1. Read manifest entry for (agent, currentPlatform). Error clearly if unsupported. - 2. dir = ~/.rowboat/engines/// - 3. If dir exists AND .meta/-.json sha256 matches → return binPath. - 4. Else acquire a cross-process lock (avoid double download), then: - a. Download tarball to a temp file, streaming, with progress events. - b. Verify integrity (sha512/sha256 from manifest). - c. Extract into a temp dir, then atomic rename into / (tar gzip). - d. chmod +x the binary (and codex's rg) on unix. - e. Write .meta/-.json {sha256, size, downloaded_at_unix_ms}. - 5. Return absolute path to the engine executable (binRelPath joined to dir). -``` -- **Progress + cancellation** surfaced over IPC for the first-run "Downloading engine…" UI. -- **Offline / failure** → typed error with a clear, actionable message (not a hang). -- **Resumability / atomicity:** download to temp, verify, then rename; never leave a - half-extracted version dir that passes the existence check. - -### 5.3 Wire provisioning into launch - -In `agents.ts` `getAgentLaunchSpec()` (currently sets only `CLAUDE_CODE_EXECUTABLE`): -- Make it (or its caller in `client.ts` / `manager.ts`) `await ensureEngine(agent)` first, - then set: - - claude → `env.CLAUDE_CODE_EXECUTABLE = ` - - codex → `env.CODEX_PATH = ` (and ensure its `rg` sibling resolves; - keep the vendor dir layout intact so codex finds `../path/rg`). -- Keep `ELECTRON_RUN_AS_NODE=1` for the adapter spawn (unchanged). -- The provisioner replaces both the bundled-engine dependency *and* the reverted - "resolve user's local install" path. (Optionally: if a healthy provisioned engine is - absent but a compatible local install exists, we *may* fall back to it — but the default - and reliable path is the provisioned engine. Decide in review; default = provisioned only.) - -### 5.4 Status & UX (`status.ts` + renderer) -- `checkCodeModeAgentStatus()` becomes: engine = provisioned? (instead of "installed on - PATH?"); auth = existing credentials present? (unchanged logic). -- First-run flow: user picks agent → if engine missing, show "Downloading engine - (~200 MB), one time…" with progress; on completion, proceed. Subsequent uses: instant. -- Clear error states: download failed / offline / unsupported platform / auth missing. - -## 6. File-by-file change plan - -| File | Change | -|---|---| -| `apps/main/forge.config.cjs` | Stage adapter JS closure into `.package/acp/node_modules`; exempt `.package` from node_modules ignore; generate + copy `engine-manifest.json`. (No native engines staged.) | -| `apps/main/scripts/gen-engine-manifest.mjs` *(new)* | Build-time: read lockfile/packuments → emit `engine-manifest.json` (versions, tarball URLs, integrity, bin paths). | -| `packages/core/src/code-mode/acp/engine-provisioner.ts` *(new)* | `ensureEngine()` — download, verify, extract, lock, progress, `.meta` ledger. | -| `packages/core/src/code-mode/acp/agents.ts` | Resolve adapter from staged `.package/acp` first (fallback to node_modules in dev); `await ensureEngine`; set `CLAUDE_CODE_EXECUTABLE`/`CODEX_PATH` to provisioned binaries. | -| `packages/core/src/code-mode/acp/client.ts` / `manager.ts` | Await provisioning before spawn; surface provisioning progress/errors; keep startup deadline. | -| `packages/core/src/code-mode/status.ts` | Engine status = provisioned (not PATH); keep auth checks. | -| `apps/main/src/ipc.ts` + preload + renderer | IPC for provisioning progress + first-run download UI + error states. | -| CI (optional) | Smoke: packaged app boots each adapter, provisions a (small fake) engine, sets env var, answers ACP `initialize`; offline → clear error not a hang. | - -## 7. Edge cases & risks - -- **R1 — Codex needs `rg`.** The codex platform package bundles ripgrep at - `vendor//path/rg`. We must extract/keep the vendor layout so codex finds it - (don't extract the bare binary). Verify codex `app-server` boots from the provisioned dir. -- **R2 — Codex pnpm patch. RESOLVED (irrelevant).** The patch targets the JS launcher; we - point `CODEX_PATH` at the native binary, so it does not apply. -- **R3 — Platform/arch matrix.** Manifest must cover darwin x64/arm64, linux x64/arm64 - (+musl for claude), win32 x64/arm64. Windows engine is `claude.exe`/`codex.exe`. -- **R4 — Integrity & supply chain.** Always verify the manifest integrity hash before - chmod/exec. Treat a hash mismatch as a hard failure. -- **R5 — Disk + upgrades.** Versioned dirs accumulate. Add a cleanup that keeps only the - active pinned version per agent. -- **R6 — First-run network.** Required once per agent; cached forever after. Must be a - clear, cancellable UX, never a silent hang (reuse the #614 startup-deadline lesson). -- **R7 — code signing / Gatekeeper (macOS).** Downloaded native binaries aren't covered by - our app's signature. Verify they run under Gatekeeper (they're already - signed/notarized by Anthropic/OpenAI; quarantine attr may need clearing). Conductor runs - them fine from app-support — confirm we do too. -- **R8 — `engine-manifest` staleness.** Manifest must regenerate whenever the adapter/engine - versions change; tie generation into the build so it can't drift. - -## 8. Phasing - -1. **P1 — Packaging fix (unblocks dev→packaged parity):** stage adapter JS into `.package`, - resolver checks staged path first. Verify packaged code mode can at least *spawn* the - adapter. (Independent of provisioning; the part of #614 worth keeping.) -2. **P2 — Provisioner (core):** `ensureEngine()` + manifest + wire env vars. Verify both - engines provision + boot from `~/.rowboat/engines` on macOS arm64. -3. **P3 — UX + status:** first-run download UI, status panel, error states. -4. **P4 — Cross-platform + CI smoke:** matrix manifest, mac/linux/win smoke. -5. **P5 — Polish:** version cleanup, cancellation, offline messaging, optional local-install - fallback. - -## 9. Decisions & remaining questions - -### Resolved (locked) -1. **Source = npm platform packages, adapter-pinned versions** (not self-host, not curl - installer, not release bucket). -2. **Always provision; no local-install fallback.** -3. **Codex pnpm patch irrelevant.** - -### Still open (can decide during implementation) -- **Provision timing:** on first code-mode *use* (lazy) vs first app launch (eager, - background download). Plan assumes **lazy + cached**. -- **Version-dir cleanup:** keep only the active pinned version per agent (R5). -- **Verify R1** (codex `rg`) and **R7** (Gatekeeper on downloaded macOS binaries) during P2. diff --git a/apps/x/GRANOLA_PARITY.md b/apps/x/GRANOLA_PARITY.md deleted file mode 100644 index 53af3bd9..00000000 --- a/apps/x/GRANOLA_PARITY.md +++ /dev/null @@ -1,153 +0,0 @@ -# Granola Parity — Research & Gap Analysis - -Goal: make Rowboat's meeting feature work exactly like Granola (granola.ai). We copy Granola's behavior — no new invention. This doc is the source of truth: how Granola works, what Rowboat has today (with file:line pointers), the gaps, and the parity plan. - -Research basis: Granola's official docs (docs.granola.ai), granola.ai blog/security/jobs pages, third-party reviews and reverse-engineering writeups, plus a full audit of this codebase. Claims that are inferred (not directly documented by Granola) are marked **[inference]**. - ---- - -## Part 1 — How Granola works - -Granola is an Electron app (confirmed by their own job postings: "Granola is an Electron app with deep OS integrations", React/TypeScript UI + native OS helpers). No bot ever joins the call — it captures audio locally on the device, so nothing is visible to other meeting participants. - -### 1.1 Always-running app (launch at login) - -- The product model is an **always-running, menu-bar-resident app**. Granola must be running for detection/notifications to work ("You must have Granola open... for transcription to run" — their troubleshooting docs). -- A literal "open at login" toggle is **not publicly documented**. **[inference]** A clone should register as a login item (Electron `app.setLoginItemSettings`), default on, with a setting to turn it off — the entire detection value prop collapses if the app isn't running. -- The app keeps running when the window closes (menu bar presence remains). - -### 1.2 Menu bar icon - -- Granola "sits in your Mac's menu bar". Documented affordance: **click the menu bar icon to start recording / open the app**. -- Recording status is NOT primarily shown in the menu bar. The documented indicators are: - - Inside the note: "green dancing bars" at the bottom while capturing. - - When another app has focus: a **floating always-on-top "live meeting" pill** on the right side of the screen. -- Exact dropdown contents (upcoming meetings list, icon state change while recording) are not documented. The "Coming up" list lives on the home screen, not the tray. - -### 1.3 Meeting detection — two signals, never auto-record - -Granola detects meetings via **calendar events + microphone-in-use detection**, and it **never records without a user action** ("Granola only starts transcribing when you open a note for that meeting"). - -**Signal A — Calendar:** -- Google / Microsoft calendar sync. Pre-creates an auto-titled note per event with 2+ attendees; filters declined events, OOO, Focus Time. -- **In-app popup (Granola-drawn, not macOS Notification Center) 1 minute before** any scheduled call with 2+ attendees. One click on it **opens the meeting URL AND starts transcription** at once. -- **Armed auto-start:** if you have the upcoming meeting's note open before it starts, recording starts automatically at the scheduled time. - -**Signal B — Mic-in-use (ad-hoc calls, incl. browser meetings):** -- "Granola notices when your microphone is in use and offers to start taking notes." -- It's app-aware: notification title is "**Huddle detected**" (Slack), "**Call detected**" (FaceTime/WhatsApp), "**Meeting detected**" (anything else, including browsers). Buttons: "Take Notes". Ad-hoc popups have a dashed left border; calendar ones a solid colored bar. -- If the ad-hoc call starts within **15 minutes** of a scheduled event, the popup adopts that event's name (merged). -- **[inference]** Mechanism: poll CoreAudio `kAudioDevicePropertyDeviceIsRunningSomewhere` on input devices for "mic in use", plus enumerate running processes against a known meeting-app list to get the app-specific title. There's no public macOS API for "process X is using the mic", so it's a heuristic combo. Browser meetings degrade to the generic "Meeting detected" title; calendar linkage supplies identity. -- Notifications are configurable in Settings (off entirely, or per-application). -- Related preference: "**Reposition Granola for meetings**" — window auto-repositions/resizes alongside the call when a meeting starts (toggleable, default on). - -### 1.4 Audio capture - -- Capture = **default microphone + system audio output**, at the OS level. Works with any app or browser (Chrome/Safari/Firefox/Edge) with no extension. -- Cannot isolate per-app audio — it's the combined system stream; uses the OS default sound devices. -- The two streams stay separate through transcription → transcript UI shows **grey bubbles (left) = system audio ("Them"), green bubbles (right) = your mic ("Me")**. No true live diarization on desktop — just Me/Them. -- **macOS permissions: exactly two** — Microphone, and Screen & System Audio Recording (macOS bundles system-audio capture under screen recording). No Accessibility permission. -- **[inference]** System audio mechanism: ScreenCaptureKit audio loopback (macOS 13+ baseline, they require 13+ / recommend 14.2+), possibly CoreAudio process taps on 14.4+. No virtual audio driver (setup has no driver install step). -- **Audio is never stored** — streamed to the ASR provider in real time and discarded. - -### 1.5 Transcription - -- Cloud, real-time streaming ASR. Subprocessors named on their security page: **Deepgram and AssemblyAI** for ASR; **OpenAI and Anthropic** for note enhancement. -- Live transcript accrues during the call, hidden behind a waveform-icon toggle in the note. - -### 1.6 Meeting end & post-meeting flow - -- **Auto-stop conditions** (documented): (a) call-end heuristic — transcript inactivity + whether the call software is still in use + scheduled end time for calendar-linked meetings; (b) **15 minutes of silence**; (c) computer sleeps; (d) manual stop. Note: "on macOS, meeting auto-end detection requires admin rights" — without it, manual stop only. -- **On stop, enhancement runs automatically** — no click needed. Enhanced notes are ready in seconds (~30s max reported). -- A "**notes ready**" notification fires when enhancement completes — that's the re-entry point back into the app if you switched away. The note itself was already open (recording ran inside it), so the enhanced version replaces the raw view in place. There's no documented "force-focus the window" behavior; the notification does the redirecting. -- Extras: auto-drafted follow-up emails (toggleable), pre-meeting briefs. - -### 1.7 Notes UI - -**During the meeting:** a plain notepad — the user types rough bullets; transcription runs invisibly. Green dancing bars at the bottom = capturing. Waveform button (next to the per-note "Ask anything" chat bar) opens the live transcript side-by-side. Mid-meeting you can ask the chat to catch you up. - -**After the meeting (enhanced notes):** -- Enhancement merges exactly three inputs: **transcript + your raw notes + calendar metadata**, through a template. -- Signature affordance: **your words render black, AI-generated text grey**. -- Per-line provenance: magnifying-glass icon on a line reveals where it came from in the transcript. -- Templates: built-ins (1:1, standup, sales discovery) + custom; re-enhance with a different template (✨), regenerate (🔁), edit raw notes and re-enhance. -- Chat: per-note "Ask anything" bar, global chat (⌘J), cross-meeting chat over folders; edit-by-asking. -- Home screen: "**Coming up**" strip (next ~5 meetings) + reverse-chronological past notes list; folders; recurring meetings grouped by recurring-event ID + title; shareable links. - ---- - -## Part 2 — What Rowboat has today (audit of main) - -Two separate feature families exist in `apps/x`; only the second is Granola-adjacent: -- **"Calls" (video mode)** — live voice/video chat *with the Rowboat AI* (`VIDEO_MODE.md`). Not meeting capture. -- **"Meetings"** — real meeting capture → transcript → AI notes. Working pipeline, but **calendar-triggered and manual-click only**. - -### What EXISTS (and is solid) - -| Area | Status | Where | -|---|---|---| -| Mic + system-audio capture, 2-channel | ✅ | `apps/renderer/src/hooks/useMeetingTranscription.ts:282-299` — mic `getUserMedia` (ch 0) + `getDisplayMedia({audio})` loopback (ch 1), merged to 16 kHz PCM (`:424-485`); loopback auto-approved in `apps/main/src/main.ts:216-226` | -| Realtime ASR | ✅ | Deepgram realtime WS, `nova-3`, multichannel + diarize (`useMeetingTranscription.ts:9-21`); proxy or raw key (`:253-280`) | -| Speaker labels | ✅ | ch 0 → "You", ch 1 → diarized `Speaker N` (`:335-347`) | -| Transcript storage | ✅ | Markdown + frontmatter + fenced transcript block, `knowledge/Meetings/rowboat//.md` (`:93-136`, `:487-510`) | -| AI meeting notes on stop | ✅ | `packages/core/src/knowledge/summarize_meeting.ts` — LLM summary, attendee-name resolution from calendar; orchestrated in `App.tsx:5601-5661`; notes prepended above transcript | -| Auto-stop heuristics | ✅ (partial) | silence RMS backstop, calendar-end gating, system-track ended/muted — `useMeetingTranscription.ts:378-549` | -| Calendar sync | ✅ | Google OAuth via `googleapis`, `packages/core/src/knowledge/sync_calendar.ts`, per-event JSON in `~/.rowboat/calendar_sync/` | -| Pre-meeting notification | ✅ (system toast) | `packages/core/src/knowledge/notify_calendar_meetings.ts` — polls every 30s, notifies ~1 min before, deep-links `rowboat://action?type=join-and-take-meeting-notes` | -| Meetings screen | ✅ | `apps/renderer/src/components/meetings-view.tsx` — "Coming up" (+Join / Take-notes buttons, inline prep) + past-notes table | -| Transcript rendering | ✅ | `apps/renderer/src/extensions/transcript-block.tsx` (TipTap, colored speakers, collapsible) | -| Meeting prep briefs | ✅ | `meeting_prep_scheduler.ts`, `meeting_prep_brief.ts` (Granola has this too) | -| Permissions flows | ✅ | mic `ipc.ts:2090-2106`; screen recording check/open-settings `ipc.ts:2003-2018`; Info.plist strings `forge.config.cjs:201-204`; entitlements OK | - -### What is MISSING (the entire "ambient" layer) - -| # | Gap | Detail | -|---|---|---| -| 1 | **Launch at login** | Zero `setLoginItemSettings` / auto-launch code anywhere. | -| 2 | **Menu bar (tray) icon** | Zero `Tray` usage. No background presence: on macOS closing the window leaves only the Dock; no way to start recording without the full window. | -| 3 | **Meeting detection** | No mic-in-use detection, no process detection (Zoom/Teams/Slack/FaceTime), no browser awareness. Only calendar (time-based). Ad-hoc calls are invisible. | -| 4 | **Auto/one-click start UX** | Capture starts only via explicit button click or clicking the calendar toast. No "Meeting detected — Take notes?" popup, no armed auto-start when a note is open at meeting start time. | -| 5 | **Headless capture** | Capture depends on the renderer window holding a live `getDisplayMedia` stream. No native audio helper → can't capture while the app is "in the background" the way Granola does. | -| 6 | **Meeting-end redirect** | On stop, the note refreshes in place, but there's **no "notes ready" notification** and no bring-the-user-back moment if they're in another app. | -| 7 | **Notes UI polish (Granola signatures)** | No black-vs-grey authorship distinction (we replace, they merge raw notes + transcript), no during-meeting notepad-first flow (we show the transcript note), no per-line provenance, no templates/re-enhance, no floating "recording" pill. | - -### Honest assessment - -Rowboat has built the *second half* of Granola well — what happens once you're recording, and after the meeting ends. It has essentially none of the *first half* — noticing a meeting is happening and quietly being there without the user opening the app. That first half is exactly items 1–5 above, and it's where all the work is. - ---- - -## Part 3 — Parity plan (copy Granola, no new stuff) - -Ordered so each phase is shippable and testable on its own. - -### Phase 1 — Resident app: login item + tray -- `app.setLoginItemSettings({ openAtLogin: true })`, default on, toggle in Settings. When launched at login: no window, tray only **[inference — Granola undocumented, but implied]**. -- Electron `Tray` with template icon; menu: "Start recording", "Open Rowboat", recording status line, Quit. Keep app alive on window close (already macOS default; add tray so it's reachable). -- Acceptance: reboot → icon in menu bar, no window; click tray → start an ad-hoc meeting note. - -### Phase 2 — Meeting detection + "Take notes?" popup -- Native signal (small Swift helper or node addon, polled from main): mic-in-use via CoreAudio `kAudioDevicePropertyDeviceIsRunningSomewhere` + running-process match against a known list (zoom.us, Teams, Slack, FaceTime, browsers…). -- Granola-style app-drawn popup (small always-on-top BrowserWindow, like the existing video popout): "Huddle detected / Call detected / Meeting detected — [Take Notes]". Merge with a calendar event if within 15 min. Never auto-record. -- Upgrade the existing 1-min-before calendar toast to the same popup style; one click = open meeting URL + start capture (deep-link plumbing already exists in `deeplink.ts`). -- Armed auto-start: meeting note open before start time → auto-start at start time. -- Per-app notification settings. -- Acceptance: start a Meet call in Chrome with no calendar event → popup appears; click → recording. - -### Phase 3 — Meeting end → notes ready redirect -- Keep existing auto-stop heuristics; add the missing "call software no longer active" signal from the Phase-2 process watcher; keep 15-min silence backstop (ours is stricter — align to Granola's 15 min). -- On summary completion: fire a **"Your meeting notes are ready" notification**; clicking focuses the app on the note (deep link exists). This is the "redirect when we cut the call" the feature needs. -- Acceptance: leave the call → recording stops on its own → notification within ~30s → click lands on finished notes. - -### Phase 4 — Notes UI parity -- During meeting: notepad-first note (user types; transcript behind a waveform toggle); green capture indicator in-note; floating "live meeting" pill when app unfocused (reuse the video-popout window machinery). -- After meeting: enhancement merges **raw notes + transcript + calendar event** (today we only use the transcript); render user text black / AI text grey; ✨ re-enhance + 🔁 regenerate; keep transcript below as today. -- Home: Meetings view already ≈ Granola's home (Coming up + past list) — minor polish only. - -### Phase 5 (only if needed for true parity) — Headless capture -- Native mic + ScreenCaptureKit system-audio capture in main/helper process so recording doesn't require the renderer window. Biggest lift; Phases 1–4 deliver the Granola experience with the window opening on record start, which is acceptable Granola-like behavior (their note opens on start too). - -### Key implementation notes -- Permissions stay exactly two (mic + screen-recording) — we already handle both. -- Deepgram nova-3 multichannel already matches Granola's Me/Them model — no ASR change needed. -- Reuse: popup ← video popout window pattern; deep links ← `deeplink.ts`; detection loop ← same main-process init pattern as `notify_calendar_meetings.ts`. diff --git a/apps/x/LIVE_NOTE.md b/apps/x/LIVE_NOTE.md index 85f16786..2fc43786 100644 --- a/apps/x/LIVE_NOTE.md +++ b/apps/x/LIVE_NOTE.md @@ -70,7 +70,7 @@ The `once` trigger from the prior model has been **dropped** — it didn't fit t Two paths, both producing identical on-disk YAML: 1. **Hand-written** — type the `live:` block directly into the note's frontmatter. The scheduler picks it up on its next 15-second tick. -2. **Sidebar chat** — mention a note (or have it attached) and ask Copilot for something dynamic. Copilot is tuned to recognize a wide range of phrasings beyond "live" or "track" (see "Prompts Catalog → Copilot trigger paragraph"); it loads the `live-note` skill, edits the frontmatter via `file-editText`, then **runs the agent once by default** so the user immediately sees content. The auto-run is skipped only when the user explicitly says not to run yet. +2. **Sidebar chat** — mention a note (or have it attached) and ask Copilot for something dynamic. Copilot is tuned to recognize a wide range of phrasings beyond "live" or "track" (see "Prompts Catalog → Copilot trigger paragraph"); it loads the `live-note` skill, edits the frontmatter via `workspace-edit`, then **runs the agent once by default** so the user immediately sees content. The auto-run is skipped only when the user explicitly says not to run yet. When the note is **already live** and the user asks to track something new, Copilot extends the existing `live.objective` in natural-language prose. It does not create a second `live:` block. @@ -92,8 +92,8 @@ When a trigger fires, the live-note agent receives a short message: - For event runs only: the matching `eventMatchCriteria` text and the event payload, with a Pass-2 decision directive ("only edit if the event genuinely warrants it"). The agent's system prompt tells it to: -1. Call `file-readText` to read the current note (the body may be long; no body snapshot is passed in the message — fetch fresh). -2. Make small, **patch-style** edits with `file-editText` — change one region, re-read, change the next region — rather than one-shot rewrites. +1. Call `workspace-readFile` to read the current note (the body may be long; no body snapshot is passed in the message — fetch fresh). +2. Make small, **patch-style** edits with `workspace-edit` — change one region, re-read, change the next region — rather than one-shot rewrites. 3. Follow default body structure unless the objective overrides: H1 stays the title; a 1-3 sentence rolling summary at the top; H2 sub-topic sections below, freshest first. 4. Never modify YAML frontmatter — that's owned by the user and the runtime. 5. End with a 1-2 sentence summary stored as `lastRunSummary`. @@ -115,7 +115,7 @@ Backend (main process) ├─ Event processor (5 s) ──┼──► runLiveNoteAgent() ──► live-note-agent └─ Builtin tool │ │ run-live-note-agent ────┘ ▼ - file-readText / -edit + workspace-readFile / -edit │ ▼ body region(s) rewritten on disk @@ -175,7 +175,7 @@ Internal trigger enum (`LiveNoteTriggerType`) is `'manual' | 'cron' | 'window' | `buildMessage` always emits a `**Trigger:**` paragraph in the agent's run message — one paragraph per kind. `manual` and the two timed variants (`cron`, `window`) include any optional `context` as a `**Context:**` block. `event` includes the eventMatchCriteria + payload + Pass 2 decision directive (no `**Context:**`; the payload *is* the context). -This lets the user-authored objective branch on trigger kind when warranted (for example, an email digest can scan `gmail_sync/` from scratch on cron/window runs, while event runs integrate just the new thread). The skill teaches the pattern under "Per-trigger guidance (advanced)". +This lets the user-authored objective branch on trigger kind when warranted (the canonical example is the Today.md emails section: cron/window scans `gmail_sync/` from scratch, event integrates the new thread). The skill teaches the pattern under "Per-trigger guidance (advanced)". ### Run flow (`runLiveNoteAgent`) @@ -249,20 +249,22 @@ The contract (defined in the run-agent system prompt — `packages/core/src/know - Then content organized by sub-topic under H2 headings, freshest/most-important first. - Tightness over decoration. - **Override** — if the objective specifies a different layout (e.g. "show the top 5 stories at the top, with a one-paragraph summary above them"), follow that exactly. -- **Patch-style updates** — make small, incremental `file-editText` calls (read → edit one region → re-read → next), not one-shot whole-body rewrites. This preserves user-added content the agent didn't account for and keeps diffs reviewable. +- **Patch-style updates** — make small, incremental `workspace-edit` calls (read → edit one region → re-read → next), not one-shot whole-body rewrites. This preserves user-added content the agent didn't account for and keeps diffs reviewable. - **Boundaries**: never modify the frontmatter; the agent is the sole writer of the body below the H1. --- -## Default Note Policy +## Daily-Note Template & Migrations -Rowboat no longer creates a default `Today.md` live dashboard for new users. Live notes are user-created notes with an explicit `live:` frontmatter block. +`Today.md` is the canonical demo of what a live note can do. It ships with one objective covering an Overview / Calendar / Emails / What you missed / Priorities layout — driven by three windows and an event-match criterion for in-day signals. -**Deprecated Today.md migration** — `packages/core/src/knowledge/deprecate_today_note.ts` runs once per workspace on app start: +**Versioning** — `packages/core/src/knowledge/ensure_daily_note.ts` carries a `CANONICAL_DAILY_NOTE_VERSION` constant and a `templateVersion` scalar in the frontmatter. On app start, `ensureDailyNote()`: -- File missing → mark processed and do nothing. -- File present → set `live.active: false` if a `live:` block exists, prepend a user-facing deprecation notice once, and preserve the note body. -- Future launches → no-op via `config/today-note-deprecation.json`, so a user who re-enables the note is not paused again. +- File missing → fresh write at canonical version. +- File at-or-above canonical → no-op. +- File below canonical → rename existing to `Today.md.bkp.` (which doesn't end in `.md`, so the scheduler/event router skip it), then write the canonical template body-from-scratch (live notes regenerate their own body). + +The bump from v1 (the old `track:` array model) to v2 (the live-note rewrite) is handled by this same path. Pre-v2 notes get backed up and replaced. --- @@ -316,7 +318,7 @@ Every LLM-facing prompt in the feature, with file pointers. After any edit: `cd - **Purpose**: the user message seeded into each agent run. - **File**: `packages/core/src/knowledge/live-note/runner.ts` (`buildMessage`). - **Inputs**: `filePath` (presented as `knowledge/${filePath}` in the message), `live.objective`, `live.triggers?.eventMatchCriteria` (only on event runs), `trigger`, optional `context`, plus `localNow` / `tz`. -- **Behavior**: tells the agent to call `file-readText` itself (no body snapshot included, since the body can be long and may have been edited by a concurrent run) and to make patch-style edits. +- **Behavior**: tells the agent to call `workspace-readFile` itself (no body snapshot included, since the body can be long and may have been edited by a concurrent run) and to make patch-style edits. Three branches by `trigger`: - **`manual`** — base message. If `context` is passed, it's appended as a `**Context:**` section. The `run-live-note-agent` tool uses this path for both plain refreshes and context-biased backfills. @@ -326,7 +328,7 @@ Three branches by `trigger`: ### 5. Live Note skill (Copilot-facing) - **Purpose**: teaches Copilot the `live:` model — operational posture (act-first), the strong/medium/anti-signal taxonomy and how to act on each, the **always-extend-not-fork** rule for already-live notes, user-facing language (call them "live notes"; surface the **Live Note panel** by name), the auto-run-once-on-create/edit default, schema, triggers, YAML-safety rules, insertion workflow, and the `run-live-note-agent` tool with `context` backfills. -- **File**: `packages/core/src/runtime/assembly/skills/live-note/skill.ts`. Exported `skill` constant. +- **File**: `packages/core/src/application/assistant/skills/live-note/skill.ts`. Exported `skill` constant. - **Schema interpolation**: at module load, `stringifyYaml(z.toJSONSchema(LiveNoteSchema))` is interpolated into the "Canonical Schema" section. Edits to `LiveNoteSchema` propagate automatically. - **Output**: markdown, injected into the Copilot system prompt when `loadSkill('live-note')` fires. - **Invoked by**: Copilot's `loadSkill` builtin tool. Registration in `skills/index.ts`. @@ -334,7 +336,7 @@ Three branches by `trigger`: ### 6. Copilot trigger paragraph - **Purpose**: tells Copilot *when* to load the `live-note` skill, and frames how aggressively to act once loaded. -- **File**: `packages/core/src/runtime/assembly/copilot/instructions.ts` (look for the "Live Notes" paragraph). +- **File**: `packages/core/src/application/assistant/instructions.ts` (look for the "Live Notes" paragraph). - **Strong signals (load + act without asking)**: cadence words ("every morning / daily / hourly…"), living-document verbs ("keep a running summary of…", "maintain a digest of…"), watch/monitor verbs, pin-live framings ("always show the latest X here"), direct ("track / follow X"), event-conditional ("whenever a relevant email comes in…"). - **Medium signals (load + answer the one-off + offer)**: time-decaying questions ("what's the weather?", "USD/INR right now?", "service X status?"), note-anchored snapshots ("show me my schedule here"), recurring artifacts ("morning briefing", "weekly review", "Acme dashboard"), topic-following / catch-up. - **Anti-signals (do NOT make live)**: definitional questions, one-off lookups, manual document editing. @@ -343,7 +345,7 @@ Three branches by `trigger`: ### 7. `run-live-note-agent` tool — `context` parameter description - **Purpose**: a mini-prompt (a Zod `.describe()`) that guides Copilot on when to pass extra context for a run. -- **File**: `packages/core/src/runtime/tools/catalog.ts` (the `run-live-note-agent` tool definition). +- **File**: `packages/core/src/application/lib/builtin-tools.ts` (the `run-live-note-agent` tool definition). - **Inputs**: `filePath` (workspace-relative; the tool strips the `knowledge/` prefix internally), optional `context`. - **Output**: flows into `runLiveNoteAgent(..., 'manual')` → `buildMessage` → appended as `**Context:**` in the agent message. - **Key use case**: backfill a newly-made-live note so its body isn't empty on day 1. @@ -391,13 +393,13 @@ Conventions: | Run orchestrator (`runLiveNoteAgent`, `buildMessage`) | `packages/core/src/knowledge/live-note/runner.ts` | | Live-note agent definition (`LIVE_NOTE_AGENT_INSTRUCTIONS`, `buildLiveNoteAgent`) | `packages/core/src/knowledge/live-note/agent.ts` | | Live-note bus (pub-sub for lifecycle events) | `packages/core/src/knowledge/live-note/bus.ts` | -| Deprecated Today.md one-time migration | `packages/core/src/knowledge/deprecate_today_note.ts` | +| Daily-note template + version migration | `packages/core/src/knowledge/ensure_daily_note.ts` | | Gmail event producer | `packages/core/src/knowledge/sync_gmail.ts` | | Calendar event producer + digest | `packages/core/src/knowledge/sync_calendar.ts` | -| Copilot skill | `packages/core/src/runtime/assembly/skills/live-note/skill.ts` | -| Skill registration | `packages/core/src/runtime/assembly/skills/index.ts` | -| Copilot trigger paragraph | `packages/core/src/runtime/assembly/copilot/instructions.ts` | -| `run-live-note-agent` builtin tool | `packages/core/src/runtime/tools/catalog.ts` | +| Copilot skill | `packages/core/src/application/assistant/skills/live-note/skill.ts` | +| Skill registration | `packages/core/src/application/assistant/skills/index.ts` | +| Copilot trigger paragraph | `packages/core/src/application/assistant/instructions.ts` | +| `run-live-note-agent` builtin tool | `packages/core/src/application/lib/builtin-tools.ts` | | Editor toolbar (Radio button → panel) | `apps/renderer/src/components/editor-toolbar.tsx` | | Live Note panel (single-view editor) | `apps/renderer/src/components/live-note-sidebar.tsx` | | Status hook (`useLiveNoteAgentStatus`) | `apps/renderer/src/hooks/use-live-note-agent-status.ts` | diff --git a/apps/x/MINI_APPS_PLAN.md b/apps/x/MINI_APPS_PLAN.md deleted file mode 100644 index 72a82a29..00000000 --- a/apps/x/MINI_APPS_PLAN.md +++ /dev/null @@ -1,307 +0,0 @@ -# Mini Apps — Implementation Plan - -> Status: **Phase 1 (UI + rendering, hand-coded apps)** in progress. -> Branch: `feature/mini-apps`. Working dir: `apps/x`. - -## 1. What we're building (agreed design) - -A **Mini App** is a user-created, personal app that lives inside Rowboat under a -**"Mini Apps"** sidebar tab, shown as cards; click a card to open and use it like -a standalone app. - -The settled mental model: - -> A Mini App = **custom UI code** (a single self-contained HTML file: React + -> Tailwind, no build step, runs in a sandboxed iframe) + an **agent "backend"** -> that produces fresh **data** on a trigger + an optional **state store** + a -> **scoped bridge** that lets the UI call real **Composio** actions. - -Key decisions (the "why" behind the architecture): - -- **Opens to a finished, populated screen.** The agent runs *before* open (on a - trigger); the UI just renders the latest result. Maps onto the existing - background-agents engine. -- **Frontend/backend split.** Code = frontend, written once. Agent = backend, - produces fresh `data.json` each run. The UI is *not* regenerated per run. - Cheaper, faster, and the UI can't break on refresh. -- **A data contract** ties the three pieces together: at creation the builder - produces a matched triple — UI code + data schema + agent instructions. -- **Fully custom UI per app** (generated code), not a fixed block kit. -- **Real interactivity** via a **scoped bridge** to Composio (each app declares - the integrations it may touch; scope drives both enforcement and the auth - prompt). -- **Built by whatever engine is selected in the chat box** — Code Mode - (Claude Code / Codex) if the user has it, else the Copilot. -- **Living apps:** refined by chatting; breakage is first-class (app surfaces - errors → "fix with copilot"). -- **State is an optional provided primitive** (per-app persistent store the agent - and UI can read/write). Stateless apps ignore it. -- **Personal/local only** for now, but each app is a **self-contained folder** - so it can become portable later. - -### Framework for the apps - -Single self-contained `index.html` rendered in a sandboxed iframe. - -> **Learning (Phase 1):** remote CDNs do NOT work inside the sandboxed, -> opaque-origin `srcdoc` iframe (React/Tailwind/Babel from unpkg/cdn.tailwindcss -> failed → blank screen). Apps must be **fully self-contained, no network**. -> Phase 1 therefore ships **vanilla HTML + CSS + JS** (no build, no transpile). -> The React+Tailwind LLM-friendly target still stands for generation — but via a -> **locally-bundled** runtime (esbuild-at-save, libs injected into the HTML), -> never remote CDNs. The `window.rowboat` bridge is unchanged either way. - -The UI talks to the host through a `window.rowboat` **bridge** (the product -surface *and* the security boundary): - -``` -rowboat.getData(): Promise // latest agent output -rowboat.onData(cb): void // re-render on refresh -rowboat.getState() / setState(patch) // per-app persistent store -rowboat.callAction(scope, action, args) // scoped Composio call -rowboat.ready() // handshake: "send me data" -``` - -## 2. Phased roadmap - -1. **Surface + one real app (UI-first — THIS PHASE).** "Mini Apps" sidebar tab, - card grid, click to open. One **hardcoded** app (Twitter client) rendered in a - sandboxed iframe from a self-contained HTML file, reading static data through a - minimal bridge. Proves the *experience* and the *rendering path*. No agent, - no storage, no real Composio yet. -2. **The scoped bridge.** Real interactive buttons → real Composio actions, - enforced against a per-app manifest scope; auth prompt on demand. -3. **Generation.** Copilot/Code-Mode generates the UI+schema+agent triple from a - chat description. Rides the existing chat mode selector + `code-with-agents`. -4. **Living apps + breakage recovery.** Conversational edits; error surfacing. -5. **Scale & polish.** Context-overload windowing (e.g. 100 tweets); app - notifications (reuse `notify-user`). -6. **(V2)** Drag an app into the main workspace (macOS-style). - -## 2b. Phase 2.5 — On-disk apps + `app://miniapp` serving (CURRENT) - -Move built-in apps out of source into `~/.rowboat/apps//`, served as static -assets, with an optional background agent producing `data.json`. Sets up the -copilot builder path (it will write the same folder layout). Team-agreed. - -### On-disk layout (one folder per app) -``` -~/.rowboat/apps// - manifest.json # id, title, description, source, scope[], active, lastRun, - # entry (default "dist/index.html"), agent (optional bg-task slug) - dist/ # static assets served via app://miniapp//... - index.html # the app (self-contained; bridge shim inlined) - data.json # latest agent output (read by the host, pushed to the app) -``` - -### Serving — `app://miniapp//` (option A) -- Extend `registerAppProtocol` (`apps/main/src/main.ts`) with a new host - `miniapp`: map `app://miniapp//` → `~/.rowboat/apps//dist/` - (path-traversal guarded; default `index.html`). `app://` is already registered - privileged (standard/secure/fetch/cors) so apps get a **real origin** — - remote CDNs/images and `fetch` work, unlike the opaque `srcdoc` origin. -- The iframe loads via `src="app://miniapp//index.html"` (not `srcDoc`). - Sandbox becomes `allow-scripts allow-same-origin allow-popups allow-forms - allow-modals allow-downloads` — same-origin is its OWN `app://miniapp` origin, - still isolated from the renderer (different host). - -### Data path -- Built-in/static `data` is seeded to `data.json`. A future background agent - (reuse bg-tasks engine, linked via manifest `agent`) overwrites it on schedule. -- App keeps using `rowboat.onData`; the **host** now sources that data by reading - `~/.rowboat/apps//data.json` (via IPC) and posting it on `ready`. Composio - still flows through the bridge RPC. (GitHub app needs no `data.json` — it pulls - live via Composio.) - -### Steps -1. **Shared schema** — `packages/shared/src/mini-app.ts`: `MiniAppManifest` zod + - type. Export it. New IPC channels in `shared/src/ipc.ts`: - - `mini-apps:seed` (req: `{apps:[{manifest, html, data?}]}`) — idempotent. - - `mini-apps:list` (res: `{manifests: MiniAppManifest[]}`). - - `mini-apps:get-data` (req `{id}`, res `{data: unknown|null}`). -2. **Main** — `apps/main/src/mini-apps-handler.ts`: `seedApps` (write - manifest/dist/data to `~/.rowboat/apps//` only if absent), `listApps` - (read manifests), `getAppData` (read data.json). Register handlers in - `ipc.ts`. Extend `registerAppProtocol` for the `miniapp` host. -3. **Renderer** — keep `MiniApp` defs + `buildMiniAppHtml`; `registry.ts` adds - `toSeed(app)` (→ `{manifest, html, data}`). `MiniAppsView`: on mount → `seed` - → `list` → render cards from manifests. `MiniAppFrame`: take the manifest, - load `src=app://miniapp//`, on `ready` fetch `mini-apps:get-data` - and post it; RPC scope from `manifest.scope`. -4. **Verify**: GitHub app end-to-end from disk (`~/.rowboat/apps/github-radar/`), - then the others; confirm connect + live PRs still work. - -### Out of scope here (next: copilot builder, tomorrow) -Copilot writing a new app folder + an associated background agent; the agent -browsing via embedded browser for social feeds; copilot verifying Composio wiring -by actually calling tools before finalizing. - -## 2c. Remaining work (after on-disk move) - -Done so far: surface + runtime (Phase 1), real scoped Composio bridge (Phase 2), -apps on disk served via `app://miniapp` (Phase 2.5). - -**Next — Copilot builder (demo target).** -- Copilot creates an app folder in `~/.rowboat/apps//` via the `mini-apps:seed` - install primitive (writes `manifest.json` + `dist/index.html`). -- Copilot must **verify wiring by actually calling Composio tools** and inspecting - the returned data before finalizing — never speculate the shape. -- Give generated apps the bridge contract — prefer **serving a canonical shim** - from the protocol (e.g. `app://miniapp/__bridge__.js`) over inlining. - -**Background-agent data pipeline (deterministic).** -- Reuse the existing bg-tasks engine; link via `manifest.agent`. -- The agent does NOT write files. It **returns structured data validated against - the app's data schema**; the bg-task **runner (code) atomically writes the - app's `data.json`** (temp→rename, keep last-good on failure). Path / atomicity - / validation / fallback all live in code — only the *content* is LLM-driven. -- Social feeds (Twitter/LinkedIn/Reddit): the agent browses via the embedded - browser, curates, returns data → runner writes `data.json`. - -**Later (roadmap).** -- Living apps + breakage recovery (edit by chat; surface errors → fix-with-copilot). -- Scale/polish: context-overload windowing; app notifications; design-consistency - / component templates (team-deferred). -- V2: drag an app into the main workspace. - -## 2d. Mini App builder skill — spec - -A Copilot skill (`build-mini-app`, in `packages/core/src/application/assistant/ -skills/`) that turns a chat request into an installed app under `~/.rowboat/ -apps//`. Copilot orchestrates; the actual code-writing is delegated by the -chat's active engine (the chip), but the on-disk artifact is identical either way. - -**Trigger + intent gate** (like live-note/background-task signal tiers): -- **Strong (build directly):** "make/build/create an app · mini app · dashboard - for …", "turn this into an app". -- **Medium (CONFIRM FIRST):** requests that could be a one-off answer or a - recurring app — e.g. "show me my open PRs", "track competitor launches". Ask: - "Want this as a Mini App you can reopen, or just a one-time answer?" Build only - on yes. (Building installs a folder + maybe a bg agent + an OAuth prompt — too - heavy to trigger on a casual question.) -- **Anti (do NOT build):** clearly one-off lookups/questions → just answer. - -**Flow:** -1. **Scope the app** — derive `id` (slug), `title`, `description`, `source`, the - Composio `scope[]`, and whether it's **agent-backed** (scheduled data) or - **live** (calls Composio on use). -2. **Verify wiring BEFORE building** (mandatory, engine-agnostic) — ensure the - toolkits are connected (prompt OAuth if not), then actually call the needed - tools (`composio-search-tools` → `composio-execute-tool`), inspect the REAL - returned data, and derive the **data schema from actual responses** — never - guess the shape. -3. **Pick the writer (branch):** - - **Code Mode active** → create the folder + a manifest skeleton, then - `code_agent_run` with `cwd = ~/.rowboat/apps//` to author - `dist/index.html` against the verified schema + bridge contract; it can - iterate/test on-device. - - **No Code Mode** → Copilot writes `dist/index.html` itself (from the app - template + bridge shim) and installs via `mini-apps:seed`. -4. **Bridge contract** — generated app references the canonical shim - (`app://miniapp/__bridge__.js`) and uses `window.rowboat` - (`getData/onData`, `isConnected/connect`, `searchTools/callAction`). -5. **Data pipeline (if agent-backed)** — create a background task (existing - bg-tasks engine), set `manifest.agent` to its slug. The agent **returns - schema-validated data**; the **runner writes `data.json` deterministically** - (temp→rename, last-good on failure). App reads it via `onData`. -6. **Finalize** — write `manifest.json` (incl. `scope`, `agent?`), ensure - `dist/index.html`, install → app appears in the gallery (`mini-apps:list`). - Confirm end-to-end (open it; data loads). - -**Infra this needs (repo side):** -- Serve the canonical bridge shim from the protocol: `app://miniapp/__bridge__.js` - (move the shim out of per-app inlining into served infra). -- bg-tasks runner: when a task is app-linked, validate its structured output - against the app schema and write that app's `data.json` (deterministic write - mode) instead of `index.md`. -- A `build-mini-app` skill registered in the skills catalog. - -## 3. Phase 1 — detailed implementation - -UI-first. Everything hand-coded; no `~/.rowboat` storage, no IPC, no agent yet. -We *do* mirror the eventual shapes so later phases slot in. - -### 3.1 New files (renderer) - -| File | Purpose | -|------|---------| -| `apps/renderer/src/mini-apps/types.ts` | `MiniApp` type (id, name, description, icon, accent, scope, html, data). | -| `apps/renderer/src/mini-apps/registry.ts` | Hardcoded list of `MiniApp`s for Phase 1. | -| `apps/renderer/src/mini-apps/apps/twitter-client.ts` | The sample app: self-contained HTML (React+Tailwind+Babel CDN) + static `data`. | -| `apps/renderer/src/components/mini-apps-view.tsx` | Card grid; internal `selectedAppId` state; renders grid or open app. | -| `apps/renderer/src/components/mini-app-frame.tsx` | Sandboxed iframe (`srcdoc`) + `window.rowboat` postMessage bridge (data wired; actions stubbed → toast/log). | - -The bridge message protocol (host ↔ iframe), defined once and shared: -- iframe → host: `{ type: 'rowboat:mini-app:ready' }`, - `{ type: 'rowboat:mini-app:action', id, scope, action, args }`, - `{ type: 'rowboat:mini-app:setState', patch }` -- host → iframe: `{ type: 'rowboat:mini-app:data', data }`, - `{ type: 'rowboat:mini-app:state', state }`, - `{ type: 'rowboat:mini-app:action-result', id, ok, result?, error? }` - -### 3.2 Wiring into `App.tsx` (mirror the `bg-tasks` view exactly) - -Add a first-class `apps` view. Edit sites (all mirror an existing view): - -1. **Tab path const** (~L198): add `const APPS_TAB_PATH = '__rowboat_mini_apps__'`. -2. **Tab predicate** (~L374): `const isAppsTabPath = (path) => path === APPS_TAB_PATH`. -3. **ViewState union** (~L636): add `| { type: 'apps' }`. -4. **`parseDeepLink`** (~L713): add `case 'apps': return { type: 'apps' }`. -5. **State flag** (~L825): `const [isAppsOpen, setIsAppsOpen] = useState(false)`. -6. **Tab title** (~L1253): `if (isAppsTabPath(tab.path)) return 'Mini Apps'`. -7. **Tab activation → flag** (~L3269, ~L3443): set `isAppsOpen` from tab path. -8. **Closing-tab guard** (~L3376): add `&& !isAppsTabPath(closingTab.path)`. -9. **`currentViewState`** (~L3770): `if (isAppsOpen) return { type: 'apps' }` - (place alongside bg-tasks; add to deps array). -10. **`ensureAppsFileTab`** (~L3853): mirror `ensureBgTasksFileTab`. -11. **`openAppsView`** (~L3953): mirror `openBgTasksView`; reset all other flags, - `setIsAppsOpen(true)`, `ensureAppsFileTab()`. -12. **`applyViewState` switch** (~L4195): add `case 'apps':` mirroring `bg-tasks`. -13. **Add `setIsAppsOpen(false)`** to every *other* view's reset block (the - shared multi-`set...(false)` lines) so opening another view clears apps and - closing it doesn't fall back to apps. Use targeted edits per block. -14. **Derived booleans** that OR all view flags (isFullScreenChat, - rightPaneAvailable, inFileView, rightPaneContext, viewOpen, keyboard guards): - add `|| isAppsOpen` / `&& !isAppsOpen` consistently (search `isBgTasksOpen`). -15. **Tab-path mapping** (~L4668): add `: isAppsOpen ? APPS_TAB_PATH`. -16. **Render branch** (~L5894): add `) : isAppsOpen ? (`. - -### 3.3 Sidebar entry (`sidebar-content.tsx`) - -- Add `onOpenApps?: () => void` prop (~L163) and destructure (~L412). -- Add a `SidebarMenuButton` with `isActive={activeNav === 'apps'}` and a - `LayoutGrid` (lucide) icon, label "Mini Apps", in the lower nav group near - "Background agents" (~L830). -- Thread `activeNav === 'apps'` from the parent (App.tsx passes `activeNav` based - on `isAppsOpen`, mirroring how `'agents'` is derived ~L5651). -- Wire `onOpenApps={openAppsView}` at the `SidebarContent`/home call sites - (~L5657, ~L5832). - -### 3.4 The sample app (twitter-client) - -Self-contained HTML string. React + ReactDOM + Babel-standalone + Tailwind, all -via CDN. Renders three sections — **To read / To repost / To respond** (reply -pre-drafted) — from data delivered via `rowboat.onData`. Buttons call -`rowboat.callAction('twitter', ...)`; in Phase 1 the host just toasts/logs and -returns a fake ok. Static `data` lives next to the HTML. - -> Note: CDN scripts require network + `allow-scripts` in the sandbox. The frame -> uses `sandbox="allow-scripts allow-popups allow-forms allow-modals"` — **no** -> `allow-same-origin` (keeps the app from reaching host cookies/storage; the -> bridge is the only channel). Verify CDN loads under this sandbox during testing; -> if blocked, fall back to bundling the libs locally as a renderer asset. - -### 3.5 Verify - -- `cd apps/x && npm run deps && npm run lint` compiles clean. -- `npm run dev`: "Mini Apps" appears in the sidebar; opens the card grid; clicking - the Twitter card renders the app full-screen in the pane; sections populate from - data; clicking a button shows the stubbed action result; tabs/back/forward and - switching to other views behave like every other view. - -## 4. Out of scope for Phase 1 (later phases) - -`~/.rowboat/apps//` storage + IPC; the agent backend (reuse bg-tasks -engine); real Composio execution through the bridge; per-app scope enforcement & -auth prompting; generation via Copilot/Code-Mode; persistent state store; -conversational editing & error recovery; the V2 workspace drag. diff --git a/apps/x/VIDEO_MODE.md b/apps/x/VIDEO_MODE.md deleted file mode 100644 index 999a7f56..00000000 --- a/apps/x/VIDEO_MODE.md +++ /dev/null @@ -1,253 +0,0 @@ -# Calls (Video Mode) — Deep Dive - -Calls let the user talk to the assistant hands-free while it *sees* them -(webcam) and their screen (screen share). There is ONE call engine — -continuous listening, auto-submitted utterances, forced read-aloud TTS, frame -capture — entered through four presets that differ only in starting devices. -This doc covers the product flow, the technical pipeline, and the LLM prompt -surface with exact pointers. - -## Product flow - -The composer has a **call split-button** (`chat-input-with-mentions.tsx`). -The main click is the "work together" default — preset `share`: screen -sharing ON, camera OFF, floating pill, so the user keeps working while the -assistant watches along (the button tooltip discloses the screen share). The -chevron menu holds the deviations. While a call is live the button turns red -and ends it. - -| Preset | Starting devices | First surface | -|--------|------------------|---------------| -| `share` — main click | screen on, camera off | floating pill | -| `voice` — "Voice call" | camera off, screen off | floating mascot pill | -| `video` — "Video call" | camera on | full-screen call | -| `practice` — "Practice session" | camera on, + coaching persona | full-screen call | - -**One surface rule** (`callSurface` in `App.tsx`): full screen and screen -sharing are mutually exclusive in both directions — a full-screen call covers -the screen, so sharing it would show the call itself. - -- sharing → floating popout, always (pill = working) -- not sharing → full screen unless `callMinimized` (full screen = facing - each other) -- expanding the pill auto-STOPS any share; minimizing the full-screen call - auto-STARTS one (the pill exists to work together) — presenting from full - screen likewise collapses to the pill -- the camera toggle never changes the surface: turning it on from the pill - puts your video IN the pill; expanding is its own explicit action - -**Screen-share consent** is three-layered: a toast the moment any share -starts ("Your screen is being shared… [Stop sharing]"), a persistent -"Sharing screen" badge on the pill, and macOS's purple recording indicator. -If the auto-share fails (Screen Recording permission not granted) the call -starts anyway as a voice call, with a toast linking to System Settings. -Practice/coaching is always an explicit choice — expanding to full screen -never turns the coach on. - -In-call controls (identical bar on both surfaces): mic mute, camera toggle -(silhouette avatar while off, no webcam frames captured), screen share -toggle, mascot ⇄ "R" letter avatar, end call. **Mute is a full input -pause**, not just audio — mic audio stops reaching Deepgram -(`useVoiceMode.setPaused`, OR'd with the automatic thinking/speaking pause) -AND camera/screen frame capture stops (`useVideoMode.setCapturePaused`; -`collectFrames()` returns nothing while muted, so typed messages carry no -frames either), letting the user talk to someone in the room without the -assistant listening in. Devices stay acquired for instant unmute (camera -light and macOS share indicator stay on — the pill's share badge switches to -"Sharing paused"), the status chip shows "Muted" instead of "Listening", -and assistant output is unaffected (in-flight speech keeps playing; Stop -handles that). Mute resets to off at call start/end. While the assistant is thinking or speaking, a -red **Stop** button appears on the mascot tile — it silences TTS instantly, -skips queued voice segments, and aborts the run if it's still generating -(stopping a run from anywhere, including the composer, also silences TTS). Captions of the in-progress utterance and the -assistant's spoken line run along the bottom. Typing in the composer still -works mid-call; frames ride along with typed messages too. - -Outside calls the composer keeps exactly one voice affordance: the **mic -button** (push-to-talk dictation, untouched). Spoken responses exist only -inside calls (forced full read-aloud, off on hang-up). The old video -dropdown, talking-head toggle, read-aloud headphones toggle, and summary/full -TTS dropdown are all retired — a per-message "read aloud" action on assistant -messages is the planned replacement for text-in/voice-out. - -The call button is disabled unless both voice input (Deepgram) and voice -output (TTS) are configured. `call_started` (with `preset`) is captured in -PostHog — the adoption metric for this feature. - -**Popout mechanics**: a small always-on-top frameless window (camera tile -when on + mascot tile, live caption, control bar) floating over every app — -including Rowboat. Control-bar actions round-trip `video:popoutAction` → -main → `video:popout-action` → app window, which owns the mic/camera/capture; -`expand` also refocuses the app window (handled in main). - -## Frame pipeline - -`apps/renderer/src/hooks/useVideoMode.ts` runs one capture pipe per source -(stream → offscreen `