diff --git a/.github/workflows/electron-build.yml b/.github/workflows/electron-build.yml index 787e28e6..e68ce4f4 100644 --- a/.github/workflows/electron-build.yml +++ b/.github/workflows/electron-build.yml @@ -163,12 +163,25 @@ jobs: run: pnpm install --frozen-lockfile working-directory: apps/x + - name: Build node-pty native binary for Linux + working-directory: apps/x + run: | + # node-pty ships prebuilt binaries only for darwin/win32; compile the + # linux-x64 binary so bundle.mjs can stage it into the package. Without + # this the Linux app crashes on launch (missing prebuilds/linux-x64/pty.node). + PTY="node_modules/.pnpm/node-pty@1.1.0/node_modules/node-pty" + cd "$PTY" + npx node-gyp rebuild + mkdir -p prebuilds/linux-x64 + cp build/Release/pty.node prebuilds/linux-x64/ + - name: Build electron app env: VITE_PUBLIC_POSTHOG_KEY: ${{ secrets.VITE_PUBLIC_POSTHOG_KEY }} VITE_PUBLIC_POSTHOG_HOST: ${{ secrets.VITE_PUBLIC_POSTHOG_HOST }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: npx electron-forge publish --arch=x64,arm64 --platform=linux + ROWBOAT_SKIP_PACMAN: '1' # Arch Linux package is built locally only, never in CI + run: npx electron-forge publish --arch=x64 --platform=linux working-directory: apps/x/apps/main - name: Upload workflow artifacts diff --git a/.github/workflows/x-tests.yml b/.github/workflows/x-tests.yml new file mode 100644 index 00000000..f657d478 --- /dev/null +++ b/.github/workflows/x-tests.yml @@ -0,0 +1,40 @@ +name: apps/x tests + +on: + pull_request: + paths: + - "apps/x/**" + - ".github/workflows/x-tests.yml" + push: + branches: [main] + paths: + - "apps/x/**" + - ".github/workflows/x-tests.yml" + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: apps/x + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: + version: 10 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: apps/x/pnpm-lock.yaml + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # Builds @x/shared (core/renderer tests import its dist), then runs + # the shared, core, and renderer vitest suites in order. + - name: Run tests + run: npm test diff --git a/.gitignore b/.gitignore index 086ea0b5..c9eee447 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ data/ .venv/ .claude/ + +# Local-only meeting-prep planning doc +/PLAN.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..0b40683a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,101 @@ +# AGENTS.md — Working on the Rowboat runtime + +Context for AI coding agents (and humans) working on this repo. General +codebase orientation lives in `CLAUDE.md`; this file covers the **new +turn/session runtime** in `apps/x` — its storage, its debugging tools, and +the invariants you must not break. + +## The runtime in one paragraph + +Chats are **sessions**; each user message starts a **turn**. Both are +append-only JSONL event logs under `~/.rowboat/storage/{turns,sessions}/YYYY/MM/DD/`. +All state is derived by pure reducers (`reduceTurn`, `reduceSession` in +`@x/shared/src/turns.ts` / `sessions.ts`) shared byte-for-byte between the +main process and the renderer. Design specs: +`apps/x/packages/core/docs/turn-runtime-design.md` and `session-design.md` — +read the relevant spec before changing runtime behavior. + +## Storage is reference-based — files store each fact once + +Three applications of the same mechanism: + +1. **Context**: a session turn's `context` is `{ previousTurnId }`; the + conversation prefix is materialized by walking the chain + (`TurnRepoContextResolver`). +2. **Model requests**: `model_call_requested.request.messages` is a list of + string refs into the turn's own events — `"context"`, `"input"`, + `"assistant:"`, `"toolResult:"` — recording only what + is NEW since the previous call. +3. **Agent snapshots**: when a turn's system prompt + tools are + byte-identical to its predecessor's, `turn_created.agent.resolved` is + `{ agentId, model, inheritedFrom }` instead of re-persisting ~70KB. The + model stays concrete (mid-session model switches still inherit). + +The exact provider payload is rebuilt by `composeModelRequest` +(`packages/core/src/turns/compose-model-request.ts`) — **the same code path +the loop transmits through**, so the file plus the composer reproduce the +wire bytes exactly (there is a property test asserting composed == sent). + +## Inspecting turns and sessions + +```bash +cd apps/x/packages/core + +# A whole session: title, per-turn status/size/input preview +npm run inspect -- + +# One turn: per model call, the EXACT provider payload — resolved system +# prompt, tool list, wire-form messages (user-message context woven in, +# tool-result envelopes), and the response/failure +npm run inspect -- [modelCallIndex] [--full] + +# Cascade full turn inspection across a session +npm run inspect -- --turns +``` + +`--full` prints untruncated system prompts and message contents. Turn vs +session ids are auto-detected. This is the intended way to see "what did the +model actually receive" — the raw JSONL deliberately stores structural facts +and references, never the derived wire form. + +## The legacy runs runtime is code-mode-only + +The old runs runtime (`packages/core/src/runs/`, `AgentRuntime`/`streamAgent` +in `packages/core/src/agents/runtime.ts`, the `runs:*` IPC channels, and the +renderer's legacy chat-tab state machine in `App.tsx`) remains in the tree +**solely for code-mode sessions** — a deliberate, documented carve-out: + +- A code session IS a run (`sessionId === runId`). Direct-mode prompts drive + an external ACP agent and hand-assemble `code-run-event`s into the run log + (`code-mode/sessions/service.ts`); rowboat-mode code tabs go through + `runs:createMessage` → the old `AgentRuntime` loop. +- Everything else — main chat, background tasks, live notes, knowledge + pipelines, scheduled agents — runs on turns/sessions. Do NOT add new + callers of `createRun`/`createMessage`/`streamAgent`; headless work uses + `packages/core/src/agents/headless.ts` (`startHeadlessAgent`/ + `runHeadlessAgent`). +- Temporary bridges that die when code-mode is unified: the renderer + transcript loader's `runs:fetch` fallback (`lib/agent-transcript.ts`), the + `runs:downloadLog` fallback in the chat sidebar, and the `notify-user` + gate's `fetchRun` fallback (`application/lib/builtin-tools.ts`). +- The remaining migration (designed but deferred): port code sessions onto + sessions/turns — rowboat-mode prompts become normal turns with + `codeMode`/`codeCwd` composition (already supported by the agent + resolver's composition overrides), direct-mode prompts become a delegated + turn kind carrying opaque code events. That project deletes `runs/` + entirely. + +## Invariants to respect + +- Turn/session files are **append-only**; reducers reject impossible + histories loudly (`TurnCorruptionError`). Never hand-edit files. +- Durable events are persisted **before** side effects (model calls, tool + invocations). Deltas (`text_delta`, …) are stream-only, never persisted. +- The reducers in `@x/shared` must stay pure (no I/O, no node imports) — + the renderer imports them directly. +- Every behavior change needs tests: reducers in `packages/shared`, + runtime/sessions in `packages/core`, renderer stores/views in + `apps/renderer` (all vitest; run `npm test` per package). +- Schema changes: the schema is pre-release (`schemaVersion: 1` throughout); + breaking changes are acceptable but require wiping `~/.rowboat/storage` + and a note in the commit message. diff --git a/CLAUDE.md b/CLAUDE.md index 75a0f6a5..5e762a33 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,7 +109,9 @@ Long-form docs for specific features. Read the relevant file before making chang | Feature | Doc | |---------|-----| | Live Notes — single `live:` frontmatter block (one objective + optional cron / windows / eventMatchCriteria) that turns a note into a self-updating artifact, panel UI, Copilot skill, prompts catalog | `apps/x/LIVE_NOTE.md` | +| Calls (video mode) — one hands-free call engine with four presets (voice / video / share screen / practice coaching), device-derived surfaces (full-screen ⇄ floating popout), frame pipeline, prompts catalog | `apps/x/VIDEO_MODE.md` | | Analytics — PostHog event catalog, person properties, use-case taxonomy, how to add a new event | `apps/x/ANALYTICS.md` | +| Turn/session runtime — event-sourced storage, reference model, the `npm run inspect` debugger | `AGENTS.md` (repo root), `apps/x/packages/core/docs/turn-runtime-design.md`, `session-design.md` | ## Common Tasks diff --git a/README.md b/README.md index 361b87a0..ade23320 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ - - rowboat-github-2 - -
+

Rowboat

+

A desktop AI coworker with a memory of your work and built-in surfaces to act on it.

+

rowboatlabs/rowboat | Trendshift @@ -25,28 +24,95 @@

-# Rowboat -**Open-source AI coworker that turns work into a knowledge graph and acts on it** - -Rowboat connects to your email and meeting notes, builds a long-lived knowledge graph, and uses that context to help you get work done - privately, on your machine. +Rowboat indexes your work into a living knowledge graph and uses that to get work done on your machine. It includes work surfaces for collaborating with AI: email client, notes, browser, code mode, meeting note taker, and workspaces for different projects. -You can do things like: -- `Build me a deck about our next quarter roadmap` → generates a PDF using context from your knowledge graph -- `Prep me for my meeting with Alex` → pulls past decisions, open questions, and relevant threads into a crisp brief (or a voice note) -- Track a person, company or topic through live notes -- Visualize, edit, and update your knowledge graph anytime (it’s just Markdown) -- Record voice memos that automatically capture and update key takeaways in the graph Download latest for Mac/Windows/Linux: [Download](https://www.rowboatlabs.com/downloads) +

+ Screenshot 2026-06-24 at 11 40 45 PM +

+ + +

+ Demo - email to code · Demo - knowledge graph +

+ + ⭐ If you find Rowboat useful, please star the repo. It helps more people find it. -## Demo -[![Demo](https://github.com/user-attachments/assets/8b9a859b-d4f1-47ca-9d1d-9d26d982e15d)](https://www.youtube.com/watch?v=7xTpciZCfpw) +--- +## Overview -[Watch the full video](https://www.youtube.com/watch?v=7xTpciZCfpw) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

Brain

+Rowboat indexes email, meetings, slack and assistant conversations into a living Obsidian-style backlinked knowledge graph. +		 +Screenshot 2026-06-24 at 11 22 52 PM +
+

Email

+The built-in email client sorts emails into important and everything else. Rowboat automatically drafts responses for important email using all the work context. +		 +Email screenshot +
+

Background agents

+You can set up background agents that run on events like new email or on schedule like every day at 8am. They can connect to tools, search the web, use the browser and write code using Claude Code or Codex. +		 +Background agents screenshot + +
+

Built-in Browser

+Rowboat includes a browser that lets you and assistant collaborate on web tasks. Because its isolated from your main browser, you can log in only to the accounts that want the assistant to access. +		 +Browser screenshot +
+

Meeting Notes

+A local meeting note-taker that taps into mic & speaker, produces live transcript and summarizes the meeting in a markdown file and updates the knowledge graph. +		 +Meeting notes screenshot +
+

Code Mode

+Code mode lets you spin up parallel coding agents with Claude Code or Codex, and have Rowboat drive them with all the work context where needed. +		 +Code mode screenshot +
+

Integrations

+Includes one-click integrations to most popular products. +		 +Integrations screenshot +
--- @@ -81,23 +147,6 @@ All API key files use the same format: } ``` -## What it does - -Rowboat is a **local-first AI coworker** that can: -- **Remember** the important context you don’t want to re-explain (people, projects, decisions, commitments) -- **Understand** what’s relevant right now (before a meeting, while replying to an email, when writing a doc) -- **Help you act** by drafting, summarizing, planning, and producing real artifacts (briefs, emails, docs, PDF slides) - -Under the hood, Rowboat maintains an **Obsidian-compatible vault** of plain Markdown notes with backlinks — a transparent “working memory” you can inspect and edit. - -## Integrations - -Rowboat builds memory from the work you already do, including: -- **Gmail** (email) -- **Google Calendar** -- **Rowboat meeting notes** or **Fireflies** - -It also contains a library of product integrations through Composio.dev ## How it’s different @@ -111,24 +160,6 @@ Rowboat maintains **long-lived knowledge** instead: The result is memory that compounds, rather than retrieval that starts cold every time. -## What you can do with it - -- **Meeting prep** from prior decisions, threads, and open questions -- **Email drafting** grounded in history and commitments -- **Docs & decks** generated from your ongoing context (including PDF slides) -- **Follow-ups**: capture decisions, action items, and owners so nothing gets dropped -- **On-your-machine help**: create files, summarize into notes, and run workflows using local tools (with explicit, reviewable actions) - -## Live notes - -Live notes are notes that stay updated automatically. You can create one by typing '@rowboat' on a note. - -- Track a competitor or market topic across X, Reddit, and the news -- Monitor a person, project, or deal across web or your communications -- Keep a running summary of any subject you care about - -Everything is written back into your local Markdown vault. You control what runs and when. - ## Bring your own model Rowboat works with the model setup you prefer: diff --git a/apps/x/.gitignore b/apps/x/.gitignore index db195fb4..2c5f63fd 100644 --- a/apps/x/.gitignore +++ b/apps/x/.gitignore @@ -1,2 +1,3 @@ node_modules/ test-fixtures/ +*.tsbuildinfo diff --git a/apps/x/.pnpm-store/v11/index.db b/apps/x/.pnpm-store/v11/index.db new file mode 100644 index 00000000..7f9770bd Binary files /dev/null and b/apps/x/.pnpm-store/v11/index.db differ diff --git a/apps/x/ANALYTICS.md b/apps/x/ANALYTICS.md index 5ddfcf6e..b58d5f25 100644 --- a/apps/x/ANALYTICS.md +++ b/apps/x/ANALYTICS.md @@ -92,6 +92,8 @@ All in `apps/renderer/src/lib/analytics.ts`: - `chat_message_sent` — `{ voice_input, voice_output, search_enabled }` - `oauth_connected` / `oauth_disconnected` — `{ provider }` - `voice_input_started` — no properties +- `call_started` — `{ preset: 'voice' | 'video' | 'share' | 'practice' }` — a hands-free call began (see `apps/x/VIDEO_MODE.md`) +- `call_turn_latency` — `{ endpoint_to_submit_ms, submit_to_speak_ms, speak_to_audio_ms, total_ms }` — voice-to-voice latency breakdown for one call turn (utterance accepted → submitted → first TTS speak → audio playing) - `search_executed` — `{ types: string[] }` - `note_exported` — `{ format }` diff --git a/apps/x/VIDEO_MODE.md b/apps/x/VIDEO_MODE.md new file mode 100644 index 00000000..653521cb --- /dev/null +++ b/apps/x/VIDEO_MODE.md @@ -0,0 +1,253 @@ +# Calls (Video Mode) — Deep Dive + +Calls let the user talk to the assistant hands-free while it *sees* them +(webcam) and their screen (screen share). There is ONE call engine — +continuous listening, auto-submitted utterances, forced read-aloud TTS, frame +capture — entered through four presets that differ only in starting devices. +This doc covers the product flow, the technical pipeline, and the LLM prompt +surface with exact pointers. + +## Product flow + +The composer has a **call split-button** (`chat-input-with-mentions.tsx`). +The main click is the "work together" default — preset `share`: screen +sharing ON, camera OFF, floating pill, so the user keeps working while the +assistant watches along (the button tooltip discloses the screen share). The +chevron menu holds the deviations. While a call is live the button turns red +and ends it. + +| Preset | Starting devices | First surface | +|--------|------------------|---------------| +| `share` — main click | screen on, camera off | floating pill | +| `voice` — "Voice call" | camera off, screen off | floating mascot pill | +| `video` — "Video call" | camera on | full-screen call | +| `practice` — "Practice session" | camera on, + coaching persona | full-screen call | + +**One surface rule** (`callSurface` in `App.tsx`): full screen and screen +sharing are mutually exclusive in both directions — a full-screen call covers +the screen, so sharing it would show the call itself. + +- sharing → floating popout, always (pill = working) +- not sharing → full screen unless `callMinimized` (full screen = facing + each other) +- expanding the pill auto-STOPS any share; minimizing the full-screen call + auto-STARTS one (the pill exists to work together) — presenting from full + screen likewise collapses to the pill +- the camera toggle never changes the surface: turning it on from the pill + puts your video IN the pill; expanding is its own explicit action + +**Screen-share consent** is three-layered: a toast the moment any share +starts ("Your screen is being shared… [Stop sharing]"), a persistent +"Sharing screen" badge on the pill, and macOS's purple recording indicator. +If the auto-share fails (Screen Recording permission not granted) the call +starts anyway as a voice call, with a toast linking to System Settings. +Practice/coaching is always an explicit choice — expanding to full screen +never turns the coach on. + +In-call controls (identical bar on both surfaces): mic mute, camera toggle +(silhouette avatar while off, no webcam frames captured), screen share +toggle, mascot ⇄ "R" letter avatar, end call. **Mute is a full input +pause**, not just audio — mic audio stops reaching Deepgram +(`useVoiceMode.setPaused`, OR'd with the automatic thinking/speaking pause) +AND camera/screen frame capture stops (`useVideoMode.setCapturePaused`; +`collectFrames()` returns nothing while muted, so typed messages carry no +frames either), letting the user talk to someone in the room without the +assistant listening in. Devices stay acquired for instant unmute (camera +light and macOS share indicator stay on — the pill's share badge switches to +"Sharing paused"), the status chip shows "Muted" instead of "Listening", +and assistant output is unaffected (in-flight speech keeps playing; Stop +handles that). Mute resets to off at call start/end. While the assistant is thinking or speaking, a +red **Stop** button appears on the mascot tile — it silences TTS instantly, +skips queued voice segments, and aborts the run if it's still generating +(stopping a run from anywhere, including the composer, also silences TTS). Captions of the in-progress utterance and the +assistant's spoken line run along the bottom. Typing in the composer still +works mid-call; frames ride along with typed messages too. + +Outside calls the composer keeps exactly one voice affordance: the **mic +button** (push-to-talk dictation, untouched). Spoken responses exist only +inside calls (forced full read-aloud, off on hang-up). The old video +dropdown, talking-head toggle, read-aloud headphones toggle, and summary/full +TTS dropdown are all retired — a per-message "read aloud" action on assistant +messages is the planned replacement for text-in/voice-out. + +The call button is disabled unless both voice input (Deepgram) and voice +output (TTS) are configured. `call_started` (with `preset`) is captured in +PostHog — the adoption metric for this feature. + +**Popout mechanics**: a small always-on-top frameless window (camera tile +when on + mascot tile, live caption, control bar) floating over every app — +including Rowboat. Control-bar actions round-trip `video:popoutAction` → +main → `video:popout-action` → app window, which owns the mic/camera/capture; +`expand` also refocuses the app window (handled in main). + +## Frame pipeline + +`apps/renderer/src/hooks/useVideoMode.ts` runs one capture pipe per source +(stream → offscreen `