mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-21 21:31:12 +02:00
feat(x): ambient meeting detection + take-notes popup (Granola parity phase 2)
- mic-monitor Swift helper: polls CoreAudio for mic-in-use, with owning PIDs on macOS 14.4+ (process-object API) so calls attribute to the app that actually holds the mic (Chrome vs Zoom vs Slack); no TCC permission needed. Compiled best-effort by bundle.mjs; exits when stdin closes - core meetings/detector: 5s mic debounce, PID→process attribution with running-app fallback, Granola labels (Huddle/Call/Meeting detected), calendar merge (event start −15min through end), once per mic session (30s idle reset), suppressed while Rowboat itself captures (recording or assistant voice call), gated by a meeting_detection notification category (Settings > Notifications) - popup: lean Granola-style bar (title + platform + Take notes pill, floating ×) in a transparent non-activating panel at screen-saver level with fullscreen-auxiliary behavior — floats over fullscreen meetings on the cursor's display; activation policy restored immediately so the Dock icon survives and Take Notes can foreground Rowboat - Take Notes routes into the existing take-meeting-notes flow with a synthetic or matched calendar event (source: detected); never records without a click - tray icon bumped to 18pt
This commit is contained in:
Gagan
parent
9b0776596d
commit
66865b4f6b
13 changed files with 942 additions and 12 deletions
|
|
@ -91,6 +91,26 @@ if (!fs.existsSync(stagedBinary)) {
|
|||
}
|
||||
console.log('✅ node-pty staged in .package/node_modules');
|
||||
|
||||
// Compile the mic-monitor helper (ambient meeting detection) on macOS.
|
||||
// Best-effort: without swiftc — or on other platforms — the app still works,
|
||||
// ad-hoc meeting detection just stays off (main checks the binary exists).
|
||||
if (process.platform === 'darwin') {
|
||||
const swiftSrc = path.join(here, 'native', 'mic-monitor.swift');
|
||||
const helperOut = path.join(here, '.package', 'dist', 'mic-monitor');
|
||||
const upToDate = fs.existsSync(helperOut) &&
|
||||
fs.statSync(helperOut).mtimeMs >= fs.statSync(swiftSrc).mtimeMs;
|
||||
if (upToDate) {
|
||||
console.log('✅ mic-monitor helper up to date');
|
||||
} else {
|
||||
try {
|
||||
execSync(`swiftc -O "${swiftSrc}" -o "${helperOut}"`, { stdio: 'inherit' });
|
||||
console.log('✅ mic-monitor helper compiled');
|
||||
} catch {
|
||||
console.warn('⚠️ mic-monitor helper not built (swiftc unavailable?) — meeting detection disabled');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bundle the vendored agent-slack CLI into a single self-contained script next
|
||||
// to main.cjs. It runs as a child process (process.execPath with
|
||||
// ELECTRON_RUN_AS_NODE=1), so it must exist as a real file on disk — it can't
|
||||
|
|
|
|||
123
apps/x/apps/main/native/mic-monitor.swift
Normal file
123
apps/x/apps/main/native/mic-monitor.swift
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
// mic-monitor: prints a JSON line whenever the set of processes using the
|
||||
// microphone changes.
|
||||
//
|
||||
// This is the ambient meeting-detection signal (Granola-style): when another
|
||||
// app (Zoom, Meet in a browser, Slack huddle, FaceTime…) opens the
|
||||
// microphone, we report it. No audio is captured, so this requires no
|
||||
// microphone permission (TCC) — it's device state, not content.
|
||||
//
|
||||
// Two signals, best available wins:
|
||||
// - macOS 14.4+: per-process audio objects (kAudioHardwarePropertyProcessObjectList
|
||||
// + kAudioProcessPropertyIsRunningInput) give the PIDs that own the mic,
|
||||
// so the consumer can attribute the call to Chrome vs Zoom vs Slack.
|
||||
// - Fallback: kAudioDevicePropertyDeviceIsRunningSomewhere on the default
|
||||
// input device — mic in use by *someone*, no attribution.
|
||||
//
|
||||
// Protocol: one JSON object per line on stdout, emitted on every state
|
||||
// change (and once at startup): {"micInUse":true|false,"pids":[123,456]}
|
||||
// ("pids" is empty when attribution is unavailable.)
|
||||
// The process exits when stdin closes, so it can never outlive the app.
|
||||
//
|
||||
// Compiled by apps/main/bundle.mjs (best-effort, macOS only) to
|
||||
// .package/dist/mic-monitor.
|
||||
|
||||
import Foundation
|
||||
import CoreAudio
|
||||
|
||||
func defaultInputDeviceID() -> AudioDeviceID? {
|
||||
var deviceID = AudioDeviceID(0)
|
||||
var size = UInt32(MemoryLayout<AudioDeviceID>.size)
|
||||
var addr = AudioObjectPropertyAddress(
|
||||
mSelector: kAudioHardwarePropertyDefaultInputDevice,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain)
|
||||
let status = AudioObjectGetPropertyData(
|
||||
AudioObjectID(kAudioObjectSystemObject), &addr, 0, nil, &size, &deviceID)
|
||||
guard status == noErr, deviceID != kAudioObjectUnknown else { return nil }
|
||||
return deviceID
|
||||
}
|
||||
|
||||
func isRunningSomewhere(_ device: AudioDeviceID) -> Bool {
|
||||
var running = UInt32(0)
|
||||
var size = UInt32(MemoryLayout<UInt32>.size)
|
||||
var addr = AudioObjectPropertyAddress(
|
||||
mSelector: kAudioDevicePropertyDeviceIsRunningSomewhere,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain)
|
||||
let status = AudioObjectGetPropertyData(device, &addr, 0, nil, &size, &running)
|
||||
return status == noErr && running != 0
|
||||
}
|
||||
|
||||
func audioProcessObjectIDs() -> [AudioObjectID] {
|
||||
var addr = AudioObjectPropertyAddress(
|
||||
mSelector: kAudioHardwarePropertyProcessObjectList,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain)
|
||||
var size = UInt32(0)
|
||||
let sysID = AudioObjectID(kAudioObjectSystemObject)
|
||||
guard AudioObjectGetPropertyDataSize(sysID, &addr, 0, nil, &size) == noErr, size > 0 else {
|
||||
return []
|
||||
}
|
||||
var ids = [AudioObjectID](repeating: 0, count: Int(size) / MemoryLayout<AudioObjectID>.size)
|
||||
guard AudioObjectGetPropertyData(sysID, &addr, 0, nil, &size, &ids) == noErr else { return [] }
|
||||
return ids
|
||||
}
|
||||
|
||||
func processIsRunningInput(_ object: AudioObjectID) -> Bool {
|
||||
var addr = AudioObjectPropertyAddress(
|
||||
mSelector: kAudioProcessPropertyIsRunningInput,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain)
|
||||
var value = UInt32(0)
|
||||
var size = UInt32(MemoryLayout<UInt32>.size)
|
||||
let status = AudioObjectGetPropertyData(object, &addr, 0, nil, &size, &value)
|
||||
return status == noErr && value != 0
|
||||
}
|
||||
|
||||
func processPID(_ object: AudioObjectID) -> Int32? {
|
||||
var addr = AudioObjectPropertyAddress(
|
||||
mSelector: kAudioProcessPropertyPID,
|
||||
mScope: kAudioObjectPropertyScopeGlobal,
|
||||
mElement: kAudioObjectPropertyElementMain)
|
||||
var pid = pid_t(-1)
|
||||
var size = UInt32(MemoryLayout<pid_t>.size)
|
||||
guard AudioObjectGetPropertyData(object, &addr, 0, nil, &size, &pid) == noErr, pid >= 0 else {
|
||||
return nil
|
||||
}
|
||||
return Int32(pid)
|
||||
}
|
||||
|
||||
/// PIDs of processes currently capturing from any input device (macOS 14.4+;
|
||||
/// returns [] where unsupported and the device-level fallback takes over).
|
||||
func micOwningPIDs() -> [Int32] {
|
||||
var pids: [Int32] = []
|
||||
for object in audioProcessObjectIDs() where processIsRunningInput(object) {
|
||||
if let pid = processPID(object) { pids.append(pid) }
|
||||
}
|
||||
return pids.sorted()
|
||||
}
|
||||
|
||||
// Exit when the parent closes our stdin (app quit/crash) — never orphan.
|
||||
Thread {
|
||||
while readLine(strippingNewline: false) != nil {}
|
||||
exit(0)
|
||||
}.start()
|
||||
|
||||
setbuf(stdout, nil)
|
||||
|
||||
var lastInUse: Bool? = nil
|
||||
var lastPids: [Int32] = []
|
||||
while true {
|
||||
let pids = micOwningPIDs()
|
||||
// Re-resolve the default device every poll: the user can switch input
|
||||
// devices (AirPods in/out) mid-session.
|
||||
let deviceInUse = defaultInputDeviceID().map(isRunningSomewhere) ?? false
|
||||
let inUse = deviceInUse || !pids.isEmpty
|
||||
if inUse != lastInUse || pids != lastPids {
|
||||
lastInUse = inUse
|
||||
lastPids = pids
|
||||
let pidList = pids.map(String.init).joined(separator: ",")
|
||||
print("{\"micInUse\":\(inUse),\"pids\":[\(pidList)]}")
|
||||
}
|
||||
Thread.sleep(forTimeInterval: 1.0)
|
||||
}
|
||||
|
|
@ -67,7 +67,18 @@ import { syncSlackKnowledgeSources, triggerSync as triggerSlackKnowledgeSync, ge
|
|||
import { isOnboardingComplete, markOnboardingComplete } from '@x/core/dist/config/note_creation_config.js';
|
||||
import { loadNotificationSettings, saveNotificationSettings } from '@x/core/dist/config/notification_config.js';
|
||||
import { saveAppSettings } from '@x/core/dist/config/app_settings.js';
|
||||
import { setSelfCaptureActive } from '@x/core/dist/meetings/detector.js';
|
||||
import { consumePendingToggleMeetingNotes, setTrayRecordingState } from './tray.js';
|
||||
import { closeMeetingPopup, getMeetingPopupPayload, handleMeetingPopupAction } from './meeting-popup.js';
|
||||
|
||||
// Ambient meeting detection must ignore Rowboat's own mic use: meeting
|
||||
// capture and assistant voice/video calls both hold the mic. Either being
|
||||
// active suppresses "Meeting detected" prompts.
|
||||
let meetingRecordingActive = false;
|
||||
let voiceCallActive = false;
|
||||
function updateSelfCaptureState() {
|
||||
setSelfCaptureActive(meetingRecordingActive || voiceCallActive);
|
||||
}
|
||||
import * as composioHandler from './composio-handler.js';
|
||||
import * as appsIndexer from '@x/core/dist/apps/indexer.js';
|
||||
import * as appsServer from '@x/core/dist/apps/server.js';
|
||||
|
|
@ -864,8 +875,25 @@ export function setupIpcHandlers() {
|
|||
},
|
||||
'meeting:setRecordingState': async (_event, args) => {
|
||||
setTrayRecordingState(args.recording);
|
||||
meetingRecordingActive = args.recording;
|
||||
updateSelfCaptureState();
|
||||
// Recording started through another path — a lingering "Take Notes?"
|
||||
// popup is stale now.
|
||||
if (args.recording) closeMeetingPopup();
|
||||
return { success: true as const };
|
||||
},
|
||||
'voice:setCallActive': async (_event, args) => {
|
||||
voiceCallActive = args.active;
|
||||
updateSelfCaptureState();
|
||||
return { success: true as const };
|
||||
},
|
||||
'meetingDetect:getPayload': async () => {
|
||||
return { payload: getMeetingPopupPayload() };
|
||||
},
|
||||
'meetingDetect:action': async (_event, args) => {
|
||||
handleMeetingPopupAction(args.action);
|
||||
return {};
|
||||
},
|
||||
'analytics:bootstrap': async () => {
|
||||
return {
|
||||
installationId: getInstallationId(),
|
||||
|
|
|
|||
|
|
@ -67,7 +67,9 @@ import {
|
|||
import { disconnectGoogleIfScopesStale } from "./oauth-handler.js";
|
||||
import { startModelsDevRefresh } from "@x/core/dist/models/models-dev.js";
|
||||
import { loadAppSettings, saveAppSettings } from "@x/core/dist/config/app_settings.js";
|
||||
import { createAppTray, hasTray, markPendingToggleMeetingNotes } from "./tray.js";
|
||||
import { init as initMeetingDetection } from "@x/core/dist/meetings/detector.js";
|
||||
import { createAppTray, hasTray, isRecordingActive, markPendingToggleMeetingNotes } from "./tray.js";
|
||||
import { initMeetingPopup, showMeetingPopup } from "./meeting-popup.js";
|
||||
|
||||
// Captured as early as possible so it reflects actual process start. Used to
|
||||
// gate grace-eligible notifications (e.g. the burst of background-task
|
||||
|
|
@ -273,9 +275,12 @@ function showApp(): void {
|
|||
if (!mainWindow.isVisible()) mainWindow.maximize();
|
||||
mainWindow.show();
|
||||
mainWindow.focus();
|
||||
return;
|
||||
} else {
|
||||
createWindow();
|
||||
}
|
||||
createWindow();
|
||||
// The user is usually in another app (a meeting!) when this runs — a plain
|
||||
// focus() won't take the foreground from it.
|
||||
app.focus({ steal: true });
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -504,6 +509,37 @@ app.whenReady().then(async () => {
|
|||
},
|
||||
});
|
||||
|
||||
// Ambient meeting detection (Granola-style): the mic-monitor helper +
|
||||
// running-app scan produce "Meeting detected" events; the popup asks
|
||||
// before anything records. Clicking "Take Notes" routes into the same
|
||||
// renderer flow as the calendar notification.
|
||||
initMeetingPopup({
|
||||
onTakeNotes: (meeting) => {
|
||||
showApp();
|
||||
// The user may have started recording between popup and click —
|
||||
// sending the take-notes flow then would toggle it OFF.
|
||||
if (isRecordingActive()) return;
|
||||
const payload = {
|
||||
event: meeting.calendarEvent ?? { summary: meeting.noteTitle },
|
||||
openMeeting: false,
|
||||
source: "detected",
|
||||
};
|
||||
const win = mainWindow;
|
||||
if (!win || win.isDestroyed()) return;
|
||||
if (win.webContents.isLoading()) {
|
||||
win.webContents.once("did-finish-load", () => {
|
||||
if (!win.isDestroyed()) win.webContents.send("app:takeMeetingNotes", payload);
|
||||
});
|
||||
return;
|
||||
}
|
||||
win.webContents.send("app:takeMeetingNotes", payload);
|
||||
},
|
||||
});
|
||||
initMeetingDetection({
|
||||
helperPath: path.join(__dirname, "mic-monitor"),
|
||||
onDetected: (meeting) => showMeetingPopup(meeting),
|
||||
});
|
||||
|
||||
// Start workspace watcher as a main-process service
|
||||
// Watcher runs independently and catches ALL filesystem changes:
|
||||
// - Changes made via IPC handlers (workspace:writeFile, etc.)
|
||||
|
|
|
|||
156
apps/x/apps/main/src/meeting-popup.ts
Normal file
156
apps/x/apps/main/src/meeting-popup.ts
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
import { app, BrowserWindow, screen } from "electron";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type { DetectedMeeting } from "@x/core/dist/meetings/detector.js";
|
||||
|
||||
/**
|
||||
* The "Meeting detected — Take Notes?" popup: a small frameless panel in the
|
||||
* top-left corner of the display the user is on. A macOS panel (NSPanel) at
|
||||
* screen-saver level with fullscreen-auxiliary behavior, so it floats over
|
||||
* whatever the user is looking at — including a fullscreen Meet/Zoom — and
|
||||
* appears without stealing focus. Auto-dismisses after a while and never
|
||||
* records anything by itself: clicking "Take Notes" hands off to the main
|
||||
* window's existing take-meeting-notes flow.
|
||||
*/
|
||||
|
||||
// Lean bar + margins for the overhanging × and the CSS drop shadow.
|
||||
const POPUP_WIDTH = 448;
|
||||
const POPUP_HEIGHT = 96;
|
||||
const AUTO_DISMISS_MS = 45_000;
|
||||
|
||||
// Display names, Granola-style ("Chrome", not "Google Chrome").
|
||||
const SHORT_APP_NAMES: Record<string, string> = {
|
||||
"Google Chrome": "Chrome",
|
||||
"Microsoft Edge": "Edge",
|
||||
"Brave Browser": "Brave",
|
||||
"Microsoft Teams": "Teams",
|
||||
};
|
||||
|
||||
export interface MeetingPopupPayload {
|
||||
title: string;
|
||||
message: string;
|
||||
hasCalendarEvent: boolean;
|
||||
}
|
||||
|
||||
let popupWin: BrowserWindow | null = null;
|
||||
let currentPayload: MeetingPopupPayload | null = null;
|
||||
let currentMeeting: DetectedMeeting | null = null;
|
||||
let dismissTimer: NodeJS.Timeout | null = null;
|
||||
let onTakeNotes: ((meeting: DetectedMeeting) => void) | null = null;
|
||||
|
||||
/** Main registers the take-notes handoff once at startup. */
|
||||
export function initMeetingPopup(handlers: { onTakeNotes: (meeting: DetectedMeeting) => void }): void {
|
||||
onTakeNotes = handlers.onTakeNotes;
|
||||
}
|
||||
|
||||
export function getMeetingPopupPayload(): MeetingPopupPayload | null {
|
||||
return currentPayload;
|
||||
}
|
||||
|
||||
export function handleMeetingPopupAction(action: "take-notes" | "dismiss"): void {
|
||||
const meeting = currentMeeting;
|
||||
closeMeetingPopup();
|
||||
if (action === "take-notes" && meeting) {
|
||||
onTakeNotes?.(meeting);
|
||||
}
|
||||
}
|
||||
|
||||
export function closeMeetingPopup(): void {
|
||||
if (dismissTimer) {
|
||||
clearTimeout(dismissTimer);
|
||||
dismissTimer = null;
|
||||
}
|
||||
currentPayload = null;
|
||||
currentMeeting = null;
|
||||
if (popupWin && !popupWin.isDestroyed()) popupWin.destroy();
|
||||
popupWin = null;
|
||||
}
|
||||
|
||||
export function showMeetingPopup(meeting: DetectedMeeting): void {
|
||||
const eventSummary =
|
||||
typeof meeting.calendarEvent?.summary === "string"
|
||||
? (meeting.calendarEvent.summary as string).trim()
|
||||
: "";
|
||||
const appLabel = SHORT_APP_NAMES[meeting.appName] ?? meeting.appName;
|
||||
const payload: MeetingPopupPayload = {
|
||||
// Lean two-liner: "Meeting detected" / "Chrome" — or the event name
|
||||
// over the platform when a calendar event is happening now.
|
||||
title: eventSummary ? eventSummary : meeting.title,
|
||||
message: appLabel,
|
||||
hasCalendarEvent: Boolean(meeting.calendarEvent),
|
||||
};
|
||||
|
||||
// Replace any popup that's still up (stale detection loses to fresh).
|
||||
closeMeetingPopup();
|
||||
currentPayload = payload;
|
||||
currentMeeting = meeting;
|
||||
|
||||
// Top-left of the display the user is actually on (cursor display), not
|
||||
// necessarily the primary one.
|
||||
const display = screen.getDisplayNearestPoint(screen.getCursorScreenPoint());
|
||||
const workArea = display.workArea;
|
||||
const popupDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const preloadPath = app.isPackaged
|
||||
? path.join(popupDir, "../preload/dist/preload.js")
|
||||
: path.join(popupDir, "../../../preload/dist/preload.js");
|
||||
|
||||
const win = new BrowserWindow({
|
||||
width: POPUP_WIDTH,
|
||||
height: POPUP_HEIGHT,
|
||||
x: workArea.x + 24,
|
||||
y: workArea.y + 24,
|
||||
// NSPanel (macOS): non-activating, and — unlike a regular window with
|
||||
// visibleOnFullScreen — can float over fullscreen Spaces without
|
||||
// turning Rowboat into an "agent" app that loses its Dock icon.
|
||||
...(process.platform === "darwin" ? { type: "panel" as const } : {}),
|
||||
frame: false,
|
||||
resizable: false,
|
||||
alwaysOnTop: true,
|
||||
skipTaskbar: true,
|
||||
show: false,
|
||||
// Transparent window: the rounded card + overhanging × render inside;
|
||||
// the shadow is CSS (native shadows artifact on transparent windows).
|
||||
transparent: true,
|
||||
hasShadow: false,
|
||||
webPreferences: {
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
preload: preloadPath,
|
||||
},
|
||||
});
|
||||
// Screen-saver level + fullscreen-auxiliary: visible over fullscreen
|
||||
// meeting apps on every workspace — the whole point of the popup is to be
|
||||
// seen while the user is IN the meeting, wherever that is.
|
||||
win.setAlwaysOnTop(true, "screen-saver");
|
||||
win.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
|
||||
// visibleOnFullScreen flips the app's activation policy to "accessory":
|
||||
// the Dock icon disappears and the app can no longer take foreground
|
||||
// focus — clicking "Take Notes" then hands focus to whatever is next in
|
||||
// the window stack instead of Rowboat. Restore the policy immediately:
|
||||
// the popup keeps its fullscreen-auxiliary collection behavior (it's a
|
||||
// non-activating panel, same trick as Zoom's floating controls), while
|
||||
// Rowboat stays a regular app.
|
||||
if (process.platform === "darwin") {
|
||||
void app.dock?.show();
|
||||
}
|
||||
win.webContents.once("did-finish-load", () => {
|
||||
if (win.isDestroyed()) return;
|
||||
win.webContents.send("meetingDetect:payload", payload);
|
||||
// showInactive: the user is in a meeting app right now — appearing
|
||||
// must not steal focus from it.
|
||||
win.showInactive();
|
||||
});
|
||||
win.on("closed", () => {
|
||||
if (popupWin === win) popupWin = null;
|
||||
});
|
||||
popupWin = win;
|
||||
|
||||
if (app.isPackaged) {
|
||||
win.loadURL("app://-/index.html#meeting-detected");
|
||||
} else {
|
||||
win.loadURL("http://localhost:5173/#meeting-detected");
|
||||
}
|
||||
|
||||
dismissTimer = setTimeout(() => closeMeetingPopup(), AUTO_DISMISS_MS);
|
||||
}
|
||||
|
|
@ -11,10 +11,10 @@ import { app, Menu, Tray, nativeImage } from "electron";
|
|||
* packaged layouts. Template rendering makes macOS tint it correctly
|
||||
* in light/dark menu bars and while highlighted.
|
||||
*/
|
||||
const TRAY_ICON_16 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABD0lEQVR4nKSSz+pBQRTH585vfgtrllYiSkpJHkAegLJQlvIE9soD2MvSE3gCG57ARiwoe2Kl/Lm+J2d0XJd7y7c+dc898z1zzswYFV6OwAU3+zNIf9LglwwyWqKgBKogBpZAa+XfKukKIqABxmABJqAPcnat8Zi1aLUNOiAl8hfeYMaxazw7kzkNRqAoOpFjHcCc45sWZpqzxtVpxqPIac6TpmDP/5UtQMkkz1YGCZAHA89YDndnv1+u8R+c1buGoMVFtiADTjYpb+HMsSNy9N3llinusfnpC/OQSBuwAwWO7Xko88VkDzarHo+owvHLpkEFSHXQBCtR9G3RJ9HBxsHazxymgFzn+iWM+sFMugMAAP//whjxNQAAAAZJREFUAwCzrTaLm3OP9AAAAABJRU5ErkJggg==";
|
||||
const TRAY_ICON_32 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAACfUlEQVR4nMSXO2gUURSGz8yuYuGjsBHFBza+UGwULUxjoY0iWPnA0lZQURDsFC1E7Owt7JQ0SZGQLk2SNoQEQhICCXlUCXk/ZvOf3HM2Z2/uZDM7m90fPnbm3jn3P/e5M0VqsorUZBWocYqogQnEhj1VzwTUsOTBOgJOg1WwYYMiyqdITDe9RK6CW+A2uAYugOPgBhiXuO3kal2E1pg5BO6BJ+A+uEK7OzcJpuVaR6amBArG+Bx4BZ6LqZUOdSI+I+SmoEJZEoilMTa+CN6Cl+CEMUrkuUgS5d91+e33OpApAQ06Cj6CN+CY1G14pqryPMt1b6jhagloI2z+EPwCl4xxgXabqnQXFCW+R8oTaxDvw5z1DbSL+Zpp2D4bpcTz7zAYMomVVaxizkP+FzwydYfJ9Uh3QskYWXGZrolOiamY/7QE1Pwk6AaXwTzoA7OghdyhQtJY7PVWzcnU/fPKK8xC5mdBB1gGv6UHY/IMr/rH4D25gyUtCS3n4efDaJ1Seuvf8xC/BhPgv1dnTz1+7qskogvSrhsu4wPqM/hCgeEPJUC0s99VBbkveWXa2A/wzktC45fILdxJL7kKM1+JV7cZCNRFyIYfwICXqA7/HzGPA22Ue5KmElWXjtYceCrXuiVXwDOp27OBPNJpaAVT5HaVTsVPcv98qb0nqs/7ABvw4XQTXJf7UfBCEkyqBeeV9u6UXHObvIsW9xOcNwFd2WfAHWnvO+iinUV5oAlo/ANyx3Yb+EQpez6keryS8QgMkjvp7oIFStnzIeX5LtAtyK9h3OOWrOZ5E9C/5PPk3gdnspoT5Z8Ce2xnNtegeqgmc1YjP82CavrHadMT2AIAAP//JKuLxQAAAAZJREFUAwAJ75pCeqjxVQAAAABJRU5ErkJggg==";
|
||||
const TRAY_ICON_18 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAABIAAAASCAYAAABWzo5XAAABRElEQVR4nKySMS9EQRSF7773REEUlFpRiEg0Oj1RqTQqCiLRKNV+gkLvF5CoJRqJPyBEqyMURJB9zzl5Z3bvTuZtdjdu8mVm7sw9c+fOzWw0a4HMO3Ib3DJRaV0NK5QrqHLBM2AB/IAvOgrrn0EJ2pqvgg2N82AarIBX7jcJtSQyCXbALliKzryDJ83LIiEQnrAFTsBcOOxg3CN4C4FFQmQCnINN+b/BmPZDvTi/0T597SwS4dtvwTK4VPrj1mu5zl9oXQWBMDKTfXAPruSfBcfgwOqi03j5A1h0vo5QJ0W3Dr9GOwN7eiYzPASn/owXSgmEBmTfsLhT4NnqHvqMg2IrozkvewHXyvpIIj2xqYyajJncWf2bPuvGjFL76+ADbFu3WW0YIdaHvcYfWrOoLt4GeVquC3+t228jCfmzVb/b/sX+AAAA//+fzjrgAAAABklEQVQDAHsTRGNUkus5AAAAAElFTkSuQmCC";
|
||||
const TRAY_ICON_36 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAACQAAAAkCAYAAADhAJiYAAAC3klEQVR4nOyYu4sUQRDGa3fm8DgxUxB8ICqKqBj4QBOFMxMDRQUfgS/0D1A0MRQMTA1MxBcngpFmBiocgokgCopo4PnCSE193e74FV11XTtMz+zs9sLB3Qc/drenp/ubrqruuUtpmimlaaYZb6gBmvK9DbKiDoNUU+bIxEClEoqvRIxkOYbBcrAJ7ALzwATlFiVWyDQULYE1F2wE28E2sBYsJr8I58Fjc18UQw2hLYOOgJ1gHxgFSwru+SsmXsnvjjzq1ZA1wgOuBMfBQXJhUbUF7c8akraJWIZ4yVsyEIfhjBgZket6TRPa5mkmbd/AZ9PWk6GGmXAhuABOgTnSPikmmoH79F7+/Rr8IV+BtQ3xJFq2HJpLYkqNJNS5EkXbiVYb65kZt1XXkIZoAbhKLmGtkdRMFNrXMmOA+4zn2qdUtTGqma1gDKwAv2XgtOD+MkOaV5w/XAS/ijqmXZjZA+6QT9ph00erLDGTFpnitkn5fCRmbBpUGtLYHgU3pe0teAKegnVgL7kqI+lrjwlrKh+ue1Si0NPwICfBFXAX3CaXiP9MP16V3eSqbXOFKb32BawmF/ZCNQvaeDDOlWVgvRgbFzOJgSd5QO5YuEyd+5OaUnRzHBMzwTM0lIRD5FcjofBpnZDPIzZ1jnz12Qdk8ZGxBnykgv2nyhDLnthl0oOVjb0AG8itlOYnPxg/4C1wjALJrCp7/agykjfF/b+CI9QZNq2ww+BH1UBNiiPNkYfgHflQtuT7NfCe/KoHFfMFTZN6PthBblU4bLwq+ymwEeYVa4WsXspnW8Y/C75TSSJbDeIlf1Q+eUe/T24PK01kq9gv+fyAb8Aq8IncK+xPudZVkcQKmY6zhfzBeYBc/nQVKlXskJ0Wc4fAc6oRKlWMkOkKLCUXphPgBvmqqz1Yv9K95SL4AK73aiaWIRYfDYvInVO1w0QDMKTqywwr5k5dq5pCGsTf9n1p9h9WVfoPAAD//0eu+ckAAAAGSURBVAMAjdCu7L3gD4wAAAAASUVORK5CYII=";
|
||||
|
||||
interface TrayActions {
|
||||
openApp: () => void;
|
||||
|
|
@ -44,11 +44,11 @@ function buildTrayIcon() {
|
|||
const icon = nativeImage.createEmpty();
|
||||
icon.addRepresentation({
|
||||
scaleFactor: 1,
|
||||
buffer: Buffer.from(TRAY_ICON_16, "base64"),
|
||||
buffer: Buffer.from(TRAY_ICON_18, "base64"),
|
||||
});
|
||||
icon.addRepresentation({
|
||||
scaleFactor: 2,
|
||||
buffer: Buffer.from(TRAY_ICON_32, "base64"),
|
||||
buffer: Buffer.from(TRAY_ICON_36, "base64"),
|
||||
});
|
||||
icon.setTemplateImage(true);
|
||||
return icon;
|
||||
|
|
@ -80,6 +80,10 @@ export function hasTray(): boolean {
|
|||
return tray !== null;
|
||||
}
|
||||
|
||||
export function isRecordingActive(): boolean {
|
||||
return recording;
|
||||
}
|
||||
|
||||
export function setTrayRecordingState(isRecording: boolean): void {
|
||||
if (recording === isRecording) return;
|
||||
recording = isRecording;
|
||||
|
|
|
|||
|
|
@ -1083,6 +1083,13 @@ function App() {
|
|||
// floating popout; camera on → full-screen call; camera off → popout
|
||||
// (mascot pill). Handlers live below the voice/submit plumbing they drive.
|
||||
const video = useVideoMode()
|
||||
// Assistant calls hold the mic — tell main so ambient meeting detection
|
||||
// doesn't mistake our own capture for an external meeting.
|
||||
useEffect(() => {
|
||||
void window.ipc
|
||||
.invoke('voice:setCallActive', { active: video.state !== 'idle' })
|
||||
.catch(() => { /* detection may be unavailable */ })
|
||||
}, [video.state])
|
||||
const [inCall, setInCall] = useState(false)
|
||||
const inCallRef = useRef(false)
|
||||
// User explicitly shrank the full-screen call to the floating pill.
|
||||
|
|
@ -4702,7 +4709,7 @@ function App() {
|
|||
// Reuses the same flow as the in-app "Join meeting & take notes" button.
|
||||
// When `openMeeting` is true, also opens the meeting URL in the system browser.
|
||||
useEffect(() => {
|
||||
return window.ipc.on('app:takeMeetingNotes', ({ event, openMeeting }) => {
|
||||
return window.ipc.on('app:takeMeetingNotes', ({ event, openMeeting, source }) => {
|
||||
const e = event as {
|
||||
summary?: string
|
||||
start?: { dateTime?: string; date?: string; timeZone?: string }
|
||||
|
|
@ -4726,7 +4733,7 @@ function App() {
|
|||
location: e.location,
|
||||
htmlLink: e.htmlLink,
|
||||
conferenceLink,
|
||||
source: 'calendar-sync',
|
||||
source: source ?? 'calendar-sync',
|
||||
}
|
||||
window.dispatchEvent(new Event('calendar-block:join-meeting'))
|
||||
})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
import { useEffect, useState } from 'react'
|
||||
import { X } from 'lucide-react'
|
||||
|
||||
type PopupPayload = {
|
||||
title: string
|
||||
message: string
|
||||
hasCalendarEvent: boolean
|
||||
}
|
||||
|
||||
const dragRegion = { WebkitAppRegion: 'drag' } as React.CSSProperties
|
||||
const noDragRegion = { WebkitAppRegion: 'no-drag' } as React.CSSProperties
|
||||
|
||||
// Rowboat sail glyph (black on transparent, same asset as the tray icon) —
|
||||
// rendered on a white chip inside the "Take notes" pill.
|
||||
const SAIL_ICON =
|
||||
'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAACfUlEQVR4nMSXO2gUURSGz8yuYuGjsBHFBza+UGwULUxjoY0iWPnA0lZQURDsFC1E7Owt7JQ0SZGQLk2SNoQEQhICCXlUCXk/ZvOf3HM2Z2/uZDM7m90fPnbm3jn3P/e5M0VqsorUZBWocYqogQnEhj1VzwTUsOTBOgJOg1WwYYMiyqdITDe9RK6CW+A2uAYugOPgBhiXuO3kal2E1pg5BO6BJ+A+uEK7OzcJpuVaR6amBArG+Bx4BZ6LqZUOdSI+I+SmoEJZEoilMTa+CN6Cl+CEMUrkuUgS5d91+e33OpApAQ06Cj6CN+CY1G14pqryPMt1b6jhagloI2z+EPwCl4xxgXabqnQXFCW+R8oTaxDvw5z1DbSL+Zpp2D4bpcTz7zAYMomVVaxizkP+FzwydYfJ9Uh3QskYWXGZrolOiamY/7QE1Pwk6AaXwTzoA7OghdyhQtJY7PVWzcnU/fPKK8xC5mdBB1gGv6UHY/IMr/rH4D25gyUtCS3n4efDaJ1Seuvf8xC/BhPgv1dnTz1+7qskogvSrhsu4wPqM/hCgeEPJUC0s99VBbkveWXa2A/wzktC45fILdxJL7kKM1+JV7cZCNRFyIYfwICXqA7/HzGPA22Ue5KmElWXjtYceCrXuiVXwDOp27OBPNJpaAVT5HaVTsVPcv98qb0nqs/7ABvw4XQTXJf7UfBCEkyqBeeV9u6UXHObvIsW9xOcNwFd2WfAHWnvO+iinUV5oAlo/ANyx3Yb+EQpez6keryS8QgMkjvp7oIFStnzIeX5LtAtyK9h3OOWrOZ5E9C/5PPk3gdnspoT5Z8Ce2xnNtegeqgmc1YjP82CavrHadMT2AIAAP//JKuLxQAAAAZJREFUAwAJ75pCeqjxVQAAAABJRU5ErkJggg=='
|
||||
|
||||
/**
|
||||
* Content of the "Meeting detected" always-on-top popup panel (see
|
||||
* `showMeetingPopup` in the main process). Granola-style consent prompt —
|
||||
* a lean horizontal bar: dashed rule (ad-hoc) or solid accent (calendar
|
||||
* match), "Meeting detected" + platform, and a "Take notes" pill. The ×
|
||||
* floats on the card's top-left corner; the window itself is transparent.
|
||||
* Detection never records — the user has to click "Take notes".
|
||||
*/
|
||||
export function MeetingDetectedPopup() {
|
||||
const [payload, setPayload] = useState<PopupPayload | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
const cleanup = window.ipc.on('meetingDetect:payload', (next) => setPayload(next))
|
||||
// The main process pushes on did-finish-load, but that can race this
|
||||
// listener's registration — fetch explicitly too.
|
||||
window.ipc
|
||||
.invoke('meetingDetect:getPayload', null)
|
||||
.then(({ payload: cached }) => {
|
||||
if (cached) setPayload(cached)
|
||||
})
|
||||
.catch(() => {})
|
||||
return cleanup
|
||||
}, [])
|
||||
|
||||
const act = (action: 'take-notes' | 'dismiss') => {
|
||||
void window.ipc.invoke('meetingDetect:action', { action }).catch(() => {})
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="h-screen w-screen relative bg-transparent" style={dragRegion}>
|
||||
{/* Close — floats over the card's top-left corner */}
|
||||
<button
|
||||
onClick={() => act('dismiss')}
|
||||
className="absolute left-0.5 top-0.5 z-10 flex size-6 items-center justify-center rounded-full bg-neutral-800 border border-neutral-600 text-neutral-200 shadow-md hover:bg-neutral-700 transition-colors"
|
||||
style={noDragRegion}
|
||||
aria-label="Dismiss"
|
||||
>
|
||||
<X className="size-3.5" strokeWidth={2.5} />
|
||||
</button>
|
||||
|
||||
{/* Card */}
|
||||
<div className="absolute left-3 right-2 top-3 h-16 rounded-2xl bg-[#1d1d1d] shadow-[0_8px_28px_rgba(0,0,0,0.55)] flex items-center gap-3.5 pl-4 pr-2">
|
||||
<div
|
||||
className={`h-8 w-0 border-l-2 ${
|
||||
payload?.hasCalendarEvent
|
||||
? 'border-emerald-500'
|
||||
: 'border-dashed border-neutral-500'
|
||||
}`}
|
||||
/>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="text-[15px] font-semibold text-white leading-tight truncate">
|
||||
{payload?.title ?? 'Meeting detected'}
|
||||
</div>
|
||||
<div className="text-sm text-neutral-400 leading-tight truncate">
|
||||
{payload?.message ?? ''}
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => act('take-notes')}
|
||||
className="flex shrink-0 items-center gap-2.5 rounded-xl bg-neutral-800/90 border border-neutral-700 pl-2.5 pr-4 py-2 hover:bg-neutral-700 transition-colors"
|
||||
style={noDragRegion}
|
||||
>
|
||||
<span className="flex size-7 items-center justify-center rounded-lg bg-white">
|
||||
<img src={SAIL_ICON} alt="" className="size-4" />
|
||||
</span>
|
||||
<span className="text-[15px] font-semibold text-white">Take notes</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -2149,7 +2149,7 @@ function CodeModeSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
|||
|
||||
// --- Notification Settings ---
|
||||
|
||||
type NotificationCategoryKey = "chat_completion" | "new_email" | "agent_permission" | "background_task"
|
||||
type NotificationCategoryKey = "chat_completion" | "new_email" | "agent_permission" | "background_task" | "meeting_detection"
|
||||
|
||||
const NOTIFICATION_CATEGORIES: { key: NotificationCategoryKey; label: string; description: string }[] = [
|
||||
{
|
||||
|
|
@ -2172,6 +2172,11 @@ const NOTIFICATION_CATEGORIES: { key: NotificationCategoryKey; label: string; de
|
|||
label: "Background agents",
|
||||
description: "When a background agent you've set up has something to surface. Click to open it on the background tasks page.",
|
||||
},
|
||||
{
|
||||
key: "meeting_detection",
|
||||
label: "Meeting detection",
|
||||
description: "A popup offering to take notes when Rowboat notices you're in a call or meeting. Nothing records until you accept.",
|
||||
},
|
||||
]
|
||||
|
||||
function NotificationSettings({ dialogOpen }: { dialogOpen: boolean }) {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import type { CaptureResult } from 'posthog-js'
|
|||
import { ThemeProvider } from '@/contexts/theme-context'
|
||||
import { configureAnalyticsContext } from './lib/analytics'
|
||||
import { VideoPopout } from '@/components/video-popout'
|
||||
import { MeetingDetectedPopup } from '@/components/meeting-detected-popup'
|
||||
|
||||
// Fetch the stable installation ID from main so renderer + main share one
|
||||
// PostHog distinct_id. Falls back to PostHog's auto-generated anonymous ID
|
||||
|
|
@ -66,6 +67,13 @@ if (window.location.hash === '#video-popout') {
|
|||
<VideoPopout />
|
||||
</StrictMode>,
|
||||
)
|
||||
} else if (window.location.hash === '#meeting-detected') {
|
||||
// "Meeting detected — Take Notes?" popup window; same pattern.
|
||||
createRoot(document.getElementById('root')!).render(
|
||||
<StrictMode>
|
||||
<MeetingDetectedPopup />
|
||||
</StrictMode>,
|
||||
)
|
||||
} else {
|
||||
bootstrap()
|
||||
}
|
||||
|
|
|
|||
406
apps/x/packages/core/src/meetings/detector.ts
Normal file
406
apps/x/packages/core/src/meetings/detector.ts
Normal file
|
|
@ -0,0 +1,406 @@
|
|||
import { spawn, execFile } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { WorkDir } from "../config/config.js";
|
||||
import { isNotificationCategoryEnabled } from "../config/notification_config.js";
|
||||
|
||||
/**
|
||||
* Ambient meeting detection (Granola-style).
|
||||
*
|
||||
* Signal: the mic-monitor helper (apps/main/native/mic-monitor.swift) reports
|
||||
* when ANY process starts using the microphone — with the owning PIDs on
|
||||
* macOS 14.4+, so the call is attributed to the app that actually holds the
|
||||
* mic (Chrome vs Zoom vs Slack), not just whatever meeting app happens to be
|
||||
* running. When the mic has been in use continuously for a few seconds — and
|
||||
* it isn't Rowboat's own capture — we label the popup ("Huddle detected" for
|
||||
* Slack, "Call detected" for FaceTime/WhatsApp, "Meeting detected"
|
||||
* otherwise), merge with a calendar event whose window covers now
|
||||
* (start − 15 min through end), and emit a DetectedMeeting.
|
||||
*
|
||||
* Never auto-records: the consumer (main process) shows a "Take Notes?"
|
||||
* popup and waits for a click.
|
||||
*
|
||||
* Fires at most once per continuous mic-in-use session; the session resets
|
||||
* after the mic has been idle for MIC_SESSION_RESET_MS.
|
||||
*/
|
||||
|
||||
const POLL_INTERVAL_MS = 2_000;
|
||||
// Mic must be in use continuously this long before we prompt — filters out
|
||||
// Siri, dictation bursts, and app mic-permission probes.
|
||||
const MIC_DEBOUNCE_MS = 5_000;
|
||||
// Mic idle this long ends the "session" and re-arms the prompt.
|
||||
const MIC_SESSION_RESET_MS = 30_000;
|
||||
// A calendar event merges with a detected call from 15 min before its start
|
||||
// (Granola merges ad-hoc calls within 15 minutes of a scheduled event).
|
||||
const CALENDAR_MERGE_LEAD_MS = 15 * 60_000;
|
||||
const CALENDAR_SYNC_DIR = path.join(WorkDir, "calendar_sync");
|
||||
const HELPER_MAX_RESTARTS = 3;
|
||||
|
||||
export type DetectedMeetingKind = "huddle" | "call" | "meeting";
|
||||
|
||||
export interface DetectedMeeting {
|
||||
kind: DetectedMeetingKind;
|
||||
/** Popup title, Granola wording: "Huddle detected" / "Call detected" / "Meeting detected". */
|
||||
title: string;
|
||||
/** Human label of the app that likely owns the call, e.g. "Slack", "Zoom", "Google Chrome". */
|
||||
appName: string;
|
||||
/** Suggested note title when no calendar event matched, e.g. "Slack huddle". */
|
||||
noteTitle: string;
|
||||
/** Raw calendar event JSON when a nearby event matched. */
|
||||
calendarEvent?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
// Matched against process names by case-insensitive prefix (Electron/Chromium
|
||||
// apps capture through helper processes: "Google Chrome Helper (Renderer)",
|
||||
// "Slack Helper"…). When mic-owner PIDs are available this is exact
|
||||
// attribution; otherwise it's a running-app heuristic ordered by confidence:
|
||||
// dedicated conferencing apps first, always-running chat apps after, browsers
|
||||
// as the generic fallback.
|
||||
const MEETING_APPS: Array<{ proc: string; app: string; kind: DetectedMeetingKind }> = [
|
||||
{ proc: "zoom.us", app: "Zoom", kind: "meeting" },
|
||||
{ proc: "MSTeams", app: "Microsoft Teams", kind: "meeting" },
|
||||
{ proc: "Microsoft Teams", app: "Microsoft Teams", kind: "meeting" },
|
||||
{ proc: "Webex", app: "Webex", kind: "meeting" },
|
||||
{ proc: "FaceTime", app: "FaceTime", kind: "call" },
|
||||
{ proc: "WhatsApp", app: "WhatsApp", kind: "call" },
|
||||
{ proc: "Slack", app: "Slack", kind: "huddle" },
|
||||
{ proc: "Discord", app: "Discord", kind: "call" },
|
||||
];
|
||||
|
||||
const BROWSERS = [
|
||||
"Google Chrome",
|
||||
"Safari",
|
||||
"Arc",
|
||||
"Brave Browser",
|
||||
"Microsoft Edge",
|
||||
"Firefox",
|
||||
"Dia",
|
||||
"Comet",
|
||||
];
|
||||
|
||||
const KIND_TITLES: Record<DetectedMeetingKind, string> = {
|
||||
huddle: "Huddle detected",
|
||||
call: "Call detected",
|
||||
meeting: "Meeting detected",
|
||||
};
|
||||
|
||||
interface DetectorOptions {
|
||||
/** Absolute path to the compiled mic-monitor helper binary. */
|
||||
helperPath: string;
|
||||
onDetected: (meeting: DetectedMeeting) => void;
|
||||
}
|
||||
|
||||
let started = false;
|
||||
let selfCaptureActive = false;
|
||||
let micInUse = false;
|
||||
// PIDs currently capturing from the mic (macOS 14.4+; empty = unknown).
|
||||
let micPids: number[] = [];
|
||||
let micInUseSince: number | null = null;
|
||||
let micIdleSince: number | null = null;
|
||||
let sessionNotified = false;
|
||||
let helperRestarts = 0;
|
||||
|
||||
/**
|
||||
* Rowboat's own capture (meeting recording, assistant voice/video call) also
|
||||
* flips the mic-in-use signal — the consumer reports it here so we never
|
||||
* prompt about our own audio.
|
||||
*/
|
||||
export function setSelfCaptureActive(active: boolean): void {
|
||||
selfCaptureActive = active;
|
||||
if (active) {
|
||||
// Whatever mic session is in flight is ours — don't prompt when the
|
||||
// user stops and the mic lingers.
|
||||
sessionNotified = true;
|
||||
}
|
||||
}
|
||||
|
||||
export function init(options: DetectorOptions): void {
|
||||
if (started) return;
|
||||
if (process.platform !== "darwin") return;
|
||||
if (!existsSync(options.helperPath)) {
|
||||
console.warn(
|
||||
`[MeetingDetect] mic-monitor helper not found at ${options.helperPath} — ambient detection disabled`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
started = true;
|
||||
console.log("[MeetingDetect] starting ambient meeting detection");
|
||||
|
||||
spawnHelper(options.helperPath);
|
||||
setInterval(() => {
|
||||
try {
|
||||
tick(options.onDetected);
|
||||
} catch (err) {
|
||||
console.error("[MeetingDetect] tick failed:", err);
|
||||
}
|
||||
}, POLL_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function spawnHelper(helperPath: string): void {
|
||||
let child: ReturnType<typeof spawn>;
|
||||
try {
|
||||
child = spawn(helperPath, [], { stdio: ["pipe", "pipe", "ignore"] });
|
||||
} catch (err) {
|
||||
console.error("[MeetingDetect] failed to spawn mic-monitor:", err);
|
||||
return;
|
||||
}
|
||||
|
||||
let buffer = "";
|
||||
child.stdout?.setEncoding("utf8");
|
||||
child.stdout?.on("data", (chunk: string) => {
|
||||
buffer += chunk;
|
||||
let idx: number;
|
||||
while ((idx = buffer.indexOf("\n")) >= 0) {
|
||||
const line = buffer.slice(0, idx).trim();
|
||||
buffer = buffer.slice(idx + 1);
|
||||
if (!line) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
if (typeof parsed?.micInUse === "boolean") {
|
||||
micInUse = parsed.micInUse;
|
||||
micPids = Array.isArray(parsed.pids)
|
||||
? parsed.pids.filter((p: unknown): p is number => typeof p === "number")
|
||||
: [];
|
||||
}
|
||||
} catch {
|
||||
// Ignore malformed lines.
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
child.on("exit", (code) => {
|
||||
micInUse = false;
|
||||
micPids = [];
|
||||
if (helperRestarts < HELPER_MAX_RESTARTS) {
|
||||
helperRestarts += 1;
|
||||
const delay = 5_000 * helperRestarts;
|
||||
console.warn(
|
||||
`[MeetingDetect] mic-monitor exited (code ${code}); restart ${helperRestarts}/${HELPER_MAX_RESTARTS} in ${delay / 1000}s`,
|
||||
);
|
||||
setTimeout(() => spawnHelper(helperPath), delay);
|
||||
} else {
|
||||
console.error("[MeetingDetect] mic-monitor kept exiting — ambient detection disabled");
|
||||
}
|
||||
});
|
||||
child.on("error", (err) => {
|
||||
console.error("[MeetingDetect] mic-monitor error:", err);
|
||||
});
|
||||
}
|
||||
|
||||
function tick(onDetected: DetectorOptions["onDetected"]): void {
|
||||
const now = Date.now();
|
||||
|
||||
if (!micInUse) {
|
||||
micInUseSince = null;
|
||||
if (micIdleSince === null) micIdleSince = now;
|
||||
if (sessionNotified && now - micIdleSince >= MIC_SESSION_RESET_MS) {
|
||||
sessionNotified = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
micIdleSince = null;
|
||||
if (micInUseSince === null) micInUseSince = now;
|
||||
if (sessionNotified) return;
|
||||
if (selfCaptureActive) {
|
||||
// Mark the session as handled so a call that started as ours doesn't
|
||||
// prompt the moment our capture stops while the meeting app holds on.
|
||||
sessionNotified = true;
|
||||
return;
|
||||
}
|
||||
if (now - micInUseSince < MIC_DEBOUNCE_MS) return;
|
||||
|
||||
// Claim the session before the async work — a slow ps/calendar scan must
|
||||
// not let a second tick double-fire.
|
||||
sessionNotified = true;
|
||||
void detect(onDetected);
|
||||
}
|
||||
|
||||
async function detect(onDetected: DetectorOptions["onDetected"]): Promise<void> {
|
||||
if (!isNotificationCategoryEnabled("meeting_detection")) return;
|
||||
|
||||
const source = await findLikelyMeetingApp();
|
||||
if (!source) {
|
||||
// Mic in use but nothing call-capable is running (dictation, voice
|
||||
// memo, unknown recorder) — stay quiet to avoid noise.
|
||||
return;
|
||||
}
|
||||
|
||||
const calendarEvent = await findNearbyCalendarEvent();
|
||||
const eventSummary =
|
||||
typeof calendarEvent?.summary === "string" ? calendarEvent.summary.trim() : "";
|
||||
|
||||
const meeting: DetectedMeeting = {
|
||||
kind: source.kind,
|
||||
title: KIND_TITLES[source.kind],
|
||||
appName: source.app,
|
||||
noteTitle: eventSummary || defaultNoteTitle(source),
|
||||
...(calendarEvent ? { calendarEvent } : {}),
|
||||
};
|
||||
|
||||
console.log(
|
||||
`[MeetingDetect] ${meeting.title} (app: ${meeting.appName}` +
|
||||
(eventSummary ? `, calendar: "${eventSummary}")` : ")"),
|
||||
);
|
||||
onDetected(meeting);
|
||||
}
|
||||
|
||||
function defaultNoteTitle(source: { app: string; kind: DetectedMeetingKind }): string {
|
||||
if (source.kind === "huddle") return `${source.app} huddle`;
|
||||
if (source.kind === "call") return `${source.app} call`;
|
||||
return BROWSERS.includes(source.app) ? "Meeting" : `${source.app} meeting`;
|
||||
}
|
||||
|
||||
/** Match one process name to a platform, by case-insensitive prefix. */
|
||||
function matchProcessName(name: string): { app: string; kind: DetectedMeetingKind } | null {
|
||||
const lower = name.toLowerCase();
|
||||
// Safari captures through WebKit's out-of-process media stack.
|
||||
if (lower.startsWith("com.apple.webkit")) return { app: "Safari", kind: "meeting" };
|
||||
// Firefox media capture lives in plugin-container.
|
||||
if (lower.startsWith("plugin-container")) return { app: "Firefox", kind: "meeting" };
|
||||
for (const candidate of MEETING_APPS) {
|
||||
if (lower.startsWith(candidate.proc.toLowerCase())) {
|
||||
return { app: candidate.app, kind: candidate.kind };
|
||||
}
|
||||
}
|
||||
for (const browser of BROWSERS) {
|
||||
if (lower.startsWith(browser.toLowerCase())) return { app: browser, kind: "meeting" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attribute the call to an app. Exact when mic-owner PIDs are known: resolve
|
||||
* their process names and match those (a Meet call in Chrome attributes to
|
||||
* Chrome even while Slack/Zoom idle in the background). If owners are known
|
||||
* but none is call-capable (voice memo, dictation, screen recorder), that's
|
||||
* NOT a meeting — stay quiet. Only without PID info (pre-14.4 macOS) fall
|
||||
* back to the running-app heuristic.
|
||||
*/
|
||||
async function findLikelyMeetingApp(): Promise<{ app: string; kind: DetectedMeetingKind } | null> {
|
||||
if (micPids.length > 0) {
|
||||
const owners = await processNamesForPids(micPids);
|
||||
if (owners.length > 0) {
|
||||
// Prefer dedicated apps over browsers when several own the mic.
|
||||
const matches = owners
|
||||
.map(matchProcessName)
|
||||
.filter((m): m is NonNullable<typeof m> => m !== null);
|
||||
const dedicated = matches.find((m) => !BROWSERS.includes(m.app));
|
||||
return dedicated ?? matches[0] ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
// No attribution available — running-app heuristic in table order
|
||||
// (dedicated conferencing apps beat always-running chat apps beat browsers).
|
||||
const names = await runningProcessNames();
|
||||
if (!names) return null;
|
||||
const hasPrefix = (prefix: string) => {
|
||||
const lower = prefix.toLowerCase();
|
||||
for (const name of names) {
|
||||
if (name.toLowerCase().startsWith(lower)) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
for (const candidate of MEETING_APPS) {
|
||||
if (hasPrefix(candidate.proc)) return { app: candidate.app, kind: candidate.kind };
|
||||
}
|
||||
for (const browser of BROWSERS) {
|
||||
if (hasPrefix(browser)) return { app: browser, kind: "meeting" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function processNamesForPids(pids: number[]): Promise<string[]> {
|
||||
return new Promise((resolve) => {
|
||||
execFile(
|
||||
"ps",
|
||||
["-o", "comm=", "-p", pids.join(",")],
|
||||
{ maxBuffer: 1024 * 1024 },
|
||||
(err, stdout) => {
|
||||
if (err) {
|
||||
resolve([]);
|
||||
return;
|
||||
}
|
||||
const names: string[] = [];
|
||||
for (const line of stdout.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
// GUI apps list as full executable paths — use the basename.
|
||||
names.push(path.basename(trimmed));
|
||||
}
|
||||
resolve(names);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function runningProcessNames(): Promise<Set<string> | null> {
|
||||
return new Promise((resolve) => {
|
||||
execFile("ps", ["-axo", "comm="], { maxBuffer: 4 * 1024 * 1024 }, (err, stdout) => {
|
||||
if (err) {
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
const names = new Set<string>();
|
||||
for (const line of stdout.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
// GUI apps list as full executable paths — match the basename.
|
||||
names.add(path.basename(trimmed));
|
||||
}
|
||||
resolve(names);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
interface CalendarEvent {
|
||||
status?: string;
|
||||
summary?: string;
|
||||
start?: { dateTime?: string };
|
||||
end?: { dateTime?: string };
|
||||
attendees?: Array<{ self?: boolean; responseStatus?: string }>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* The calendar event (if any) whose window covers "now": from 15 minutes
|
||||
* before its start through its end. Skips all-day, cancelled, and
|
||||
* self-declined events. Ties go to the event that started most recently.
|
||||
*/
|
||||
async function findNearbyCalendarEvent(): Promise<CalendarEvent | null> {
|
||||
let files: string[];
|
||||
try {
|
||||
files = await fs.readdir(CALENDAR_SYNC_DIR);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
let best: { event: CalendarEvent; startMs: number } | null = null;
|
||||
|
||||
for (const name of files) {
|
||||
if (!name.endsWith(".json") || name.startsWith("sync_state")) continue;
|
||||
let event: CalendarEvent;
|
||||
try {
|
||||
event = JSON.parse(await fs.readFile(path.join(CALENDAR_SYNC_DIR, name), "utf-8"));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (event.status === "cancelled") continue;
|
||||
const startStr = event.start?.dateTime;
|
||||
if (!startStr) continue; // all-day
|
||||
const self = event.attendees?.find((a) => a.self);
|
||||
if (self?.responseStatus === "declined") continue;
|
||||
|
||||
const startMs = Date.parse(startStr);
|
||||
if (!Number.isFinite(startMs)) continue;
|
||||
const endStr = event.end?.dateTime;
|
||||
const endMs = endStr ? Date.parse(endStr) : startMs + 60 * 60_000;
|
||||
|
||||
if (now < startMs - CALENDAR_MERGE_LEAD_MS || now > endMs) continue;
|
||||
if (!best || startMs > best.startMs) best = { event, startMs };
|
||||
}
|
||||
|
||||
return best?.event ?? null;
|
||||
}
|
||||
|
|
@ -734,6 +734,9 @@ const ipcSchemas = {
|
|||
// When true, the renderer should also open the meeting URL (Zoom/Meet/etc.)
|
||||
// in addition to triggering the take-notes flow.
|
||||
openMeeting: z.boolean().optional(),
|
||||
// Origin recorded in the note frontmatter: 'calendar-sync' (default) for
|
||||
// notification/deep-link starts, 'detected' for ambient meeting detection.
|
||||
source: z.string().optional(),
|
||||
}),
|
||||
res: z.null(),
|
||||
},
|
||||
|
|
@ -783,6 +786,48 @@ const ipcSchemas = {
|
|||
success: z.literal(true),
|
||||
}),
|
||||
},
|
||||
// Renderer → main: assistant voice/video call holds the mic — suppresses
|
||||
// ambient meeting detection (it would otherwise see our own capture).
|
||||
'voice:setCallActive': {
|
||||
req: z.object({
|
||||
active: z.boolean(),
|
||||
}),
|
||||
res: z.object({
|
||||
success: z.literal(true),
|
||||
}),
|
||||
},
|
||||
// --- Ambient meeting detection popup (own always-on-top window) ---
|
||||
// Main → popup: the detection to display.
|
||||
'meetingDetect:payload': {
|
||||
req: z.object({
|
||||
title: z.string(),
|
||||
message: z.string(),
|
||||
// Calendar-linked detections render a solid accent bar; ad-hoc ones a
|
||||
// dashed one (Granola's affordance).
|
||||
hasCalendarEvent: z.boolean(),
|
||||
}),
|
||||
res: z.null(),
|
||||
},
|
||||
// Popup → main: fetch the payload (the push can race listener registration).
|
||||
'meetingDetect:getPayload': {
|
||||
req: z.null(),
|
||||
res: z.object({
|
||||
payload: z
|
||||
.object({
|
||||
title: z.string(),
|
||||
message: z.string(),
|
||||
hasCalendarEvent: z.boolean(),
|
||||
})
|
||||
.nullable(),
|
||||
}),
|
||||
},
|
||||
// Popup → main: user clicked a button.
|
||||
'meetingDetect:action': {
|
||||
req: z.object({
|
||||
action: z.enum(['take-notes', 'dismiss']),
|
||||
}),
|
||||
res: z.object({}),
|
||||
},
|
||||
'granola:getConfig': {
|
||||
req: z.null(),
|
||||
res: z.object({
|
||||
|
|
|
|||
|
|
@ -7,12 +7,14 @@ import { z } from 'zod';
|
|||
* - new_email: a new email arrived during incremental Gmail sync
|
||||
* - agent_permission: an agent is requesting permission to run a tool
|
||||
* - background_task: a background task agent pinged via the notify-user tool
|
||||
* - meeting_detection: popup when Rowboat detects you're in a call/meeting
|
||||
*/
|
||||
export const NotificationCategorySchema = z.enum([
|
||||
'chat_completion',
|
||||
'new_email',
|
||||
'agent_permission',
|
||||
'background_task',
|
||||
'meeting_detection',
|
||||
]);
|
||||
|
||||
export const NotificationCategoriesSchema = z.object({
|
||||
|
|
@ -20,6 +22,7 @@ export const NotificationCategoriesSchema = z.object({
|
|||
new_email: z.boolean(),
|
||||
agent_permission: z.boolean(),
|
||||
background_task: z.boolean(),
|
||||
meeting_detection: z.boolean(),
|
||||
});
|
||||
|
||||
export const NotificationSettingsSchema = z.object({
|
||||
|
|
@ -32,6 +35,7 @@ export const DEFAULT_NOTIFICATION_SETTINGS: NotificationSettings = {
|
|||
new_email: true,
|
||||
agent_permission: true,
|
||||
background_task: true,
|
||||
meeting_detection: true,
|
||||
},
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue