From 9cd4daa6b394e33b9a3325baa59c6244dce59d64 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 27 Apr 2026 20:35:47 +0200 Subject: [PATCH] Add a single-session desktop window picker and route screenshot assist, region crop, and fullscreen capture through the cached frame. --- surfsense_desktop/scripts/build-electron.mjs | 6 + surfsense_desktop/src/ipc/channels.ts | 3 + surfsense_desktop/src/ipc/handlers.ts | 5 +- .../src/modules/screen-region-picker.ts | 174 ++++++++----- .../src/modules/screenshot-assist.ts | 14 +- .../src/modules/window-picker.ts | 244 ++++++++++++++++++ .../src/window-picker-preload.ts | 15 ++ 7 files changed, 396 insertions(+), 65 deletions(-) create mode 100644 surfsense_desktop/src/modules/window-picker.ts create mode 100644 surfsense_desktop/src/window-picker-preload.ts diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index 0c8f08d52..ca17e4c48 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -138,6 +138,12 @@ async function buildElectron() { outfile: 'dist/screen-region-preload.js', }); + await build({ + ...shared, + entryPoints: ['src/window-picker-preload.ts'], + outfile: 'dist/window-picker-preload.js', + }); + console.log('Electron build complete'); resolveStandaloneSymlinks(); } diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index 9f084af85..1007e3a37 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -14,6 +14,9 @@ export const IPC_CHANNELS = { CAPTURE_FULL_SCREEN: 'capture-full-screen', SCREEN_REGION_SUBMIT: 'screen-region:submit', SCREEN_REGION_CANCEL: 'screen-region:cancel', + WINDOW_PICK_LIST: 'window-pick:list', + WINDOW_PICK_SUBMIT: 'window-pick:submit', + WINDOW_PICK_CANCEL: 'window-pick:cancel', CHAT_SCREEN_CAPTURE: 'chat:screen-capture', // Folder sync channels FOLDER_SYNC_SELECT_FOLDER: 'folder-sync:select-folder', diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts index 8361b9a38..d68d4a5bf 100644 --- a/surfsense_desktop/src/ipc/handlers.ts +++ b/surfsense_desktop/src/ipc/handlers.ts @@ -7,7 +7,7 @@ import { requestScreenRecording, restartApp, } from '../modules/permissions'; -import { captureCurrentDisplayDataUrl } from '../modules/screen-region-picker'; +import { pickOpenWindowCapture } from '../modules/window-picker'; import { selectFolder, addWatchedFolder, @@ -85,7 +85,8 @@ export function registerIpcHandlers(): void { requestScreenRecording(); return null; } - return captureCurrentDisplayDataUrl(); + const picked = await pickOpenWindowCapture(); + return picked?.dataUrl ?? null; }); // Folder sync handlers diff --git a/surfsense_desktop/src/modules/screen-region-picker.ts b/surfsense_desktop/src/modules/screen-region-picker.ts index cc9303040..1c4b77195 100644 --- a/surfsense_desktop/src/modules/screen-region-picker.ts +++ b/surfsense_desktop/src/modules/screen-region-picker.ts @@ -1,6 +1,17 @@ import { BrowserWindow, desktopCapturer, nativeImage, screen } from 'electron'; import path from 'path'; import { IPC_CHANNELS } from '../ipc/channels'; +function fitNativeImageToWorkArea(img: Electron.NativeImage, display: Electron.Display): Electron.NativeImage { + const wa = display.workArea; + const { width: iw, height: ih } = img.getSize(); + const scale = Math.min(1, wa.width / iw, wa.height / ih); + if (scale >= 1) return img; + return img.resize({ + width: Math.max(1, Math.floor(iw * scale)), + height: Math.max(1, Math.floor(ih * scale)), + quality: 'best', + }); +} // One getSources per pick; overlay and final crop share that bitmap (avoids a second portal session, e.g. Wayland). @@ -141,7 +152,7 @@ function buildInjectScript(dataUrl: string, iw: number, ih: number): string { })();`; } -export function pickScreenRegion(): Promise { +export function pickScreenRegion(opts?: { windowDataUrl?: string }): Promise { if (pickInProgress) return Promise.resolve(null); pickInProgress = true; @@ -175,6 +186,7 @@ export function pickScreenRegion(): Promise { }; let snapshot: DisplayCaptureSnapshot | null = null; + let cropSource: Electron.NativeImage | null = null; const onSubmit = ( _event: Electron.IpcMainEvent, @@ -185,17 +197,25 @@ export function pickScreenRegion(): Promise { finish(null); return; } - if (!snapshot) { + if (!snapshot || !cropSource) { finish(null); return; } try { - const full = nativeImage.createFromDataURL(snapshot.dataUrl); - const cropped = full.crop({ - x: Math.floor(rect.x), - y: Math.floor(rect.y), - width: Math.floor(rect.width), - height: Math.floor(rect.height), + const iw = snapshot.width; + const ih = snapshot.height; + const { width: cw, height: ch } = cropSource.getSize(); + const scaleX = cw / iw; + const scaleY = ch / ih; + const ox = Math.floor(rect.x * scaleX); + const oy = Math.floor(rect.y * scaleY); + const ow = Math.min(Math.floor(rect.width * scaleX), cw - ox); + const oh = Math.min(Math.floor(rect.height * scaleY), ch - oy); + const cropped = cropSource.crop({ + x: ox, + y: oy, + width: Math.max(1, ow), + height: Math.max(1, oh), }); finish(cropped.toDataURL()); } catch { @@ -214,66 +234,102 @@ export function pickScreenRegion(): Promise { } }; - void captureDisplaySnapshot(display) - .then((cap) => { + const openOverlay = ( + cap: DisplayCaptureSnapshot, + crop: Electron.NativeImage, + bounds: { x: number; y: number; width: number; height: number } + ) => { + snapshot = cap; + cropSource = crop; + + overlay = new BrowserWindow({ + x: bounds.x, + y: bounds.y, + width: bounds.width, + height: bounds.height, + frame: false, + transparent: true, + fullscreenable: false, + skipTaskbar: true, + alwaysOnTop: true, + focusable: true, + show: false, + autoHideMenuBar: true, + backgroundColor: '#00000000', + webPreferences: { + preload: path.join(__dirname, 'screen-region-preload.js'), + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + }); + + overlayWc = overlay.webContents; + overlayWc.on('before-input-event', onBeforeInput); + overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_SUBMIT, onSubmit); + overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_CANCEL, onCancel); + + overlay.setIgnoreMouseEvents(false); + overlay.loadURL( + 'data:text/html;charset=utf-8,' + + encodeURIComponent('') + ); + + overlay.on('closed', () => { + if (!settled) finish(null); + }); + + overlay.webContents.once('did-finish-load', () => { + if (!overlay || overlay.isDestroyed()) return; + overlay.webContents + .executeJavaScript(buildInjectScript(cap.dataUrl, cap.width, cap.height), true) + .then(() => { + overlay?.show(); + overlay?.focus(); + }) + .catch(() => { + finish(null); + }); + }); + }; + + void (async () => { + try { + if (opts?.windowDataUrl) { + const fullRes = nativeImage.createFromDataURL(opts.windowDataUrl); + if (fullRes.isEmpty()) { + finish(null); + return; + } + const fitted = fitNativeImageToWorkArea(fullRes, display); + const fw = fitted.getSize().width; + const fh = fitted.getSize().height; + const wa = display.workArea; + const x = wa.x + Math.floor((wa.width - fw) / 2); + const y = wa.y + Math.floor((wa.height - fh) / 2); + openOverlay( + { dataUrl: fitted.toDataURL(), width: fw, height: fh }, + fullRes, + { x, y, width: fw, height: fh } + ); + return; + } + + const cap = await captureDisplaySnapshot(display); if (!cap) { finish(null); return; } - snapshot = cap; - - overlay = new BrowserWindow({ + const crop = nativeImage.createFromDataURL(cap.dataUrl); + openOverlay(cap, crop, { x: display.bounds.x, y: display.bounds.y, width: display.bounds.width, height: display.bounds.height, - frame: false, - transparent: true, - fullscreenable: false, - skipTaskbar: true, - alwaysOnTop: true, - focusable: true, - show: false, - autoHideMenuBar: true, - backgroundColor: '#00000000', - webPreferences: { - preload: path.join(__dirname, 'screen-region-preload.js'), - contextIsolation: true, - nodeIntegration: false, - sandbox: true, - }, }); - - overlayWc = overlay.webContents; - overlayWc.on('before-input-event', onBeforeInput); - overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_SUBMIT, onSubmit); - overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_CANCEL, onCancel); - - overlay.setIgnoreMouseEvents(false); - overlay.loadURL( - 'data:text/html;charset=utf-8,' + - encodeURIComponent('') - ); - - overlay.on('closed', () => { - if (!settled) finish(null); - }); - - overlay.webContents.once('did-finish-load', () => { - if (!overlay || overlay.isDestroyed()) return; - overlay.webContents - .executeJavaScript(buildInjectScript(cap.dataUrl, cap.width, cap.height), true) - .then(() => { - overlay?.show(); - overlay?.focus(); - }) - .catch(() => { - finish(null); - }); - }); - }) - .catch(() => { + } catch { finish(null); - }); + } + })(); }); } diff --git a/surfsense_desktop/src/modules/screenshot-assist.ts b/surfsense_desktop/src/modules/screenshot-assist.ts index 2500bf1d5..34fd0f489 100644 --- a/surfsense_desktop/src/modules/screenshot-assist.ts +++ b/surfsense_desktop/src/modules/screenshot-assist.ts @@ -1,19 +1,25 @@ import { IPC_CHANNELS } from '../ipc/channels'; import { trackEvent } from './analytics'; import { pickScreenRegion } from './screen-region-picker'; +import { pickOpenWindowCapture } from './window-picker'; import { getMainWindow, showMainWindow } from './window'; import { hasScreenRecordingPermission, requestScreenRecording } from './permissions'; export async function runScreenshotAssistShortcut(): Promise { - showMainWindow('shortcut'); - await new Promise((r) => setTimeout(r, 400)); if (!hasScreenRecordingPermission()) { requestScreenRecording(); return; } - const url = await pickScreenRegion(); + + const picked = await pickOpenWindowCapture(); + if (!picked) return; + + const url = await pickScreenRegion({ windowDataUrl: picked.dataUrl }); + if (!url) return; + + showMainWindow('shortcut'); const mw = getMainWindow(); - if (url && mw && !mw.isDestroyed()) { + if (mw && !mw.isDestroyed()) { mw.webContents.send(IPC_CHANNELS.CHAT_SCREEN_CAPTURE, url); trackEvent('desktop_screenshot_assist_region_to_chat', {}); } diff --git a/surfsense_desktop/src/modules/window-picker.ts b/surfsense_desktop/src/modules/window-picker.ts new file mode 100644 index 000000000..0e8505bcb --- /dev/null +++ b/surfsense_desktop/src/modules/window-picker.ts @@ -0,0 +1,244 @@ +import { BrowserWindow, desktopCapturer, ipcMain, screen } from 'electron'; +import path from 'path'; +import { IPC_CHANNELS } from '../ipc/channels'; + +let pickInProgress = false; + +const PREVIEW_THUMB = { width: 280, height: 180 } as const; + +function maxCaptureThumbSize(): { width: number; height: number } { + const d = screen.getPrimaryDisplay(); + const sf = d.scaleFactor || 1; + const w = Math.min(3840, Math.max(1280, Math.round(d.size.width * sf))); + const h = Math.min(2160, Math.max(720, Math.round(d.size.height * sf))); + return { width: w, height: h }; +} + +function isDesktopWindowSourceId(s: string): boolean { + return typeof s === 'string' && s.startsWith('window:'); +} + +export type PickedWindowResult = { + sourceId: string; + /** Same pixels as the one `desktopCapturer` snapshot (max thumbnail size). */ + dataUrl: string; +}; + +function buildPickerInjectScript(): string { + return `(async function () { + const api = window.surfsenseWindowPick; + if (!api) return; + const items = await api.list(); + document.body.style.cssText = + 'margin:0;font-family:system-ui,-apple-system,sans-serif;background:#0f172a;color:#e2e8f0;min-height:100vh;padding:16px;box-sizing:border-box;'; + const top = document.createElement('div'); + top.style.cssText = + 'display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;flex-wrap:wrap;gap:8px;'; + const t = document.createElement('strong'); + t.textContent = 'Open windows'; + const hint = document.createElement('span'); + hint.style.cssText = 'opacity:0.75;font-size:13px;'; + hint.textContent = 'Click a window · Esc to cancel'; + top.appendChild(t); + top.appendChild(hint); + document.body.appendChild(top); + if (!items || !items.length) { + const p = document.createElement('p'); + p.style.cssText = 'line-height:1.5;max-width:42rem;'; + p.textContent = + 'No windows were returned by the system. On Linux, allow screen capture when prompted. If other apps are open, try again.'; + document.body.appendChild(p); + return; + } + const grid = document.createElement('div'); + grid.style.cssText = + 'display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:12px;max-height:calc(100vh - 88px);overflow:auto;padding-bottom:8px;'; + for (const it of items) { + const card = document.createElement('button'); + card.type = 'button'; + card.style.cssText = + 'text-align:left;background:#1e293b;border:1px solid #334155;border-radius:8px;padding:8px;cursor:pointer;color:inherit;'; + card.addEventListener('mouseenter', function () { + card.style.borderColor = '#38bdf8'; + }); + card.addEventListener('mouseleave', function () { + card.style.borderColor = '#334155'; + }); + const img = document.createElement('img'); + img.alt = ''; + img.src = + it.thumbDataUrl || + 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=='; + img.style.cssText = + 'width:100%;height:100px;object-fit:cover;border-radius:4px;background:#000;display:block;'; + const cap = document.createElement('div'); + cap.textContent = it.name || '(untitled)'; + cap.style.cssText = + 'margin-top:6px;font-size:12px;line-height:1.35;overflow:hidden;text-overflow:ellipsis;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;'; + card.appendChild(img); + card.appendChild(cap); + card.addEventListener('click', function () { + api.submit(it.id); + }); + grid.appendChild(card); + } + document.body.appendChild(grid); + window.addEventListener('keydown', function (e) { + if (e.key === 'Escape') api.cancel(); + }); + })();`; +} + +/** + * One OS / Chromium capture session: `getSources` runs once (important on Wayland / + * PipeWire so the portal is not opened again for the same flow). Opens our grid to + * choose a window; resolves with the chosen snapshot for region or full-frame use. + */ +export function pickOpenWindowCapture(): Promise { + if (pickInProgress) return Promise.resolve(null); + pickInProgress = true; + + return new Promise((resolve) => { + let settled = false; + let picker: BrowserWindow | null = null; + let pickerWc: Electron.WebContents | null = null; + /** Filled once before the grid runs — reused for list + final image (no second getSources). */ + let sessionSources: Electron.DesktopCapturerSource[] = []; + + const finish = (result: PickedWindowResult | null) => { + if (settled) return; + settled = true; + pickInProgress = false; + ipcMain.removeHandler(IPC_CHANNELS.WINDOW_PICK_LIST); + const wc = pickerWc; + pickerWc = null; + if (wc && !wc.isDestroyed()) { + wc.removeListener('before-input-event', onBeforeInput); + wc.ipc.removeListener(IPC_CHANNELS.WINDOW_PICK_SUBMIT, onSubmit); + wc.ipc.removeListener(IPC_CHANNELS.WINDOW_PICK_CANCEL, onCancel); + } + if (picker && !picker.isDestroyed()) { + picker.removeAllListeners('closed'); + picker.close(); + } + picker = null; + resolve(result); + }; + + const onSubmit = (_event: Electron.IpcMainEvent, sourceId: string) => { + if (settled || !picker || picker.isDestroyed()) return; + if (!isDesktopWindowSourceId(sourceId)) { + finish(null); + return; + } + const hit = sessionSources.find((s) => s.id === sourceId); + if (!hit || hit.thumbnail.isEmpty()) { + finish(null); + return; + } + finish({ sourceId, dataUrl: hit.thumbnail.toDataURL() }); + }; + + const onCancel = () => { + if (settled || !picker || picker.isDestroyed()) return; + finish(null); + }; + + const onBeforeInput = (_event: Electron.Event, input: Electron.Input) => { + if (input.type === 'keyDown' && input.key === 'Escape') { + finish(null); + } + }; + + ipcMain.handle(IPC_CHANNELS.WINDOW_PICK_LIST, async () => { + return sessionSources.map((s, i) => { + let thumbDataUrl = ''; + if (!s.thumbnail.isEmpty()) { + try { + const sm = s.thumbnail.resize({ + width: PREVIEW_THUMB.width, + height: PREVIEW_THUMB.height, + quality: 'good', + }); + thumbDataUrl = sm.toDataURL(); + } catch { + thumbDataUrl = s.thumbnail.toDataURL(); + } + } + return { + id: s.id, + name: (s.name || '').trim() || `Window ${i + 1}`, + thumbDataUrl, + }; + }); + }); + + picker = new BrowserWindow({ + width: 760, + height: 560, + show: false, + center: true, + autoHideMenuBar: true, + title: 'SurfSense — choose window', + webPreferences: { + preload: path.join(__dirname, 'window-picker-preload.js'), + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + }); + + pickerWc = picker.webContents; + + pickerWc.on('before-input-event', onBeforeInput); + pickerWc.ipc.on(IPC_CHANNELS.WINDOW_PICK_SUBMIT, onSubmit); + pickerWc.ipc.on(IPC_CHANNELS.WINDOW_PICK_CANCEL, onCancel); + + picker.on('closed', () => { + if (!settled) finish(null); + }); + + picker + .loadURL( + 'data:text/html;charset=utf-8,' + + encodeURIComponent('') + ) + .catch(() => finish(null)); + + picker.webContents.once('did-finish-load', () => { + void (async () => { + if (!picker || picker.isDestroyed()) return; + let selfId = ''; + try { + selfId = picker.getMediaSourceId(); + } catch { + selfId = ''; + } + try { + const { width, height } = maxCaptureThumbSize(); + const sources = await desktopCapturer.getSources({ + types: ['window'], + thumbnailSize: { width, height }, + fetchWindowIcons: false, + }); + sessionSources = sources.filter((s) => !(selfId && s.id === selfId)); + } catch { + sessionSources = []; + } + if (sessionSources.length === 1) { + const only = sessionSources[0]; + if (!only.thumbnail.isEmpty()) { + finish({ sourceId: only.id, dataUrl: only.thumbnail.toDataURL() }); + return; + } + } + try { + await picker.webContents.executeJavaScript(buildPickerInjectScript(), true); + if (!picker.isDestroyed()) picker.show(); + } catch { + finish(null); + } + })(); + }); + }); +} diff --git a/surfsense_desktop/src/window-picker-preload.ts b/surfsense_desktop/src/window-picker-preload.ts new file mode 100644 index 000000000..9b582cede --- /dev/null +++ b/surfsense_desktop/src/window-picker-preload.ts @@ -0,0 +1,15 @@ +import { contextBridge, ipcRenderer } from 'electron'; +import { IPC_CHANNELS } from './ipc/channels'; + +contextBridge.exposeInMainWorld('surfsenseWindowPick', { + list: () => + ipcRenderer.invoke(IPC_CHANNELS.WINDOW_PICK_LIST) as Promise< + { id: string; name: string; thumbDataUrl: string }[] + >, + submit: (sourceId: string) => { + ipcRenderer.send(IPC_CHANNELS.WINDOW_PICK_SUBMIT, sourceId); + }, + cancel: () => { + ipcRenderer.send(IPC_CHANNELS.WINDOW_PICK_CANCEL); + }, +});