From 7097f542fb878e8a277a1901305f73d24ac0d20d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 24 Apr 2026 19:13:14 +0200 Subject: [PATCH] Add native screen region capture modules and preload build --- surfsense_desktop/scripts/build-electron.mjs | 6 + .../src/modules/general-assist.ts | 21 ++ .../src/modules/screen-region-picker.ts | 271 ++++++++++++++++++ .../src/screen-region-preload.ts | 11 + 4 files changed, 309 insertions(+) create mode 100644 surfsense_desktop/src/modules/general-assist.ts create mode 100644 surfsense_desktop/src/modules/screen-region-picker.ts create mode 100644 surfsense_desktop/src/screen-region-preload.ts diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index 90d76ef7a..0c8f08d52 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -132,6 +132,12 @@ async function buildElectron() { outfile: 'dist/preload.js', }); + await build({ + ...shared, + entryPoints: ['src/screen-region-preload.ts'], + outfile: 'dist/screen-region-preload.js', + }); + console.log('Electron build complete'); resolveStandaloneSymlinks(); } diff --git a/surfsense_desktop/src/modules/general-assist.ts b/surfsense_desktop/src/modules/general-assist.ts new file mode 100644 index 000000000..9d39f068a --- /dev/null +++ b/surfsense_desktop/src/modules/general-assist.ts @@ -0,0 +1,21 @@ +import { IPC_CHANNELS } from '../ipc/channels'; +import { trackEvent } from './analytics'; +import { pickScreenRegion } from './screen-region-picker'; +import { getMainWindow, showMainWindow } from './window'; +import { hasScreenRecordingPermission, requestScreenRecording } from './permissions'; + +export async function runGeneralAssistShortcut(): Promise { + console.log('[general-assist] Shortcut triggered'); + showMainWindow('shortcut'); + await new Promise((r) => setTimeout(r, 400)); + if (!hasScreenRecordingPermission()) { + requestScreenRecording(); + return; + } + const url = await pickScreenRegion(); + const mw = getMainWindow(); + if (url && mw && !mw.isDestroyed()) { + mw.webContents.send(IPC_CHANNELS.CHAT_SCREEN_CAPTURE, url); + trackEvent('desktop_screen_region_to_chat', {}); + } +} diff --git a/surfsense_desktop/src/modules/screen-region-picker.ts b/surfsense_desktop/src/modules/screen-region-picker.ts new file mode 100644 index 000000000..0a924eec9 --- /dev/null +++ b/surfsense_desktop/src/modules/screen-region-picker.ts @@ -0,0 +1,271 @@ +import { BrowserWindow, desktopCapturer, nativeImage, screen } from 'electron'; +import path from 'path'; +import { IPC_CHANNELS } from '../ipc/channels'; + +// One getSources per pick; overlay and final crop share that bitmap (avoids a second portal session, e.g. Wayland). + +let pickInProgress = false; + +async function captureDisplayDataUrl(display: Electron.Display): Promise<{ + dataUrl: string; + width: number; + height: number; +} | null> { + try { + const sf = display.scaleFactor || 1; + const tw = Math.max(1, Math.round(display.size.width * sf)); + const th = Math.max(1, Math.round(display.size.height * sf)); + const sources = await desktopCapturer.getSources({ + types: ['screen'], + thumbnailSize: { width: tw, height: th }, + }); + if (!sources.length) return null; + const idStr = String(display.id); + let chosen = + sources.find((s) => s.display_id === idStr) || + sources.find((s) => s.display_id && s.display_id === idStr) || + null; + if (!chosen && screen.getPrimaryDisplay().id === display.id) { + chosen = sources[0]; + } + if (!chosen) chosen = sources[0]; + const dataUrl = chosen.thumbnail.toDataURL(); + const { width, height } = chosen.thumbnail.getSize(); + return { dataUrl, width, height }; + } catch { + return null; + } +} + +function buildInjectScript(dataUrl: string, iw: number, ih: number): string { + return `(() => { + const api = window.surfsenseScreenRegion; + if (!api) return; + const dataUrl = ${JSON.stringify(dataUrl)}; + const iw = ${iw}; + const ih = ${ih}; + document.body.style.margin = '0'; + document.body.style.overflow = 'hidden'; + document.body.style.background = '#000'; + const img = document.createElement('img'); + img.draggable = false; + img.src = dataUrl; + img.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;object-fit:fill;user-select:none;pointer-events:none;'; + const veil = document.createElement('div'); + veil.style.cssText = 'position:fixed;inset:0;cursor:crosshair;background:rgba(0,0,0,0.15);'; + const sel = document.createElement('div'); + sel.style.cssText = 'position:fixed;border:2px solid #38bdf8;box-shadow:0 0 0 9999px rgba(0,0,0,0.45);display:none;pointer-events:none;z-index:2;'; + document.body.appendChild(img); + document.body.appendChild(veil); + document.body.appendChild(sel); + let ax = 0, ay = 0, dragging = false; + function show(x0, y0, x1, y1) { + const l = Math.min(x0, x1), t = Math.min(y0, y1); + const w = Math.abs(x1 - x0), h = Math.abs(y1 - y0); + if (w < 2 || h < 2) { sel.style.display = 'none'; return; } + sel.style.display = 'block'; + sel.style.left = l + 'px'; + sel.style.top = t + 'px'; + sel.style.width = w + 'px'; + sel.style.height = h + 'px'; + } + function mapRect(l, t, w, h) { + const vw = window.innerWidth, vh = window.innerHeight; + const sx = Math.round((l / vw) * iw); + const sy = Math.round((t / vh) * ih); + const sw = Math.max(1, Math.round((w / vw) * iw)); + const sh = Math.max(1, Math.round((h / vh) * ih)); + const cx = Math.min(Math.max(0, sx), iw - 1); + const cy = Math.min(Math.max(0, sy), ih - 1); + const cw = Math.min(sw, iw - cx); + const ch = Math.min(sh, ih - cy); + return { x: cx, y: cy, width: cw, height: ch }; + } + function endDrag(clientX, clientY, pointerId) { + if (!dragging) return; + dragging = false; + if (typeof pointerId === 'number' && pointerId >= 0) { + try { veil.releasePointerCapture(pointerId); } catch (_) {} + } + const l = Math.min(ax, clientX), t = Math.min(ay, clientY); + const w = Math.abs(clientX - ax), h = Math.abs(clientY - ay); + if (w < 4 || h < 4) { sel.style.display = 'none'; return; } + api.submit(mapRect(l, t, w, h)); + } + veil.addEventListener('pointerdown', (e) => { + if (e.button !== 0) return; + try { veil.setPointerCapture(e.pointerId); } catch (_) {} + dragging = true; + ax = e.clientX; ay = e.clientY; + show(ax, ay, ax, ay); + }); + veil.addEventListener('pointermove', (e) => { + if (!dragging) return; + show(ax, ay, e.clientX, e.clientY); + }); + veil.addEventListener('pointerup', (e) => { + endDrag(e.clientX, e.clientY, e.pointerId); + }); + window.addEventListener('pointerup', (e) => { + endDrag(e.clientX, e.clientY, e.pointerId); + }); + document.addEventListener( + 'mouseup', + (e) => { + endDrag(e.clientX, e.clientY, -1); + }, + true + ); + veil.addEventListener('pointercancel', (e) => { + if (!dragging) return; + dragging = false; + try { veil.releasePointerCapture(e.pointerId); } catch (_) {} + sel.style.display = 'none'; + }); + window.addEventListener('keydown', (e) => { + if (e.key === 'Escape') { api.cancel(); return; } + if (e.key === 'Enter' && sel.style.display === 'block') { + const l = parseFloat(sel.style.left), t = parseFloat(sel.style.top); + const w = parseFloat(sel.style.width), h = parseFloat(sel.style.height); + if (w >= 4 && h >= 4) api.submit(mapRect(l, t, w, h)); + } + }); + })();`; +} + +export function pickScreenRegion(): Promise { + if (pickInProgress) return Promise.resolve(null); + pickInProgress = true; + + return new Promise((resolve) => { + const display = screen.getDisplayNearestPoint(screen.getCursorScreenPoint()); + let settled = false; + let overlay: BrowserWindow | null = null; + /** webContents for listener removal after `BrowserWindow` may already be destroyed. */ + let overlayWc: Electron.WebContents | null = null; + + const cleanupListeners = () => { + const wc = overlayWc; + overlayWc = null; + if (!wc || wc.isDestroyed()) return; + wc.removeListener('before-input-event', onBeforeInput); + wc.ipc.removeListener(IPC_CHANNELS.SCREEN_REGION_SUBMIT, onSubmit); + wc.ipc.removeListener(IPC_CHANNELS.SCREEN_REGION_CANCEL, onCancel); + }; + + const finish = (result: string | null) => { + if (settled) return; + settled = true; + pickInProgress = false; + cleanupListeners(); + if (overlay && !overlay.isDestroyed()) { + overlay.removeAllListeners('closed'); + overlay.close(); + } + overlay = null; + resolve(result); + }; + + let snapshot: { dataUrl: string; width: number; height: number } | null = null; + + const onSubmit = ( + _event: Electron.IpcMainEvent, + rect: { x: number; y: number; width: number; height: number } + ) => { + if (settled || !overlay || overlay.isDestroyed()) return; + if (!rect || rect.width < 1 || rect.height < 1) { + finish(null); + return; + } + if (!snapshot) { + finish(null); + return; + } + try { + const full = nativeImage.createFromDataURL(snapshot.dataUrl); + const cropped = full.crop({ + x: Math.floor(rect.x), + y: Math.floor(rect.y), + width: Math.floor(rect.width), + height: Math.floor(rect.height), + }); + finish(cropped.toDataURL()); + } catch { + finish(null); + } + }; + + const onCancel = (_event: Electron.IpcMainEvent) => { + if (settled || !overlay || overlay.isDestroyed()) return; + finish(null); + }; + + const onBeforeInput = (_event: Electron.Event, input: Electron.Input) => { + if (input.type === 'keyDown' && input.key === 'Escape') { + finish(null); + } + }; + + void captureDisplayDataUrl(display) + .then((cap) => { + if (!cap) { + finish(null); + return; + } + snapshot = cap; + + overlay = new BrowserWindow({ + x: display.bounds.x, + y: display.bounds.y, + width: display.bounds.width, + height: display.bounds.height, + frame: false, + transparent: true, + fullscreenable: false, + skipTaskbar: true, + alwaysOnTop: true, + focusable: true, + show: false, + autoHideMenuBar: true, + backgroundColor: '#00000000', + webPreferences: { + preload: path.join(__dirname, 'screen-region-preload.js'), + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + }); + + overlayWc = overlay.webContents; + overlayWc.on('before-input-event', onBeforeInput); + overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_SUBMIT, onSubmit); + overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_CANCEL, onCancel); + + overlay.setIgnoreMouseEvents(false); + overlay.loadURL( + 'data:text/html;charset=utf-8,' + + encodeURIComponent('') + ); + + overlay.on('closed', () => { + if (!settled) finish(null); + }); + + overlay.webContents.once('did-finish-load', () => { + if (!overlay || overlay.isDestroyed()) return; + overlay.webContents + .executeJavaScript(buildInjectScript(cap.dataUrl, cap.width, cap.height), true) + .then(() => { + overlay?.show(); + overlay?.focus(); + }) + .catch(() => { + finish(null); + }); + }); + }) + .catch(() => { + finish(null); + }); + }); +} diff --git a/surfsense_desktop/src/screen-region-preload.ts b/surfsense_desktop/src/screen-region-preload.ts new file mode 100644 index 000000000..6eab9f162 --- /dev/null +++ b/surfsense_desktop/src/screen-region-preload.ts @@ -0,0 +1,11 @@ +import { contextBridge, ipcRenderer } from 'electron'; +import { IPC_CHANNELS } from './ipc/channels'; + +contextBridge.exposeInMainWorld('surfsenseScreenRegion', { + submit: (rect: { x: number; y: number; width: number; height: number }) => { + ipcRenderer.send(IPC_CHANNELS.SCREEN_REGION_SUBMIT, rect); + }, + cancel: () => { + ipcRenderer.send(IPC_CHANNELS.SCREEN_REGION_CANCEL); + }, +});