Add native screen region capture modules and preload build

surfsense_desktop/scripts/build-electron.mjs

@@ -132,6 +132,12 @@ async function buildElectron() {
     outfile: 'dist/preload.js',
   });
 
+  await build({
+    ...shared,
+    entryPoints: ['src/screen-region-preload.ts'],
+    outfile: 'dist/screen-region-preload.js',
+  });
+
   console.log('Electron build complete');
   resolveStandaloneSymlinks();
 }

surfsense_desktop/src/modules/general-assist.ts

@@ -0,0 +1,21 @@
+import { IPC_CHANNELS } from '../ipc/channels';
+import { trackEvent } from './analytics';
+import { pickScreenRegion } from './screen-region-picker';
+import { getMainWindow, showMainWindow } from './window';
+import { hasScreenRecordingPermission, requestScreenRecording } from './permissions';
+
+export async function runGeneralAssistShortcut(): Promise<void> {
+  console.log('[general-assist] Shortcut triggered');
+  showMainWindow('shortcut');
+  await new Promise((r) => setTimeout(r, 400));
+  if (!hasScreenRecordingPermission()) {
+    requestScreenRecording();
+    return;
+  }
+  const url = await pickScreenRegion();
+  const mw = getMainWindow();
+  if (url && mw && !mw.isDestroyed()) {
+    mw.webContents.send(IPC_CHANNELS.CHAT_SCREEN_CAPTURE, url);
+    trackEvent('desktop_screen_region_to_chat', {});
+  }
+}

surfsense_desktop/src/modules/screen-region-picker.ts

@@ -0,0 +1,271 @@
+import { BrowserWindow, desktopCapturer, nativeImage, screen } from 'electron';
+import path from 'path';
+import { IPC_CHANNELS } from '../ipc/channels';
+
+// One getSources per pick; overlay and final crop share that bitmap (avoids a second portal session, e.g. Wayland).
+
+let pickInProgress = false;
+
+async function captureDisplayDataUrl(display: Electron.Display): Promise<{
+  dataUrl: string;
+  width: number;
+  height: number;
+} | null> {
+  try {
+    const sf = display.scaleFactor || 1;
+    const tw = Math.max(1, Math.round(display.size.width * sf));
+    const th = Math.max(1, Math.round(display.size.height * sf));
+    const sources = await desktopCapturer.getSources({
+      types: ['screen'],
+      thumbnailSize: { width: tw, height: th },
+    });
+    if (!sources.length) return null;
+    const idStr = String(display.id);
+    let chosen =
+      sources.find((s) => s.display_id === idStr) ||
+      sources.find((s) => s.display_id && s.display_id === idStr) ||
+      null;
+    if (!chosen && screen.getPrimaryDisplay().id === display.id) {
+      chosen = sources[0];
+    }
+    if (!chosen) chosen = sources[0];
+    const dataUrl = chosen.thumbnail.toDataURL();
+    const { width, height } = chosen.thumbnail.getSize();
+    return { dataUrl, width, height };
+  } catch {
+    return null;
+  }
+}
+
+function buildInjectScript(dataUrl: string, iw: number, ih: number): string {
+  return `(() => {
+    const api = window.surfsenseScreenRegion;
+    if (!api) return;
+    const dataUrl = ${JSON.stringify(dataUrl)};
+    const iw = ${iw};
+    const ih = ${ih};
+    document.body.style.margin = '0';
+    document.body.style.overflow = 'hidden';
+    document.body.style.background = '#000';
+    const img = document.createElement('img');
+    img.draggable = false;
+    img.src = dataUrl;
+    img.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;object-fit:fill;user-select:none;pointer-events:none;';
+    const veil = document.createElement('div');
+    veil.style.cssText = 'position:fixed;inset:0;cursor:crosshair;background:rgba(0,0,0,0.15);';
+    const sel = document.createElement('div');
+    sel.style.cssText = 'position:fixed;border:2px solid #38bdf8;box-shadow:0 0 0 9999px rgba(0,0,0,0.45);display:none;pointer-events:none;z-index:2;';
+    document.body.appendChild(img);
+    document.body.appendChild(veil);
+    document.body.appendChild(sel);
+    let ax = 0, ay = 0, dragging = false;
+    function show(x0, y0, x1, y1) {
+      const l = Math.min(x0, x1), t = Math.min(y0, y1);
+      const w = Math.abs(x1 - x0), h = Math.abs(y1 - y0);
+      if (w < 2 || h < 2) { sel.style.display = 'none'; return; }
+      sel.style.display = 'block';
+      sel.style.left = l + 'px';
+      sel.style.top = t + 'px';
+      sel.style.width = w + 'px';
+      sel.style.height = h + 'px';
+    }
+    function mapRect(l, t, w, h) {
+      const vw = window.innerWidth, vh = window.innerHeight;
+      const sx = Math.round((l / vw) * iw);
+      const sy = Math.round((t / vh) * ih);
+      const sw = Math.max(1, Math.round((w / vw) * iw));
+      const sh = Math.max(1, Math.round((h / vh) * ih));
+      const cx = Math.min(Math.max(0, sx), iw - 1);
+      const cy = Math.min(Math.max(0, sy), ih - 1);
+      const cw = Math.min(sw, iw - cx);
+      const ch = Math.min(sh, ih - cy);
+      return { x: cx, y: cy, width: cw, height: ch };
+    }
+    function endDrag(clientX, clientY, pointerId) {
+      if (!dragging) return;
+      dragging = false;
+      if (typeof pointerId === 'number' && pointerId >= 0) {
+        try { veil.releasePointerCapture(pointerId); } catch (_) {}
+      }
+      const l = Math.min(ax, clientX), t = Math.min(ay, clientY);
+      const w = Math.abs(clientX - ax), h = Math.abs(clientY - ay);
+      if (w < 4 || h < 4) { sel.style.display = 'none'; return; }
+      api.submit(mapRect(l, t, w, h));
+    }
+    veil.addEventListener('pointerdown', (e) => {
+      if (e.button !== 0) return;
+      try { veil.setPointerCapture(e.pointerId); } catch (_) {}
+      dragging = true;
+      ax = e.clientX; ay = e.clientY;
+      show(ax, ay, ax, ay);
+    });
+    veil.addEventListener('pointermove', (e) => {
+      if (!dragging) return;
+      show(ax, ay, e.clientX, e.clientY);
+    });
+    veil.addEventListener('pointerup', (e) => {
+      endDrag(e.clientX, e.clientY, e.pointerId);
+    });
+    window.addEventListener('pointerup', (e) => {
+      endDrag(e.clientX, e.clientY, e.pointerId);
+    });
+    document.addEventListener(
+      'mouseup',
+      (e) => {
+        endDrag(e.clientX, e.clientY, -1);
+      },
+      true
+    );
+    veil.addEventListener('pointercancel', (e) => {
+      if (!dragging) return;
+      dragging = false;
+      try { veil.releasePointerCapture(e.pointerId); } catch (_) {}
+      sel.style.display = 'none';
+    });
+    window.addEventListener('keydown', (e) => {
+      if (e.key === 'Escape') { api.cancel(); return; }
+      if (e.key === 'Enter' && sel.style.display === 'block') {
+        const l = parseFloat(sel.style.left), t = parseFloat(sel.style.top);
+        const w = parseFloat(sel.style.width), h = parseFloat(sel.style.height);
+        if (w >= 4 && h >= 4) api.submit(mapRect(l, t, w, h));
+      }
+    });
+  })();`;
+}
+
+export function pickScreenRegion(): Promise<string | null> {
+  if (pickInProgress) return Promise.resolve(null);
+  pickInProgress = true;
+
+  return new Promise((resolve) => {
+    const display = screen.getDisplayNearestPoint(screen.getCursorScreenPoint());
+    let settled = false;
+    let overlay: BrowserWindow | null = null;
+    /** webContents for listener removal after `BrowserWindow` may already be destroyed. */
+    let overlayWc: Electron.WebContents | null = null;
+
+    const cleanupListeners = () => {
+      const wc = overlayWc;
+      overlayWc = null;
+      if (!wc || wc.isDestroyed()) return;
+      wc.removeListener('before-input-event', onBeforeInput);
+      wc.ipc.removeListener(IPC_CHANNELS.SCREEN_REGION_SUBMIT, onSubmit);
+      wc.ipc.removeListener(IPC_CHANNELS.SCREEN_REGION_CANCEL, onCancel);
+    };
+
+    const finish = (result: string | null) => {
+      if (settled) return;
+      settled = true;
+      pickInProgress = false;
+      cleanupListeners();
+      if (overlay && !overlay.isDestroyed()) {
+        overlay.removeAllListeners('closed');
+        overlay.close();
+      }
+      overlay = null;
+      resolve(result);
+    };
+
+    let snapshot: { dataUrl: string; width: number; height: number } | null = null;
+
+    const onSubmit = (
+      _event: Electron.IpcMainEvent,
+      rect: { x: number; y: number; width: number; height: number }
+    ) => {
+      if (settled || !overlay || overlay.isDestroyed()) return;
+      if (!rect || rect.width < 1 || rect.height < 1) {
+        finish(null);
+        return;
+      }
+      if (!snapshot) {
+        finish(null);
+        return;
+      }
+      try {
+        const full = nativeImage.createFromDataURL(snapshot.dataUrl);
+        const cropped = full.crop({
+          x: Math.floor(rect.x),
+          y: Math.floor(rect.y),
+          width: Math.floor(rect.width),
+          height: Math.floor(rect.height),
+        });
+        finish(cropped.toDataURL());
+      } catch {
+        finish(null);
+      }
+    };
+
+    const onCancel = (_event: Electron.IpcMainEvent) => {
+      if (settled || !overlay || overlay.isDestroyed()) return;
+      finish(null);
+    };
+
+    const onBeforeInput = (_event: Electron.Event, input: Electron.Input) => {
+      if (input.type === 'keyDown' && input.key === 'Escape') {
+        finish(null);
+      }
+    };
+
+    void captureDisplayDataUrl(display)
+      .then((cap) => {
+        if (!cap) {
+          finish(null);
+          return;
+        }
+        snapshot = cap;
+
+        overlay = new BrowserWindow({
+          x: display.bounds.x,
+          y: display.bounds.y,
+          width: display.bounds.width,
+          height: display.bounds.height,
+          frame: false,
+          transparent: true,
+          fullscreenable: false,
+          skipTaskbar: true,
+          alwaysOnTop: true,
+          focusable: true,
+          show: false,
+          autoHideMenuBar: true,
+          backgroundColor: '#00000000',
+          webPreferences: {
+            preload: path.join(__dirname, 'screen-region-preload.js'),
+            contextIsolation: true,
+            nodeIntegration: false,
+            sandbox: true,
+          },
+        });
+
+        overlayWc = overlay.webContents;
+        overlayWc.on('before-input-event', onBeforeInput);
+        overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_SUBMIT, onSubmit);
+        overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_CANCEL, onCancel);
+
+        overlay.setIgnoreMouseEvents(false);
+        overlay.loadURL(
+          'data:text/html;charset=utf-8,' +
+            encodeURIComponent('<!doctype html><html><head><meta charset="utf-8"/></head><body></body></html>')
+        );
+
+        overlay.on('closed', () => {
+          if (!settled) finish(null);
+        });
+
+        overlay.webContents.once('did-finish-load', () => {
+          if (!overlay || overlay.isDestroyed()) return;
+          overlay.webContents
+            .executeJavaScript(buildInjectScript(cap.dataUrl, cap.width, cap.height), true)
+            .then(() => {
+              overlay?.show();
+              overlay?.focus();
+            })
+            .catch(() => {
+              finish(null);
+            });
+        });
+      })
+      .catch(() => {
+        finish(null);
+      });
+  });
+}

surfsense_desktop/src/screen-region-preload.ts

@@ -0,0 +1,11 @@
+import { contextBridge, ipcRenderer } from 'electron';
+import { IPC_CHANNELS } from './ipc/channels';
+
+contextBridge.exposeInMainWorld('surfsenseScreenRegion', {
+  submit: (rect: { x: number; y: number; width: number; height: number }) => {
+    ipcRenderer.send(IPC_CHANNELS.SCREEN_REGION_SUBMIT, rect);
+  },
+  cancel: () => {
+    ipcRenderer.send(IPC_CHANNELS.SCREEN_REGION_CANCEL);
+  },
+});