Merge remote-tracking branch 'upstream/dev' into fix/ui-mention-documents

surfsense_web/lib/chat/display-media-capture.ts

@@ -0,0 +1,120 @@
+/** `getDisplayMedia` → single PNG frame (data URL). */
+function getImageCaptureCtor():
+	| (new (
+			track: MediaStreamTrack
+	  ) => { grabFrame: () => Promise<ImageBitmap> })
+	| undefined {
+	if (typeof window === "undefined") return undefined;
+	const IC = (
+		window as unknown as {
+			ImageCapture?: new (track: MediaStreamTrack) => { grabFrame: () => Promise<ImageBitmap> };
+		}
+	).ImageCapture;
+	return typeof IC === "function" ? IC : undefined;
+}
+
+function stopAllTracks(stream: MediaStream): void {
+	for (const t of stream.getTracks()) {
+		t.stop();
+	}
+}
+
+async function captureTrackToPngDataUrl(
+	track: MediaStreamTrack,
+	stream: MediaStream
+): Promise<string | null> {
+	const ImageCtor = getImageCaptureCtor();
+	if (ImageCtor !== undefined) {
+		try {
+			const ic = new ImageCtor(track);
+			const bitmap = await ic.grabFrame();
+			try {
+				const canvas = document.createElement("canvas");
+				canvas.width = bitmap.width;
+				canvas.height = bitmap.height;
+				const ctx = canvas.getContext("2d");
+				if (!ctx) {
+					stopAllTracks(stream);
+					return null;
+				}
+				ctx.drawImage(bitmap, 0, 0);
+				stopAllTracks(stream);
+				return canvas.toDataURL("image/png");
+			} finally {
+				if ("close" in bitmap && typeof bitmap.close === "function") {
+					bitmap.close();
+				}
+			}
+		} catch {
+			/* fall through to <video> */
+		}
+	}
+
+	const videoEl = document.createElement("video");
+	videoEl.srcObject = stream;
+	videoEl.muted = true;
+	const haveCurrentData = 2;
+	const dataReady = new Promise<void>((resolve) => {
+		if (videoEl.readyState >= haveCurrentData) {
+			resolve();
+			return;
+		}
+		videoEl.addEventListener("loadeddata", () => resolve(), { once: true });
+	});
+	await videoEl.play();
+	await Promise.race([
+		dataReady,
+		new Promise<void>((resolve) => {
+			setTimeout(resolve, 500);
+		}),
+	]);
+	const w = videoEl.videoWidth;
+	const h = videoEl.videoHeight;
+	if (!w || !h) {
+		stopAllTracks(stream);
+		return null;
+	}
+	const canvas = document.createElement("canvas");
+	canvas.width = w;
+	canvas.height = h;
+	const ctx = canvas.getContext("2d");
+	if (!ctx) {
+		stopAllTracks(stream);
+		return null;
+	}
+	ctx.drawImage(videoEl, 0, 0);
+	stopAllTracks(stream);
+	return canvas.toDataURL("image/png");
+}
+
+export async function captureDisplayToPngDataUrl(): Promise<string | null> {
+	if (typeof navigator === "undefined" || !navigator.mediaDevices?.getDisplayMedia) {
+		return null;
+	}
+	let stream: MediaStream | null = null;
+	try {
+		stream = await navigator.mediaDevices.getDisplayMedia({
+			video: { frameRate: { ideal: 1, max: 5 } },
+			audio: false,
+			selfBrowserSurface: "exclude",
+		} as Parameters<MediaDevices["getDisplayMedia"]>[0]);
+
+		const track = stream.getVideoTracks()[0];
+		if (!track) {
+			stopAllTracks(stream);
+			return null;
+		}
+
+		const dataUrl = await captureTrackToPngDataUrl(track, stream);
+		stream = null;
+		return dataUrl;
+	} catch (e) {
+		if (typeof process !== "undefined" && process.env?.NODE_ENV !== "production") {
+			console.warn("[captureDisplayToPngDataUrl]", e);
+		}
+		if (stream) {
+			stopAllTracks(stream);
+		}
+		return null;
+	}
+}

surfsense_web/lib/chat/user-turn-api-parts.ts

@@ -0,0 +1,56 @@
+import type { AppendMessage } from "@assistant-ui/react";
+
+const MAX_IMAGES = 4;
+
+export type NewChatUserImagePayload = {
+	media_type: "image/png" | "image/jpeg" | "image/webp";
+	data: string;
+};
+
+function dataUrlToPayload(dataUrl: string): NewChatUserImagePayload | null {
+	const m = /^data:(image\/(?:png|jpeg|webp|jpg));base64,([\s\S]+)$/i.exec(dataUrl.trim());
+	if (!m) return null;
+	let media = m[1].toLowerCase() as string;
+	if (media === "image/jpg") media = "image/jpeg";
+	if (media !== "image/png" && media !== "image/jpeg" && media !== "image/webp") return null;
+	const data = m[2].replace(/\s/g, "");
+	if (!data) return null;
+	return { media_type: media as NewChatUserImagePayload["media_type"], data };
+}
+
+function collectImageDataUrlsFromParts(parts: AppendMessage["content"]): string[] {
+	const out: string[] = [];
+	for (const part of parts) {
+		if (typeof part !== "object" || part === null || !("type" in part)) continue;
+		if (part.type !== "image") continue;
+		const img = "image" in part && typeof part.image === "string" ? part.image : null;
+		if (img && dataUrlToPayload(img)) out.push(img);
+	}
+	return out;
+}
+
+export function extractUserTurnForNewChatApi(
+	message: AppendMessage,
+	extraDataUrls: readonly string[]
+): { userQuery: string; userImages: NewChatUserImagePayload[] } {
+	let userQuery = "";
+	for (const part of message.content) {
+		if (part.type === "text") {
+			userQuery += part.text;
+		}
+	}
+
+	const merged = [...extraDataUrls, ...collectImageDataUrlsFromParts(message.content)];
+	const payloads: NewChatUserImagePayload[] = [];
+	const seen = new Set<string>();
+	for (const url of merged) {
+		const p = dataUrlToPayload(url);
+		if (!p) continue;
+		if (seen.has(p.data)) continue;
+		seen.add(p.data);
+		payloads.push(p);
+		if (payloads.length >= MAX_IMAGES) break;
+	}
+
+	return { userQuery, userImages: payloads };
+}