feat: add processing mode support for document uploads and ETL pipeline, improded error handling ux

- Introduced a `ProcessingMode` enum to differentiate between basic and premium processing modes.
- Updated `EtlRequest` to include a `processing_mode` field, defaulting to basic.
- Enhanced ETL pipeline services to utilize the selected processing mode for Azure Document Intelligence and LlamaCloud parsing.
- Modified various routes and services to handle processing mode, affecting document upload and indexing tasks.
- Improved error handling and logging to include processing mode details.
- Added tests to validate processing mode functionality and its impact on ETL operations.

surfsense_web/app/(home)/blog/[slug]/page.tsx

@@ -42,9 +42,7 @@ export async function generateMetadata(props: {
 	params: Promise<{ slug: string }>;
 }): Promise<Metadata> {
 	const { slug } = await props.params;
-	const page = (source.getPages() as BlogPageItem[]).find(
-		(p) => p.slugs.join("/") === slug,
-	);
+	const page = (source.getPages() as BlogPageItem[]).find((p) => p.slugs.join("/") === slug);
 
 	if (!page) return {};
 
@@ -72,13 +70,9 @@ export async function generateMetadata(props: {
 	};
 }
 
-export default async function BlogPostPage(props: {
-	params: Promise<{ slug: string }>;
-}) {
+export default async function BlogPostPage(props: { params: Promise<{ slug: string }> }) {
 	const { slug } = await props.params;
-	const page = (source.getPages() as BlogPageItem[]).find(
-		(p) => p.slugs.join("/") === slug,
-	);
+	const page = (source.getPages() as BlogPageItem[]).find((p) => p.slugs.join("/") === slug);
 
 	if (!page) notFound();
 

surfsense_web/app/(home)/blog/blog-magazine.tsx

@@ -1,10 +1,10 @@
 "use client";
 
-import { Container } from "@/components/container";
 import { format } from "date-fns";
+import FuzzySearch from "fuzzy-search";
 import Link from "next/link";
 import { useEffect, useMemo, useState } from "react";
-import FuzzySearch from "fuzzy-search";
+import { Container } from "@/components/container";
 import type { BlogEntry } from "./page";
 
 function truncate(text: string, length: number) {
@@ -24,8 +24,10 @@ function SearchIcon({ className }: { className?: string }) {
 			strokeWidth="2"
 			strokeLinecap="round"
 			strokeLinejoin="round"
-			aria-hidden
+			aria-hidden="true"
+			role="img"
 		>
+			<title>Search</title>
 			<circle cx="11" cy="11" r="8" />
 			<path d="m21 21-4.3-4.3" />
 		</svg>
@@ -100,9 +102,7 @@ function MagazineFeatured({ blog }: { blog: BlogEntry }) {
 							{blog.author}
 						</span>
 						<span className="text-white/50">·</span>
-						<time dateTime={blog.date}>
-							{format(new Date(blog.date), "MMMM d, yyyy")}
-						</time>
+						<time dateTime={blog.date}>{format(new Date(blog.date), "MMMM d, yyyy")}</time>
 					</div>
 				</div>
 			</div>
@@ -124,7 +124,7 @@ function MagazineSearchGrid({
 			new FuzzySearch(allBlogs, ["title", "description"], {
 				caseSensitive: false,
 			}),
-		[allBlogs],
+		[allBlogs]
 	);
 
 	const [results, setResults] = useState(allBlogs);
@@ -192,9 +192,7 @@ function MagazineCard({ blog }: { blog: BlogEntry }) {
 						className="h-full w-full object-cover transition duration-300 group-hover/card:scale-105"
 					/>
 				) : (
-					<div className="flex h-full items-center justify-center text-neutral-400">
-						No image
-					</div>
+					<div className="flex h-full items-center justify-center text-neutral-400">No image</div>
 				)}
 			</div>
 			<div className="flex flex-1 flex-col p-5">
@@ -218,9 +216,7 @@ function MagazineCard({ blog }: { blog: BlogEntry }) {
 						height={24}
 						className="h-6 w-6 rounded-full object-cover"
 					/>
-					<span className="text-xs text-neutral-600 dark:text-neutral-300">
-						{blog.author}
-					</span>
+					<span className="text-xs text-neutral-600 dark:text-neutral-300">{blog.author}</span>
 				</div>
 			</div>
 		</Link>

surfsense_web/app/(home)/blog/page.tsx

@@ -5,8 +5,7 @@ import { BlogWithSearchMagazine } from "./blog-magazine";
 
 export const metadata: Metadata = {
 	title: "Blog | SurfSense - AI Search & Knowledge Management",
-	description:
-		"Product updates, tutorials, and tips from the SurfSense team.",
+	description: "Product updates, tutorials, and tips from the SurfSense team.",
 	alternates: {
 		canonical: "https://surfsense.com/blog",
 	},
@@ -53,9 +52,7 @@ export default async function BlogPage() {
 			author: page.data.author ?? "SurfSense Team",
 			authorAvatar: page.data.authorAvatar ?? "/logo.png",
 		}))
-		.sort(
-			(a, b) => new Date(b.date).getTime() - new Date(a.date).getTime(),
-		);
+		.sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime());
 
 	return <BlogWithSearchMagazine blogs={blogs} />;
 }

surfsense_web/app/(home)/contact/page.tsx

@@ -3,7 +3,8 @@ import { ContactFormGridWithDetails } from "@/components/contact/contact-form";
 
 export const metadata: Metadata = {
 	title: "Contact | SurfSense",
-	description: "Get in touch with the SurfSense team for enterprise AI search, knowledge management, or partnership inquiries.",
+	description:
+		"Get in touch with the SurfSense team for enterprise AI search, knowledge management, or partnership inquiries.",
 	alternates: {
 		canonical: "https://surfsense.com/contact",
 	},

surfsense_web/app/(home)/login/LocalLoginForm.tsx

@@ -174,31 +174,31 @@ export function LocalLoginForm() {
 					<label htmlFor="password" className="block text-sm font-medium text-foreground">
 						{t("password")}
 					</label>
-				<div className="relative mt-1">
-					<input
-						id="password"
-						type={showPassword ? "text" : "password"}
-						autoComplete="current-password"
-						required
-						placeholder="Enter your password"
-						value={password}
-						onChange={(e) => setPassword(e.target.value)}
-						className={`block w-full rounded-md border pr-10 px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${
-							error.title
-								? "border-destructive focus:border-destructive focus:ring-destructive/40"
-								: "border-border focus:border-primary focus:ring-primary/40"
-						}`}
-						disabled={isLoggingIn}
-					/>
-					<button
-						type="button"
-						onClick={() => setShowPassword((prev) => !prev)}
-						className="absolute inset-y-0 right-0 flex items-center pr-3 text-muted-foreground hover:text-foreground"
-						aria-label={showPassword ? t("hide_password") : t("show_password")}
-					>
-						{showPassword ? <EyeOff className="h-4 w-4" /> : <Eye className="h-4 w-4" />}
-					</button>
-				</div>
+					<div className="relative mt-1">
+						<input
+							id="password"
+							type={showPassword ? "text" : "password"}
+							autoComplete="current-password"
+							required
+							placeholder="Enter your password"
+							value={password}
+							onChange={(e) => setPassword(e.target.value)}
+							className={`block w-full rounded-md border pr-10 px-3 py-1.5 md:py-2 shadow-sm focus:outline-none focus:ring-1 bg-background text-foreground transition-all ${
+								error.title
+									? "border-destructive focus:border-destructive focus:ring-destructive/40"
+									: "border-border focus:border-primary focus:ring-primary/40"
+							}`}
+							disabled={isLoggingIn}
+						/>
+						<button
+							type="button"
+							onClick={() => setShowPassword((prev) => !prev)}
+							className="absolute inset-y-0 right-0 flex items-center pr-3 text-muted-foreground hover:text-foreground"
+							aria-label={showPassword ? t("hide_password") : t("show_password")}
+						>
+							{showPassword ? <EyeOff className="h-4 w-4" /> : <Eye className="h-4 w-4" />}
+						</button>
+					</div>
 				</div>
 
 				<button

surfsense_web/app/(home)/page.tsx

@@ -1,17 +1,17 @@
 import dynamic from "next/dynamic";
-import { HeroSection } from "@/components/homepage/hero-section";
 import { AuthRedirect } from "@/components/homepage/auth-redirect";
-import { FeaturesCards } from "@/components/homepage/features-card";
 import { FeaturesBentoGrid } from "@/components/homepage/features-bento-grid";
+import { FeaturesCards } from "@/components/homepage/features-card";
+import { HeroSection } from "@/components/homepage/hero-section";
 
-const WhySurfSense = dynamic(
-	() => import("@/components/homepage/why-surfsense").then((m) => ({ default: m.WhySurfSense })),
+const WhySurfSense = dynamic(() =>
+	import("@/components/homepage/why-surfsense").then((m) => ({ default: m.WhySurfSense }))
 );
 
 const ExternalIntegrations = dynamic(() => import("@/components/homepage/integrations"));
 
-const CTAHomepage = dynamic(
-	() => import("@/components/homepage/cta").then((m) => ({ default: m.CTAHomepage })),
+const CTAHomepage = dynamic(() =>
+	import("@/components/homepage/cta").then((m) => ({ default: m.CTAHomepage }))
 );
 
 export default function HomePage() {

surfsense_web/app/(home)/pricing/page.tsx

@@ -1,6 +1,6 @@
 import type { Metadata } from "next";
-import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav";
 import PricingBasic from "@/components/pricing/pricing-section";
+import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav";
 
 export const metadata: Metadata = {
 	title: "Pricing | SurfSense - Free AI Search Plans",

surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx

@@ -38,11 +38,15 @@ import { removeChatTabAtom, updateChatTabTitleAtom } from "@/atoms/tabs/tabs.ato
 import { currentUserAtom } from "@/atoms/user/user-query.atoms";
 import { ThinkingStepsDataUI } from "@/components/assistant-ui/thinking-steps";
 import { Thread } from "@/components/assistant-ui/thread";
+import {
+	createTokenUsageStore,
+	type TokenUsageData,
+	TokenUsageProvider,
+} from "@/components/assistant-ui/token-usage-context";
 import { useChatSessionStateSync } from "@/hooks/use-chat-session-state";
 import { useMessagesSync } from "@/hooks/use-messages-sync";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { getBearerToken } from "@/lib/auth-utils";
-import { createTokenUsageStore, TokenUsageProvider, type TokenUsageData } from "@/components/assistant-ui/token-usage-context";
 import { convertToThreadMessage } from "@/lib/chat/message-utils";
 import {
 	isPodcastGenerating,
@@ -823,7 +827,14 @@ export default function NewChatPage() {
 									});
 								} else {
 									const tcId = `interrupt-${action.name}`;
-									addToolCall(contentPartsState, TOOLS_WITH_UI, tcId, action.name, action.args, true);
+									addToolCall(
+										contentPartsState,
+										TOOLS_WITH_UI,
+										tcId,
+										action.name,
+										action.args,
+										true
+									);
 									updateToolCall(contentPartsState, tcId, {
 										result: { __interrupt__: true, ...interruptData },
 									});
@@ -872,7 +883,7 @@ export default function NewChatPage() {
 						const newMsgId = `msg-${savedMessage.id}`;
 						tokenUsageStore.rename(assistantMsgId, newMsgId);
 						setMessages((prev) =>
-							prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m)),
+							prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m))
 						);
 
 						// Update pending interrupt with the new persisted message ID
@@ -1159,7 +1170,14 @@ export default function NewChatPage() {
 									});
 								} else {
 									const tcId = `interrupt-${action.name}`;
-									addToolCall(contentPartsState, TOOLS_WITH_UI, tcId, action.name, action.args, true);
+									addToolCall(
+										contentPartsState,
+										TOOLS_WITH_UI,
+										tcId,
+										action.name,
+										action.args,
+										true
+									);
 									updateToolCall(contentPartsState, tcId, {
 										result: {
 											__interrupt__: true,
@@ -1206,7 +1224,7 @@ export default function NewChatPage() {
 						const newMsgId = `msg-${savedMessage.id}`;
 						tokenUsageStore.rename(assistantMsgId, newMsgId);
 						setMessages((prev) =>
-							prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m)),
+							prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m))
 						);
 					} catch (err) {
 						console.error("Failed to persist resumed assistant message:", err);
@@ -1512,7 +1530,7 @@ export default function NewChatPage() {
 						const newMsgId = `msg-${savedMessage.id}`;
 						tokenUsageStore.rename(assistantMsgId, newMsgId);
 						setMessages((prev) =>
-							prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m)),
+							prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m))
 						);
 
 						trackChatResponseReceived(searchSpaceId, threadId);
@@ -1617,17 +1635,17 @@ export default function NewChatPage() {
 
 	return (
 		<TokenUsageProvider store={tokenUsageStore}>
-		<AssistantRuntimeProvider runtime={runtime}>
-			<ThinkingStepsDataUI />
-			<div key={searchSpaceId} className="flex h-full overflow-hidden">
-				<div className="flex-1 flex flex-col min-w-0 overflow-hidden">
-					<Thread />
+			<AssistantRuntimeProvider runtime={runtime}>
+				<ThinkingStepsDataUI />
+				<div key={searchSpaceId} className="flex h-full overflow-hidden">
+					<div className="flex-1 flex flex-col min-w-0 overflow-hidden">
+						<Thread />
+					</div>
+					<MobileReportPanel />
+					<MobileEditorPanel />
+					<MobileHitlEditPanel />
 				</div>
-				<MobileReportPanel />
-				<MobileEditorPanel />
-				<MobileHitlEditPanel />
-			</div>
-		</AssistantRuntimeProvider>
+			</AssistantRuntimeProvider>
 		</TokenUsageProvider>
 	);
 }

surfsense_web/app/dashboard/error.tsx

@@ -1,13 +1,15 @@
 "use client";
 
+import { ExternalLink } from "lucide-react";
 import Link from "next/link";
-import { useEffect } from "react";
+import { useEffect, useMemo } from "react";
+import { buildIssueUrl } from "@/lib/error-toast";
 
 export default function DashboardError({
 	error,
 	reset,
 }: {
-	error: globalThis.Error & { digest?: string };
+	error: globalThis.Error & { digest?: string; code?: string; requestId?: string };
 	reset: () => void;
 }) {
 	useEffect(() => {
@@ -18,12 +20,24 @@ export default function DashboardError({
 			.catch(() => {});
 	}, [error]);
 
+	const issueUrl = useMemo(() => buildIssueUrl(error), [error]);
+
 	return (
 		<div className="flex flex-1 flex-col items-center justify-center gap-4 p-8 text-center">
 			<h2 className="text-xl font-semibold">Something went wrong</h2>
 			<p className="text-muted-foreground max-w-md">
-				An error occurred in this section. Your dashboard is still available.
+				An error occurred in this section. Your dashboard is still available. If this keeps
+				happening, please report it so we can fix it.
 			</p>
+
+			{(error.digest || error.code || error.requestId) && (
+				<div className="rounded-md border bg-muted/50 px-4 py-2 text-xs text-muted-foreground font-mono max-w-md">
+					{error.code && <span>Code: {error.code}</span>}
+					{error.requestId && <span className="ml-3">ID: {error.requestId}</span>}
+					{error.digest && <span className="ml-3">Digest: {error.digest}</span>}
+				</div>
+			)}
+
 			<div className="flex gap-2">
 				<button
 					type="button"
@@ -38,6 +52,15 @@ export default function DashboardError({
 				>
 					Go to dashboard home
 				</Link>
+				<a
+					href={issueUrl}
+					target="_blank"
+					rel="noopener noreferrer"
+					className="inline-flex items-center gap-1.5 rounded-md border border-input bg-background px-4 py-2 text-sm font-medium hover:bg-accent hover:text-accent-foreground transition-colors"
+				>
+					<ExternalLink className="h-3.5 w-3.5" />
+					Report Issue
+				</a>
 			</div>
 		</div>
 	);

surfsense_web/app/error.tsx

@@ -1,12 +1,14 @@
 "use client";
 
-import { useEffect } from "react";
+import { ExternalLink } from "lucide-react";
+import { useEffect, useMemo } from "react";
+import { buildIssueUrl } from "@/lib/error-toast";
 
 export default function ErrorPage({
 	error,
 	reset,
 }: {
-	error: globalThis.Error & { digest?: string };
+	error: globalThis.Error & { digest?: string; code?: string; requestId?: string };
 	reset: () => void;
 }) {
 	useEffect(() => {
@@ -17,19 +19,41 @@ export default function ErrorPage({
 			.catch(() => {});
 	}, [error]);
 
+	const issueUrl = useMemo(() => buildIssueUrl(error), [error]);
+
 	return (
-		<div className="flex min-h-[50vh] flex-col items-center justify-center gap-4 text-center">
+		<div className="flex min-h-[50vh] flex-col items-center justify-center gap-4 text-center px-4">
 			<h2 className="text-2xl font-semibold">Something went wrong</h2>
 			<p className="text-muted-foreground max-w-md">
-				An unexpected error occurred. Please try again.
+				An unexpected error occurred. Please try again, or report this issue if it persists.
 			</p>
-			<button
-				type="button"
-				onClick={reset}
-				className="rounded-md bg-primary px-4 py-2 text-sm font-medium text-primary-foreground hover:bg-primary/90 transition-colors"
-			>
-				Try again
-			</button>
+
+			{(error.digest || error.code || error.requestId) && (
+				<div className="rounded-md border bg-muted/50 px-4 py-2 text-xs text-muted-foreground font-mono max-w-md">
+					{error.code && <span>Code: {error.code}</span>}
+					{error.requestId && <span className="ml-3">ID: {error.requestId}</span>}
+					{error.digest && <span className="ml-3">Digest: {error.digest}</span>}
+				</div>
+			)}
+
+			<div className="flex gap-2">
+				<button
+					type="button"
+					onClick={reset}
+					className="rounded-md bg-primary px-4 py-2 text-sm font-medium text-primary-foreground hover:bg-primary/90 transition-colors"
+				>
+					Try again
+				</button>
+				<a
+					href={issueUrl}
+					target="_blank"
+					rel="noopener noreferrer"
+					className="inline-flex items-center gap-1.5 rounded-md border border-input bg-background px-4 py-2 text-sm font-medium hover:bg-accent hover:text-accent-foreground transition-colors"
+				>
+					<ExternalLink className="h-3.5 w-3.5" />
+					Report Issue
+				</a>
+			</div>
 		</div>
 	);
 }

surfsense_web/app/global-error.tsx

@@ -2,9 +2,32 @@
 
 import "./globals.css";
 import posthog from "posthog-js";
-import { useEffect } from "react";
+import { useEffect, useMemo } from "react";
 import { Button } from "@/components/ui/button";
 
+const ISSUES_URL = "https://github.com/MODSetter/SurfSense/issues/new";
+
+function buildBasicIssueUrl(error: Error & { digest?: string }) {
+	const params = new URLSearchParams();
+	const lines = [
+		"## Bug Report",
+		"",
+		"**Describe what happened:**",
+		"",
+		"",
+		"## Diagnostics (auto-filled)",
+		"",
+		`- **Error:** ${error.message}`,
+		...(error.digest ? [`- **Digest:** \`${error.digest}\``] : []),
+		`- **Timestamp:** ${new Date().toISOString()}`,
+		`- **Page:** \`${typeof window !== "undefined" ? window.location.pathname : "unknown"}\``,
+		`- **User Agent:** \`${typeof navigator !== "undefined" ? navigator.userAgent : "unknown"}\``,
+	];
+	params.set("body", lines.join("\n"));
+	params.set("labels", "bug");
+	return `${ISSUES_URL}?${params.toString()}`;
+}
+
 export default function GlobalError({
 	error,
 	reset,
@@ -16,13 +39,32 @@ export default function GlobalError({
 		posthog.captureException(error);
 	}, [error]);
 
+	const issueUrl = useMemo(() => buildBasicIssueUrl(error), [error]);
+
 	return (
 		<html lang="en">
 			<body>
-				<div className="flex min-h-screen flex-col items-center justify-center gap-4 p-4">
+				<div className="flex min-h-screen flex-col items-center justify-center gap-4 p-4 text-center">
 					<h2 className="text-xl font-semibold">Something went wrong</h2>
-					<p className="text-sm text-muted-foreground">An unexpected error occurred.</p>
-					<Button onClick={reset}>Try again</Button>
+					<p className="text-sm text-muted-foreground max-w-md">
+						An unexpected error occurred. Please try again, or report this issue if it persists.
+					</p>
+
+					{error.digest && (
+						<p className="text-xs text-muted-foreground font-mono">Digest: {error.digest}</p>
+					)}
+
+					<div className="flex gap-2">
+						<Button onClick={reset}>Try again</Button>
+						<a
+							href={issueUrl}
+							target="_blank"
+							rel="noopener noreferrer"
+							className="inline-flex items-center gap-1.5 rounded-md border border-input bg-background px-4 py-2 text-sm font-medium hover:bg-accent hover:text-accent-foreground transition-colors"
+						>
+							Report Issue
+						</a>
+					</div>
 				</div>
 			</body>
 		</html>

surfsense_web/app/layout.tsx

@@ -7,13 +7,17 @@ import { GlobalLoadingProvider } from "@/components/providers/GlobalLoadingProvi
 import { I18nProvider } from "@/components/providers/I18nProvider";
 import { PostHogProvider } from "@/components/providers/PostHogProvider";
 import { ZeroProvider } from "@/components/providers/ZeroProvider";
+import {
+	OrganizationJsonLd,
+	SoftwareApplicationJsonLd,
+	WebSiteJsonLd,
+} from "@/components/seo/json-ld";
 import { ThemeProvider } from "@/components/theme/theme-provider";
 import { Toaster } from "@/components/ui/sonner";
 import { LocaleProvider } from "@/contexts/LocaleContext";
 import { PlatformProvider } from "@/contexts/platform-context";
 import { ReactQueryClientProvider } from "@/lib/query-client/query-client.provider";
 import { cn } from "@/lib/utils";
-import { OrganizationJsonLd, SoftwareApplicationJsonLd, WebSiteJsonLd } from "@/components/seo/json-ld";
 
 const roboto = Roboto({
 	subsets: ["latin"],

surfsense_web/app/not-found.tsx

@@ -3,7 +3,8 @@ import Link from "next/link";
 
 export const metadata: Metadata = {
 	title: "Page Not Found | SurfSense",
-	description: "The page you're looking for doesn't exist. Explore SurfSense - open source enterprise AI search and knowledge management.",
+	description:
+		"The page you're looking for doesn't exist. Explore SurfSense - open source enterprise AI search and knowledge management.",
 };
 
 export default function NotFound() {