feat: update environment variables and enhance scraping capabilities

- Adjusted Google Maps and YouTube micro pricing in the .env.example file for better cost management.
- Introduced new environment variables for captcha solving and stealth browser hardening to improve scraping resilience.
- Removed outdated smoke test for scraper API endpoints to streamline testing.
- Enhanced anonymous chat agent's system prompt to clarify capabilities and suggest account creation for advanced features.
- Updated Reddit fetch logic to prioritize new session handling and improve resilience against IP-related issues.
- Added compacting functionality for scraper results to optimize data handling and presentation.
- Improved workspace and document management tools with clearer descriptions and enhanced functionality.
- Introduced new UI components for agent setup guidance in the web application.

surfsense_web/components/layout/ui/sidebar/PlaygroundSidebar.tsx

@@ -3,6 +3,7 @@
 import { History, KeyRound } from "lucide-react";
 import Link from "next/link";
 import { usePathname } from "next/navigation";
+import { ConnectAgentDialog } from "@/components/mcp/connect-agent-dialog";
 import { PLAYGROUND_PLATFORMS, type PlatformIcon } from "@/lib/playground/catalog";
 import { cn } from "@/lib/utils";
 
@@ -88,6 +89,10 @@ export function PlaygroundSidebar({ workspaceId }: PlaygroundSidebarProps) {
 					</div>
 				))}
 			</div>
+
+			<div className="shrink-0 py-1.5 before:mx-3 before:mb-1.5 before:block before:h-px before:bg-border">
+				<ConnectAgentDialog />
+			</div>
 		</div>
 	);
 }

surfsense_web/components/mcp/agent-setup-tabs.tsx

@@ -0,0 +1,85 @@
+"use client";
+
+import { Check, Copy } from "lucide-react";
+import { useState } from "react";
+import { Button } from "@/components/ui/button";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import {
+	API_KEY_PLACEHOLDER,
+	DEFAULT_SERVER_DIR,
+	MCP_CLIENTS,
+	type McpSnippetOptions,
+} from "@/lib/mcp/clients";
+
+function CopyButton({ text }: { text: string }) {
+	const [copied, setCopied] = useState(false);
+
+	const handleCopy = async () => {
+		try {
+			await navigator.clipboard.writeText(text);
+			setCopied(true);
+			setTimeout(() => setCopied(false), 2000);
+		} catch {
+			// Clipboard unavailable (permissions/insecure context); nothing to recover.
+		}
+	};
+
+	return (
+		<Button
+			variant="ghost"
+			size="sm"
+			className="absolute top-2 right-2 h-7 gap-1.5 px-2 text-xs"
+			onClick={handleCopy}
+			aria-label="Copy configuration"
+		>
+			{copied ? <Check className="size-3.5 text-brand" /> : <Copy className="size-3.5" />}
+			{copied ? "Copied" : "Copy"}
+		</Button>
+	);
+}
+
+/**
+ * Per-agent MCP setup instructions as tabs: pick a client, follow its steps,
+ * copy its exact config. Used on the /mcp-server marketing page and in the
+ * API playground; `options` fills in real values where the caller has them.
+ */
+export function AgentSetupTabs({ options }: { options?: Partial<McpSnippetOptions> }) {
+	const resolved: McpSnippetOptions = {
+		baseUrl: options?.baseUrl || "https://api.surfsense.com",
+		apiKey: options?.apiKey || API_KEY_PLACEHOLDER,
+		serverDir: options?.serverDir || DEFAULT_SERVER_DIR,
+	};
+
+	return (
+		<Tabs defaultValue={MCP_CLIENTS[0].id} className="w-full">
+			<TabsList className="flex h-auto flex-wrap justify-start gap-1">
+				{MCP_CLIENTS.map((client) => (
+					<TabsTrigger key={client.id} value={client.id}>
+						{client.label}
+					</TabsTrigger>
+				))}
+			</TabsList>
+			{MCP_CLIENTS.map((client) => {
+				const config = client.buildConfig(resolved);
+				return (
+					<TabsContent key={client.id} value={client.id} className="space-y-3">
+						<ol className="list-decimal space-y-1 pl-5 text-sm leading-relaxed text-muted-foreground">
+							{client.steps.map((step) => (
+								<li key={step}>{step}</li>
+							))}
+						</ol>
+						<div>
+							<p className="mb-1.5 font-mono text-xs text-muted-foreground">{client.configFile}</p>
+							<div className="relative">
+								<CopyButton text={config} />
+								<pre className="overflow-x-auto rounded-lg border bg-muted/50 p-4 font-mono text-xs leading-relaxed">
+									<code>{config}</code>
+								</pre>
+							</div>
+						</div>
+					</TabsContent>
+				);
+			})}
+		</Tabs>
+	);
+}

surfsense_web/components/mcp/connect-agent-dialog.tsx

@@ -0,0 +1,47 @@
+"use client";
+
+import { Cable } from "lucide-react";
+import { AgentSetupTabs } from "@/components/mcp/agent-setup-tabs";
+import {
+	Dialog,
+	DialogContent,
+	DialogDescription,
+	DialogHeader,
+	DialogTitle,
+	DialogTrigger,
+} from "@/components/ui/dialog";
+import { BACKEND_URL } from "@/lib/env-config";
+import { cn } from "@/lib/utils";
+
+/**
+ * Sidebar-footer button that opens the MCP setup guide: pick an agent
+ * (Claude Code, Codex, OpenCode, ...), copy its config, done.
+ */
+export function ConnectAgentDialog({ className }: { className?: string }) {
+	return (
+		<Dialog>
+			<DialogTrigger
+				className={cn(
+					"group/link relative flex h-9 items-center gap-2 rounded-md mx-2 px-2 text-sm text-left",
+					"transition-colors hover:bg-accent hover:text-accent-foreground",
+					"focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring",
+					className
+				)}
+			>
+				<Cable className="h-3.5 w-3.5 shrink-0" />
+				<span className="min-w-0 flex-1 truncate">Connect your AI Agent</span>
+			</DialogTrigger>
+			<DialogContent className="max-h-[85vh] overflow-y-auto sm:max-w-2xl">
+				<DialogHeader>
+					<DialogTitle>Connect to Claude Code, Codex, OpenCode…</DialogTitle>
+					<DialogDescription>
+						The SurfSense MCP server gives any coding agent these scrapers and your knowledge base
+						as native tools. You need an API key (create one under API Keys) — then pick your agent
+						and paste its config.
+					</DialogDescription>
+				</DialogHeader>
+				<AgentSetupTabs options={{ baseUrl: BACKEND_URL || undefined }} />
+			</DialogContent>
+		</Dialog>
+	);
+}