apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-26 23:51:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'upstream/dev' into feature-indeed-jobs-scraper

Browse source 
# Conflicts:
#	README.es.md
#	README.hi.md
#	README.md
#	README.pt-BR.md
#	README.zh-CN.md
#	surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py
#	surfsense_backend/tests/unit/capabilities/reddit/test_registry.py
#	surfsense_backend/tests/unit/capabilities/youtube/test_registry.py
#	surfsense_mcp/mcp_server/features/scrapers/__init__.py
#	surfsense_web/content/docs/connectors/index.mdx
#	surfsense_web/content/docs/connectors/native/index.mdx
#	surfsense_web/content/docs/connectors/native/meta.json
#	surfsense_web/content/docs/how-to/mcp-server.mdx
#	surfsense_web/lib/connectors-marketing/index.ts
#	surfsense_web/lib/playground/catalog.ts
This commit is contained in:
CREDO23 2026-07-18 22:11:30 +02:00
commit 91aa265afb
259 changed files with 13867 additions and 1883 deletions
Download patch file Download diff file Expand all files Collapse all files

4
surfsense_web/.env.example
View file

@ -5,7 +5,7 @@
# Optional packaged-client override. Leave unset in Docker so browser requests
# use same-origin relative URLs behind Caddy. Set it for packaged clients
# (e.g. Electron) or local dev that talks to a separate backend origin.
# NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000
NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000
# Server-only. Internal backend URL used by Next.js server code (RSC / route
# handlers). Cannot be a relative URL.
@ -40,7 +40,7 @@ NEXT_PUBLIC_POSTHOG_HOST=https://us.i.posthog.com
# Zero cache (real-time sync). Leave unset in Docker to use the same-origin
# "/zero" endpoint behind Caddy. Set it for local dev or packaged clients.
# ─────────────────────────────────────────────────────────────────────────────
# NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848
NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848
# Server-only shared secret that authorizes zero-cache when it calls
# /api/zero/query. Leave unset during the compatibility rollout, then set it
# once every zero-cache instance sends X-Api-Key.

2
surfsense_web/app/(home)/announcements/page.tsx
View file

@ -10,7 +10,7 @@ import { useAnnouncements } from "@/hooks/use-announcements";
// ---------------------------------------------------------------------------
export default function AnnouncementsPage() {
const { announcements, markAllRead } = useAnnouncements();
const { announcements, markAllRead } = useAnnouncements({ includeExpired: true });
// Auto-mark all visible announcements as read when the page is opened
useEffect(() => {

8
surfsense_web/app/(home)/connectors/page.tsx
View file

@ -6,7 +6,7 @@ import { getAllConnectors } from "@/lib/connectors-marketing";
const canonicalUrl = "https://www.surfsense.com/connectors";
const metaDescription =
"Platform-native scraper APIs for AI agents. Pull live data from the platforms your market uses through one typed API or the SurfSense MCP server. Explore every connector.";
"Platform-native scraper APIs for AI agents. Pull live, structured data from the platforms where answers live through one typed API or the SurfSense MCP server. Explore every connector.";
export const metadata: Metadata = {
title: "Scraper APIs for AI Agents: All Connectors | SurfSense",
@ -17,7 +17,7 @@ export const metadata: Metadata = {
"scraper api for ai agents",
"data connectors",
"mcp server",
"competitive intelligence platform",
"open web research platform",
],
alternates: { canonical: canonicalUrl },
openGraph: {
@ -38,13 +38,13 @@ export default function ConnectorsIndexPage() {
<div className="mx-auto w-full max-w-7xl px-2 md:px-8 xl:px-0">
<header className="max-w-2xl">
<h1 className="text-3xl font-bold tracking-tight text-balance sm:text-4xl lg:text-5xl">
Connectors for every platform your market uses
Connectors for every platform where answers live
</h1>
<p className="mt-5 text-base leading-relaxed text-muted-foreground sm:text-lg">
Each connector is a platform-native scraper API your AI agents can call directly, or
through the SurfSense MCP server. They are the live data behind the SurfSense{" "}
<Link href="/" className="font-medium text-foreground underline underline-offset-4">
competitive intelligence platform
open web research platform
</Link>
.
</p>

4
surfsense_web/app/(home)/external-mcp-connectors/page.tsx
View file

@ -253,7 +253,7 @@ export default function ExternalMcpConnectorsPage() {
<p className="mt-3 max-w-2xl text-muted-foreground leading-relaxed">
These apps run on their official hosted MCP servers. Sign in once with OAuth and
SurfSense manages the tokens and curates each tool list, so your agents can search
Notion, read Slack threads, or file Jira issues alongside your market intelligence.
Notion, read Slack threads, or file Jira issues alongside your live web research.
</p>
</Reveal>
<Reveal>
@ -330,7 +330,7 @@ export default function ExternalMcpConnectorsPage() {
<p className="mx-auto mt-3 max-w-xl text-muted-foreground leading-relaxed">
External MCP connectors are part of the SurfSense{" "}
<Link href="/" className="font-medium text-foreground underline underline-offset-4">
competitive intelligence platform
open web research platform
</Link>
. Start free, no credit card required.
</p>

15
surfsense_web/app/(home)/mcp-server/page.tsx
View file

@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types";
const canonicalUrl = "https://www.surfsense.com/mcp-server";
const metaDescription =
"The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
"The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
export const metadata: Metadata = {
title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools",
@ -102,6 +102,7 @@ const TOOL_GROUPS = [
"surfsense_google_maps_scrape",
"surfsense_google_maps_reviews",
"surfsense_google_search",
"surfsense_amazon_scrape",
"surfsense_web_crawl",
"surfsense_list_scraper_runs",
"surfsense_get_scraper_run",
@ -133,7 +134,7 @@ const FAQ: FaqItem[] = [
{
question: "What is the SurfSense MCP server?",
answer:
"It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
"It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
},
{
question: "Which MCP clients does it work with?",
@ -221,9 +222,9 @@ export default function McpServerPage() {
</h1>
<p className="mt-5 max-w-xl text-base leading-relaxed text-muted-foreground sm:text-lg">
The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform:
scrape Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open
web, and search, read, and write your knowledge base. One API key, typed tools, pay
as you go.
scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and
the open web, and search, read, and write your knowledge base. One API key, typed
tools, pay as you go.
</p>
<div className="mt-8 flex flex-wrap items-center gap-3">
<Button asChild size="lg">
@ -369,12 +370,12 @@ export default function McpServerPage() {
<Reveal>
<div className="rounded-2xl border bg-card p-8 text-center sm:p-12">
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
Put live market data inside your agents
Put live web data inside your agents
</h2>
<p className="mx-auto mt-3 max-w-xl text-muted-foreground leading-relaxed">
The MCP server is part of the SurfSense{" "}
<Link href="/" className="font-medium text-foreground underline underline-offset-4">
competitive intelligence platform
open web research platform
</Link>
. Start with $5 of free credit, no credit card required.
</p>

10
surfsense_web/app/(home)/page.tsx
View file

@ -4,19 +4,17 @@ import { CompareTable } from "@/components/homepage/compare-table";
import { ConnectorGrid } from "@/components/homepage/connector-grid";
import { HeroSection } from "@/components/homepage/hero-section";
import { HomeFaq } from "@/components/homepage/home-faq";
import { HowItWorks } from "@/components/homepage/how-it-works";
import { PersonaPaths } from "@/components/homepage/persona-paths";
import { UseCasesRow } from "@/components/homepage/use-cases";
import { LogoCloud } from "@/components/homepage/logo-cloud";
import { SocialProof } from "@/components/homepage/social-proof";
export default function HomePage() {
return (
<div className="min-h-screen bg-linear-to-b from-gray-50 to-gray-100 text-gray-900 dark:from-black dark:to-gray-900 dark:text-white">
<AuthRedirect />
<HeroSection />
<PersonaPaths />
<LogoCloud />
<SocialProof />
<ConnectorGrid />
<HowItWorks />
<UseCasesRow />
<CompareTable />
<HomeFaq />
<CommunityStrip />

2
surfsense_web/app/(home)/pricing/page.tsx
View file

@ -17,7 +17,7 @@ export const metadata: Metadata = {
"open source ai agent platform",
"self-hosted ai workspace",
"ai automation pricing",
"competitive intelligence pricing",
"web scraping api pricing",
],
alternates: {
canonical: canonicalUrl,

2
surfsense_web/app/dashboard/[workspace_id]/automations/components/builder/basics-section.tsx
View file

@ -18,7 +18,7 @@ export function BasicsSection({ name, description, errors, onChange }: BasicsSec
id="automation-name"
value={name}
maxLength={200}
placeholder="Weekly competitor digest"
placeholder="Weekly research digest"
onChange={(e) => onChange({ name: e.target.value })}
/>
</Field>

136
surfsense_web/app/dashboard/[workspace_id]/client-layout.tsx
View file

@ -4,15 +4,9 @@ import { useAtomValue, useSetAtom } from "jotai";
import { useParams, usePathname, useRouter } from "next/navigation";
import { useTranslations } from "next-intl";
import type React from "react";
import { useEffect, useState } from "react";
import { useEffect } from "react";
import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom";
import { myAccessAtom } from "@/atoms/members/members-query.atoms";
import {
globalLlmConfigStatusAtom,
globalModelConnectionsAtom,
modelConnectionsAtom,
modelRolesAtom,
} from "@/atoms/model-connections/model-connections-query.atoms";
import { llmSetupStatusAtomFamily } from "@/atoms/model-connections/model-connections-query.atoms";
import { activeWorkspaceIdAtom } from "@/atoms/workspaces/workspace-query.atoms";
import { ConnectorIndicator } from "@/components/assistant-ui/connector-popup";
import { DocumentUploadDialogProvider } from "@/components/assistant-ui/document-upload-popup";
@ -22,14 +16,15 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/com
import { useFolderSync } from "@/hooks/use-folder-sync";
import { useGlobalLoadingEffect } from "@/hooks/use-global-loading";
import { useElectronAPI } from "@/hooks/use-platform";
import { isLlmOnboardingComplete } from "@/lib/onboarding";
export function DashboardClientLayout({
children,
workspaceId,
initialPlaygroundSidebarCollapsed,
}: {
children: React.ReactNode;
workspaceId: string;
initialPlaygroundSidebarCollapsed: boolean;
}) {
const t = useTranslations("dashboard");
const router = useRouter();
@ -39,85 +34,38 @@ export function DashboardClientLayout({
const setActiveWorkspaceIdState = useSetAtom(activeWorkspaceIdAtom);
const setPendingUserImageUrls = useSetAtom(pendingUserImageDataUrlsAtom);
const { data: modelRoles = {}, isLoading: loading, error } = useAtomValue(modelRolesAtom);
const { data: globalConnections = [], isLoading: globalConfigsLoading } = useAtomValue(
globalModelConnectionsAtom
);
const { data: modelConnections = [], isLoading: modelConnectionsLoading } =
useAtomValue(modelConnectionsAtom);
const { data: globalConfigStatus, isLoading: globalConfigStatusLoading } =
useAtomValue(globalLlmConfigStatusAtom);
const { data: access = null, isLoading: accessLoading } = useAtomValue(myAccessAtom);
const [hasCheckedOnboarding, setHasCheckedOnboarding] = useState(false);
// Single source of truth for the onboarding gate. Keyed by the route
// workspaceId so the verdict can never lag behind a workspace switch.
const {
data: setupStatus,
isLoading: setupLoading,
error: setupError,
} = useAtomValue(llmSetupStatusAtomFamily(Number(workspaceId)));
const isOnboardingPage = pathname?.includes("/onboard");
const isOwner = access?.is_owner ?? false;
const isWorkspaceReady = activeWorkspaceId === workspaceId;
const isReady = setupStatus?.status === "ready";
// First-run (initial_setup) is the only not-ready state that redirects, so
// recovery falls through to the inline composer notice and an established
// user who lost their models is never re-onboarded. The other direction
// leaves onboarding once the workspace can chat.
useEffect(() => {
if (isWorkspaceReady) return;
setHasCheckedOnboarding(false);
}, [isWorkspaceReady]);
useEffect(() => {
if (isOnboardingPage) {
setHasCheckedOnboarding(true);
return;
}
if (
isWorkspaceReady &&
!loading &&
!accessLoading &&
!globalConfigsLoading &&
!globalConfigStatusLoading &&
!modelConnectionsLoading &&
!hasCheckedOnboarding
) {
// Onboarding is only relevant when no operator-provided
// global_llm_config.yaml exists. When it does, workspaces inherit
// the global config and should never be forced into onboarding.
if (globalConfigStatus?.exists) {
setHasCheckedOnboarding(true);
return;
}
const onboardingComplete = isLlmOnboardingComplete(
modelRoles.chat_model_id,
globalConnections,
modelConnections
);
if (onboardingComplete) {
setHasCheckedOnboarding(true);
return;
}
if (!isOwner) {
setHasCheckedOnboarding(true);
return;
}
router.push(`/dashboard/${workspaceId}/onboard`);
setHasCheckedOnboarding(true);
if (setupLoading || setupError) return;
if (setupStatus?.stage === "initial_setup" && !isOnboardingPage) {
router.replace(`/dashboard/${workspaceId}/onboard`);
} else if (isReady && isOnboardingPage) {
router.replace(`/dashboard/${workspaceId}/new-chat`);
}
}, [
isWorkspaceReady,
loading,
accessLoading,
globalConfigsLoading,
globalConfigStatusLoading,
globalConfigStatus,
modelConnectionsLoading,
modelRoles.chat_model_id,
globalConnections,
modelConnections,
setupLoading,
setupError,
setupStatus?.stage,
isReady,
isOnboardingPage,
isOwner,
router,
workspaceId,
hasCheckedOnboarding,
]);
const electronAPI = useElectronAPI();
@ -167,18 +115,19 @@ export function DashboardClientLayout({
}
}, [workspace_id, setActiveWorkspaceIdState, electronAPI]);
// Determine if we should show loading
const shouldShowLoading =
!hasCheckedOnboarding &&
(!isWorkspaceReady ||
loading ||
accessLoading ||
globalConfigsLoading ||
globalConfigStatusLoading ||
modelConnectionsLoading) &&
!isOnboardingPage;
// Suppress children during either pending redirect so neither /new-chat nor
// /onboard flashes for a frame.
const isLeavingOnboarding = isReady && isOnboardingPage;
const isEnteringOnboarding = setupStatus?.stage === "initial_setup" && !isOnboardingPage;
const isRedirecting =
!setupLoading && !setupError && (isLeavingOnboarding || isEnteringOnboarding);
// Block children until the workspace is synced and the initial verdict is
// in; afterwards the composer renders its own not-ready state in place, so
// recovery (e.g. deleting the last model) never triggers a full-screen
// loader or a redirect.
const shouldShowLoading = !setupError && (!isWorkspaceReady || setupLoading || isRedirecting);
// Use global loading screen - spinner animation won't reset
useGlobalLoadingEffect(shouldShowLoading);
// Wire desktop app file watcher -> single-file re-index API
@ -188,7 +137,7 @@ export function DashboardClientLayout({
return null;
}
if (error && !hasCheckedOnboarding && !isOnboardingPage) {
if (setupError) {
return (
<div className="flex flex-col items-center justify-center min-h-screen space-y-4">
<Card className="w-[400px] bg-background/60 backdrop-blur-sm border-destructive/20">
@ -200,7 +149,7 @@ export function DashboardClientLayout({
</CardHeader>
<CardContent>
<p className="text-sm text-muted-foreground">
{error instanceof Error ? error.message : String(error)}
{setupError instanceof Error ? setupError.message : String(setupError)}
</p>
</CardContent>
</Card>
@ -215,7 +164,10 @@ export function DashboardClientLayout({
return (
<DocumentUploadDialogProvider>
<OnboardingTour />
<LayoutDataProvider workspaceId={workspaceId}>
<LayoutDataProvider
workspaceId={workspaceId}
initialPlaygroundSidebarCollapsed={initialPlaygroundSidebarCollapsed}
>
{children}
<ConnectorIndicator showTrigger={false} />
</LayoutDataProvider>

20
surfsense_web/app/dashboard/[workspace_id]/layout.tsx
View file

@ -1,16 +1,28 @@
// Server component
import { cookies } from "next/headers";
import type React from "react";
import { use } from "react";
import { DashboardClientLayout } from "./client-layout";
export default function DashboardLayout({
const PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE = "surfsense_playground_sidebar_collapsed";
export default async function DashboardLayout({
params,
children,
}: {
params: Promise<{ workspace_id: string }>;
children: React.ReactNode;
}) {
const { workspace_id } = use(params);
const [{ workspace_id }, cookieStore] = await Promise.all([params, cookies()]);
const initialPlaygroundSidebarCollapsed =
cookieStore.get(PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE)?.value === "true";
return <DashboardClientLayout workspaceId={workspace_id}>{children}</DashboardClientLayout>;
return (
<DashboardClientLayout
workspaceId={workspace_id}
initialPlaygroundSidebarCollapsed={initialPlaygroundSidebarCollapsed}
>
{children}
</DashboardClientLayout>
);
}

50
surfsense_web/app/dashboard/[workspace_id]/onboard/page.tsx
View file

@ -2,66 +2,32 @@
import { useAtomValue } from "jotai";
import { useParams, useRouter } from "next/navigation";
import { useEffect, useMemo } from "react";
import { useEffect } from "react";
import {
globalLlmConfigStatusAtom,
globalModelConnectionsAtom,
llmSetupStatusAtomFamily,
modelConnectionsAtom,
modelRolesAtom,
} from "@/atoms/model-connections/model-connections-query.atoms";
import { Logo } from "@/components/Logo";
import { ModelProviderConnectionsPanel } from "@/components/settings/model-connections/model-provider-connections-panel";
import { Button } from "@/components/ui/button";
import { useGlobalLoadingEffect } from "@/hooks/use-global-loading";
import { useSession } from "@/hooks/use-session";
import { redirectToLogin } from "@/lib/auth-utils";
import { hasEnabledChatModel, isLlmOnboardingComplete } from "@/lib/onboarding";
export default function OnboardPage() {
const router = useRouter();
const params = useParams();
const workspaceId = Number(params.workspace_id);
const session = useSession();
const { data: globalConnections = [], isLoading: globalLoading } = useAtomValue(
globalModelConnectionsAtom
);
const { data: connections = [] } = useAtomValue(modelConnectionsAtom);
const { data: roles = {}, isLoading: rolesLoading } = useAtomValue(modelRolesAtom);
const { data: globalConfigStatus, isLoading: globalConfigStatusLoading } =
useAtomValue(globalLlmConfigStatusAtom);
const { data: setupStatus } = useAtomValue(llmSetupStatusAtomFamily(workspaceId));
useEffect(() => {
if (session.status === "unauthenticated") redirectToLogin();
}, [session.status]);
const hasUsableChatModel = useMemo(
() => hasEnabledChatModel([...globalConnections, ...connections]),
[globalConnections, connections]
);
const onboardingComplete = isLlmOnboardingComplete(
roles.chat_model_id,
globalConnections,
connections
);
const isLoading =
session.status === "loading" || globalLoading || rolesLoading || globalConfigStatusLoading;
// Onboarding only applies when no global_llm_config.yaml exists. If a global
// config is present (or onboarding is already complete), leave this page.
const shouldLeaveOnboarding =
!isLoading && (Boolean(globalConfigStatus?.exists) || onboardingComplete);
useEffect(() => {
if (shouldLeaveOnboarding) {
router.replace(`/dashboard/${workspaceId}/new-chat`);
}
}, [shouldLeaveOnboarding, router, workspaceId]);
useGlobalLoadingEffect(isLoading || shouldLeaveOnboarding);
if (isLoading || shouldLeaveOnboarding) return null;
// Leaving onboarding is the layout gate's job; here we only enable the
// explicit CTA once the workspace can chat.
const isReady = setupStatus?.status === "ready";
return (
<div className="flex min-h-screen select-none flex-col items-center justify-center bg-main-panel p-4">
@ -80,8 +46,8 @@ export default function OnboardPage() {
footerAction={
<Button
className="min-w-[112px]"
disabled={!onboardingComplete || !hasUsableChatModel}
onClick={() => router.push(`/dashboard/${workspaceId}/new-chat`)}
disabled={!isReady}
onClick={() => router.replace(`/dashboard/${workspaceId}/new-chat`)}
>
Start
</Button>

9
surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx
View file

@ -14,6 +14,11 @@ import { AppError } from "@/lib/error";
import { findVerb } from "@/lib/playground/catalog";
import { formatCost, formatDuration, formatPricing } from "@/lib/playground/format";
import { buildPayload, initialFormValues, parseSchemaFields } from "@/lib/playground/json-schema";
import {
AmazonMarketplaceHint,
getAmazonFieldOptions,
hasAmazonFranceValue,
} from "@/lib/playground/platform-overrides/amazon";
import { ApiReference } from "./api-reference";
import { OutputViewer } from "./output-viewer";
import { RunProgressPanel } from "./run-progress-panel";
@ -171,6 +176,8 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn
[run.detail]
);
const endpoint = `POST /workspaces/${workspaceId}/scrapers/${platform}/${verb}`;
const isAmazonScrape = platform === "amazon" && verb === "scrape";
const hasAmazonFranceUrl = useMemo(() => hasAmazonFranceValue(values), [values]);
useEffect(() => {
const previousStatus = previousStatusRef.current;
@ -260,7 +267,9 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn
values={values}
onChange={handleChange}
disabled={isRunning}
getFieldOptions={isAmazonScrape ? getAmazonFieldOptions : undefined}
/>
{isAmazonScrape ? <AmazonMarketplaceHint showFranceWarning={hasAmazonFranceUrl} /> : null}
<div className="flex items-center gap-2">
<Button type="button" onClick={handleRun} disabled={isRunning} className="relative">

45
surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx
View file

@ -17,11 +17,19 @@ import { Textarea } from "@/components/ui/textarea";
import type { FormField } from "@/lib/playground/json-schema";
import { cn } from "@/lib/utils";
export interface FieldOption {
label: string;
value: string;
}
type FieldOptionsResolver = (field: FormField) => FieldOption[] | undefined;
interface SchemaFormProps {
fields: FormField[];
values: Record<string, unknown>;
onChange: (name: string, value: unknown) => void;
disabled?: boolean;
getFieldOptions?: FieldOptionsResolver;
/** Field names flagged by a 422 response, shown with error styling. */
fieldErrors?: Record<string, string>;
}
@ -32,12 +40,14 @@ function FieldControl({
onChange,
disabled,
invalid,
options,
}: {
field: FormField;
value: unknown;
onChange: (value: unknown) => void;
disabled?: boolean;
invalid?: boolean;
options?: FieldOption[];
}) {
const id = `field-${field.name}`;
@ -68,6 +78,27 @@ function FieldControl({
);
}
if (options) {
return (
<Select
value={value ? String(value) : options[0]?.value}
onValueChange={onChange}
disabled={disabled}
>
<SelectTrigger id={id} className={cn("w-full", invalid && "border-destructive")}>
<SelectValue placeholder="Select…" />
</SelectTrigger>
<SelectContent>
{options.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label} ({option.value})
</SelectItem>
))}
</SelectContent>
</Select>
);
}
if (field.kind === "string_array") {
return (
<Textarea
@ -116,12 +147,14 @@ function FieldRow({
onChange,
disabled,
error,
options,
}: {
field: FormField;
value: unknown;
onChange: (value: unknown) => void;
disabled?: boolean;
error?: string;
options?: FieldOption[];
}) {
return (
<div className="space-y-1.5">
@ -143,13 +176,21 @@ function FieldRow({
onChange={onChange}
disabled={disabled}
invalid={!!error}
options={options}
/>
{error && <p className="text-xs text-destructive">{error}</p>}
</div>
);
}
export function SchemaForm({ fields, values, onChange, disabled, fieldErrors }: SchemaFormProps) {
export function SchemaForm({
fields,
values,
onChange,
disabled,
getFieldOptions,
fieldErrors,
}: SchemaFormProps) {
const [showAdvanced, setShowAdvanced] = useState(false);
const { primary, advanced } = useMemo(() => {
@ -168,6 +209,7 @@ export function SchemaForm({ fields, values, onChange, disabled, fieldErrors }:
onChange={(value) => onChange(field.name, value)}
disabled={disabled}
error={fieldErrors?.[field.name]}
options={getFieldOptions?.(field)}
/>
))}
@ -197,6 +239,7 @@ export function SchemaForm({ fields, values, onChange, disabled, fieldErrors }:
onChange={(value) => onChange(field.name, value)}
disabled={disabled}
error={fieldErrors?.[field.name]}
options={getFieldOptions?.(field)}
/>
))}
</div>

33
surfsense_web/app/layout.tsx
View file

@ -50,27 +50,26 @@ export const metadata: Metadata = {
alternates: {
canonical: "https://www.surfsense.com",
},
title: "SurfSense - NotebookLM for Competitive Intelligence",
title: "SurfSense - Open-Source NotebookLM Alternative for Web Research",
description:
"SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market through one API or MCP server.",
"The open-source NotebookLM for open web research. Your AI agents pull live, structured data from Reddit, YouTube, Amazon, and any page via one API or MCP server.",
keywords: [
"competitive intelligence platform",
"competitive intelligence software",
"market intelligence tools",
"competitor monitoring",
"competitor price monitoring",
"brand monitoring",
"rank tracking",
"social listening",
"open web research platform",
"web research for AI agents",
"live web data for agents",
"web scraping API",
"reddit scraper api",
"youtube scraper api",
"deep research agent",
"mcp server",
"agent harness",
"open source competitive intelligence",
"open source NotebookLM alternative",
"SurfSense",
],
openGraph: {
title: "SurfSense - NotebookLM for Competitive Intelligence",
title: "SurfSense - NotebookLM for Open Web Research",
description:
"SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market through one API or MCP server.",
"The open-source NotebookLM alternative for AI agents. Research the live web with structured data from Reddit, YouTube, Amazon, Google Maps, and Google Search, through one API or MCP server.",
url: "https://www.surfsense.com",
siteName: "SurfSense",
type: "website",
@ -79,16 +78,16 @@ export const metadata: Metadata = {
url: "/og-image.png",
width: 1200,
height: 630,
alt: "SurfSense - Competitive Intelligence Platform for AI Agents",
alt: "SurfSense, open-source NotebookLM alternative for AI agents",
},
],
locale: "en_US",
},
twitter: {
card: "summary_large_image",
title: "SurfSense - NotebookLM for Competitive Intelligence",
title: "SurfSense - NotebookLM for Open Web Research",
description:
"SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market through one API or MCP server.",
"The open-source NotebookLM alternative for AI agents. Research the live web with structured data from Reddit, YouTube, Amazon, Google Maps, and Google Search, through one API or MCP server.",
creator: "@SurfSenseAI",
site: "@SurfSenseAI",
images: [
@ -96,7 +95,7 @@ export const metadata: Metadata = {
url: "/og-image-twitter.png",
width: 1200,
height: 630,
alt: "SurfSense - Competitive Intelligence Platform for AI Agents",
alt: "SurfSense, open-source NotebookLM alternative for AI agents",
},
],
},

32
surfsense_web/atoms/model-connections/model-connections-mutation.atoms.ts
View file

@ -25,6 +25,27 @@ function invalidateModelConnections(workspaceId: number) {
queryClient.invalidateQueries({
queryKey: cacheKeys.modelConnections.roles(workspaceId),
});
queryClient.invalidateQueries({
queryKey: cacheKeys.modelConnections.setupStatus(workspaceId),
});
}
// After a mutation that can remove the workspace's last usable chat model,
// surface immediate feedback. The composer's inline notice covers the state on
// its own; this just makes the consequence obvious at the moment of the action.
async function warnIfWorkspaceChatDisabled(workspaceId: number) {
if (workspaceId <= 0) return;
try {
const status = await queryClient.fetchQuery({
queryKey: cacheKeys.modelConnections.setupStatus(workspaceId),
queryFn: () => modelConnectionsApiService.getLlmSetupStatus(workspaceId),
});
if (status?.status === "needs_setup") {
toast.warning("Chat is now disabled. Connect a chat model to start chatting again.");
}
} catch {
// Non-fatal: the inline composer notice still reflects the state.
}
}
function upsertModelConnection(workspaceId: number, connection: ConnectionRead) {
@ -78,9 +99,10 @@ export const deleteModelConnectionMutationAtom = atomWithMutation((get) => {
return {
mutationKey: ["model-connections", "delete"],
mutationFn: (id: number) => modelConnectionsApiService.deleteConnection(id),
onSuccess: () => {
onSuccess: async () => {
toast.success("Connection deleted");
invalidateModelConnections(workspaceId);
await warnIfWorkspaceChatDisabled(workspaceId);
},
onError: (error: Error) => toast.error(error.message || "Failed to delete connection"),
};
@ -179,7 +201,10 @@ export const bulkUpdateModelsMutationAtom = atomWithMutation((get) => {
mutationKey: ["models", "bulk-update"],
mutationFn: ({ connectionId, data }: { connectionId: number; data: ModelsBulkUpdateRequest }) =>
modelConnectionsApiService.bulkUpdateModels(connectionId, data),
onSuccess: () => invalidateModelConnections(workspaceId),
onSuccess: async () => {
invalidateModelConnections(workspaceId);
await warnIfWorkspaceChatDisabled(workspaceId);
},
onError: (error: Error) => toast.error(error.message || "Failed to update models"),
};
});
@ -208,6 +233,9 @@ export const updateModelRolesMutationAtom = atomWithMutation((get) => {
queryClient.invalidateQueries({
queryKey: cacheKeys.modelConnections.roles(workspaceId),
});
queryClient.invalidateQueries({
queryKey: cacheKeys.modelConnections.setupStatus(workspaceId),
});
},
onError: (error: Error) => toast.error(error.message || "Failed to update model roles"),
};

16
surfsense_web/atoms/model-connections/model-connections-query.atoms.ts
View file

@ -1,3 +1,4 @@
import { atomFamily } from "jotai-family";
import { atomWithQuery } from "jotai-tanstack-query";
import { modelConnectionsApiService } from "@/lib/apis/model-connections-api.service";
import { isAuthenticated } from "@/lib/auth-utils";
@ -44,3 +45,18 @@ export const modelRolesAtom = atomWithQuery((get) => {
queryFn: () => modelConnectionsApiService.getModelRoles(workspaceId),
};
});
// Keyed by the route workspaceId (not activeWorkspaceIdAtom) so the onboarding
// gate's verdict can never be computed against a stale workspace during a
// workspace switch.
export const llmSetupStatusAtomFamily = atomFamily((workspaceId: number) =>
atomWithQuery(() => ({
queryKey: cacheKeys.modelConnections.setupStatus(workspaceId),
enabled: workspaceId > 0 && isAuthenticated(),
staleTime: 5 * 60 * 1000,
// Recovery is event-driven: mutations invalidate this key; external fixes
// are caught on window focus. No polling, so not-ready tabs cost nothing.
refetchOnWindowFocus: true,
queryFn: () => modelConnectionsApiService.getLlmSetupStatus(workspaceId),
}))
);

16
surfsense_web/changelog/content/2026-07-05.mdx
View file

@ -2,11 +2,11 @@
date: "2026-07-05"
---
## SurfSense's Next Chapter: Competitive Intelligence for AI Agents
## SurfSense's Next Chapter: Open Web Research for AI Agents
Happy Independence Day to everyone celebrating in the United States! We hope you had a wonderful Fourth of July. It felt like the right weekend for a declaration of our own.
Today we are pointing SurfSense at a single mission: **give your AI agents competitive intelligence.**
Today we are pointing SurfSense at a single mission: **give your AI agents the primitives to research the open web.**
### How we got here
@ -14,20 +14,20 @@ For the past couple of months we set out to build the best general research agen
That same period settled a bigger question. Agentic tools like Claude, OpenCode, Hermes, and OpenClaw have proven themselves. Agents that plan, use tools, and finish multi-step work on their own are no longer a demo, they are how work gets done. General research assistance is becoming something every capable agent does out of the box.
So we asked what these agents still lack. Not reasoning. Not tools. **Live market data and the workflows around it.** An agent asked "what are competitors charging this week?" or "what is Reddit saying about us since the launch?" has nowhere trustworthy to look. We want to be one of the first, and hopefully the definitive, open-source competitive intelligence agent platforms. This focus also lets our team pour all of our energy into the high-value workflows that genuinely need AI automation.
So we asked what these agents still lack. Not reasoning. Not tools. **Live data from the places where answers actually live, and the workflows around it.** An agent asked "what is Reddit saying about this product since launch?" or "what do the reviews of these ten places actually complain about?" has nowhere trustworthy to look. Official platform APIs are rate-limited or missing, scraping plumbing is brittle, and driving a browser with an LLM burns minutes and tokens per page. We want to be the open-source platform that gives agents those primitives.
### What SurfSense is now
- **Platform-native connectors for live market data.** [Reddit](/reddit), [YouTube](/youtube), [Google Maps](/google-maps), [Google Search](/google-search), and the [open web](/web-crawl). Each one is a typed REST endpoint you can call from any language with your SurfSense API key.
- **Platform-native connectors for live web data.** [Reddit](/reddit), [YouTube](/youtube), [Google Maps](/google-maps), [Google Search](/google-search), and the [open web](/web-crawl). Each one is a typed REST endpoint returning structured JSON — posts, comments, transcripts, reviews, SERPs, pages — that you can call from any language with your SurfSense API key. And the catalog keeps growing beyond social platforms and search.
- **An MCP server that makes every connector an agent tool.** Add the [SurfSense MCP server](/mcp-server) to Claude, Cursor, or your own agent framework. And with [external MCP connectors](/external-mcp-connectors), your SurfSense agents can use any MCP server too, with one-click OAuth for apps like Notion, Slack, and Jira.
- **An agent harness, not just raw data.** Retries, structured output, and credit metering are built in, so agents go from a question to a brief without you building the plumbing.
- **Automations for the workflows that matter.** Scheduled and event-triggered agents watch competitor pricing, track brand mentions, monitor rankings and reviews, then deliver briefs and alerts.
- **An agent harness, not just raw data.** Retries, structured output, and credit metering are built in, so agents go from a question to a cited brief without you building the plumbing.
- **Automations for the workflows that matter.** Scheduled and event-triggered agents watch pages, track mentions, monitor rankings and reviews, then deliver briefs and alerts.
- **Simpler pricing.** Self-hosting stays free with no metering. Cloud is pay as you go: start with $5 of free credit, connectors bill per item returned, crawls per page fetched, models at provider cost. Failed calls are never billed. Details on the [pricing page](/pricing).
### To our existing users: nothing you rely on is going away
This is an addition, not a replacement. Your knowledge base, chat with citations, reports, podcasts, presentations, automations, and collaborative chats all keep working exactly as they do today. You can still upload files and sync Google Drive, OneDrive, and Dropbox. Self-hosting stays free and open source, and your existing deployments keep running without any action from you. What changes is where we invest: expect deeper connectors, sharper agent workflows, and broader market coverage.
This is an addition, not a replacement. Your knowledge base, chat with citations, reports, podcasts, presentations, automations, and collaborative chats all keep working exactly as they do today. You can still upload files and sync Google Drive, OneDrive, and Dropbox. Self-hosting stays free and open source, and your existing deployments keep running without any action from you. What changes is where we invest: expect deeper connectors, sharper agent workflows, and broader coverage of the open web.
If you are building agents, [grab an API key](/login) or add the MCP server, and start with the [connector catalog](/connectors). Tell us what you are building on [Discord](https://discord.gg/ejRNvftDp9) or [GitHub](https://github.com/MODSetter/SurfSense).
We think the next great competitive intelligence tool will not be a dashboard. It will be an agent that already knows your market. We are building the platform it runs on.
We think the next great research tool will not be another dashboard or another chat window. It will be an agent that can actually reach the live web. We are building the platform it runs on.

23
surfsense_web/components/announcements/AnnouncementSpotlight.tsx
View file

@ -20,14 +20,10 @@ import { useAnnouncements } from "@/hooks/use-announcements";
* Behaviour:
* - On load, the first active, audience-matched, unread spotlight announcement
* is shown automatically.
* - The user must explicitly acknowledge it ("Got it" or the CTA link), which
* marks it as read so it never shows again.
* - Closing via the X / Escape / outside-click only hides it for the current
* session; it reappears on the next load until the user marks it as seen.
* - Any dismissal marks it as read so it never shows again.
*/
export function AnnouncementSpotlight() {
const { announcements, markRead } = useAnnouncements();
const [sessionDismissed, setSessionDismissed] = useState<Set<string>>(() => new Set());
const [ready, setReady] = useState(false);
// Short delay so the spotlight doesn't flash during initial hydration/layout.
@ -37,11 +33,8 @@ export function AnnouncementSpotlight() {
}, []);
const current = useMemo(
() =>
announcements.find(
(a) => a.spotlight && a.isImportant && !a.isRead && !sessionDismissed.has(a.id)
) ?? null,
[announcements, sessionDismissed]
() => announcements.find((a) => a.spotlight && a.isImportant && !a.isRead) ?? null,
[announcements]
);
if (!current) return null;
@ -51,13 +44,7 @@ export function AnnouncementSpotlight() {
};
const handleOpenChange = (next: boolean) => {
if (!next) {
setSessionDismissed((prev) => {
const updated = new Set(prev);
updated.add(current.id);
return updated;
});
}
if (!next) markRead(current.id);
};
return (
@ -82,7 +69,7 @@ export function AnnouncementSpotlight() {
</DialogDescription>
<DialogFooter className="mt-2">
{current.link && (
<Button variant="outline" asChild className="gap-1.5" onClick={handleAcknowledge}>
<Button variant="secondary" asChild className="gap-1.5" onClick={handleAcknowledge}>
<Link
href={current.link.url}
target={current.link.url.startsWith("http") ? "_blank" : undefined}

2
surfsense_web/components/announcements/AnnouncementsDialog.tsx
View file

@ -11,7 +11,7 @@ import { useAnnouncements } from "@/hooks/use-announcements";
export function AnnouncementsDialog() {
const [open, setOpen] = useAtom(announcementsDialogAtom);
const { announcements, markAllRead } = useAnnouncements();
const { announcements, markAllRead } = useAnnouncements({ includeExpired: true });
// Auto-mark all visible announcements as read when the dialog opens
useEffect(() => {

2
surfsense_web/components/assistant-ui/connector-popup.tsx
View file

@ -189,7 +189,7 @@ export const ConnectorIndicator = forwardRef<ConnectorIndicatorHandle, Connector
}}
className="max-w-3xl w-[95vw] sm:w-full h-[75vh] sm:h-[85vh] flex flex-col p-0 gap-0 overflow-hidden ring-0 dark:ring-0 [&>button]:right-4 sm:[&>button]:right-12 [&>button]:top-6 sm:[&>button]:top-10 [&>button]:opacity-80 [&>button]:hover:opacity-100 [&>button]:hover:bg-accent [&>button]:hover:text-accent-foreground [&>button>svg]:size-5 select-none"
>
<DialogTitle className="sr-only">Manage External MCP Connectors</DialogTitle>
<DialogTitle className="sr-only">MCP Connectors</DialogTitle>
{/* YouTube Crawler View - shown when adding YouTube videos */}
{isYouTubeView && workspaceId ? (
<YouTubeCrawlerView workspaceId={workspaceId} onBack={handleBackFromYouTube} />

32
surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx
View file

@ -79,7 +79,7 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
}
if (isDeprecatedForConnect) {
return "Deprecated — no longer available to connect.";
return "Deprecated. No longer available to connect.";
}
return description;
@ -102,7 +102,16 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
: "bg-slate-400/5 dark:bg-white/5 border-slate-400/5 dark:border-white/5"
)}
>
{connectorType ? (
{isMCP ? (
<span
aria-hidden="true"
className="size-6 bg-current"
style={{
mask: "url('/connectors/modelcontextprotocol.svg') center / contain no-repeat",
WebkitMask: "url('/connectors/modelcontextprotocol.svg') center / contain no-repeat",
}}
/>
) : connectorType ? (
getConnectorIcon(connectorType, "size-6")
) : id === "youtube-crawler" ? (
<IconBrandYoutube className="size-6" />
@ -113,19 +122,12 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
<div className="flex-1 min-w-0">
<div className="flex items-center gap-1.5">
<span className="text-[14px] font-semibold leading-tight truncate">{title}</span>
{deprecated ? (
<span className="shrink-0 rounded-full bg-muted px-1.5 py-0.5 text-[9px] font-medium uppercase tracking-wide text-muted-foreground">
Deprecated
</span>
) : (
showWarnings &&
status.status !== "active" && (
<ConnectorStatusBadge
status={status.status}
statusMessage={statusMessage}
className="flex-shrink-0"
/>
)
{showWarnings && status.status !== "active" && (
<ConnectorStatusBadge
status={status.status}
statusMessage={statusMessage}
className="flex-shrink-0"
/>
)}
</div>
{isIndexing ? (

4
surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx
View file

@ -31,10 +31,10 @@ export const ConnectorDialogHeader: FC<ConnectorDialogHeaderProps> = ({
>
<DialogHeader>
<DialogTitle className="text-xl sm:text-3xl font-semibold tracking-tight">
Manage External MCP Connectors
MCP Connectors
</DialogTitle>
<DialogDescription className="text-xs sm:text-base text-muted-foreground/80 mt-1 sm:mt-1.5">
Connect Surfsense to your favorite tools and services.
Connect external tools and services through MCP.
</DialogDescription>
</DialogHeader>

2
surfsense_web/components/assistant-ui/image.tsx
View file

@ -233,4 +233,4 @@ Image.Preview = ImagePreview;
Image.Filename = ImageFilename;
Image.Zoom = ImageZoom;
export { Image, ImageRoot, ImagePreview, ImageFilename, ImageZoom, imageVariants };
export { Image, ImageFilename, ImagePreview, ImageRoot, ImageZoom, imageVariants };

120
surfsense_web/components/assistant-ui/thread.tsx
View file

@ -24,7 +24,7 @@ import {
} from "lucide-react";
import { AnimatePresence, motion } from "motion/react";
import Image from "next/image";
import { useParams } from "next/navigation";
import { useParams, useRouter } from "next/navigation";
import { type FC, useCallback, useEffect, useMemo, useRef, useState } from "react";
import {
agentToolsAtom,
@ -47,11 +47,7 @@ import {
import { connectorDialogOpenAtom } from "@/atoms/connector-dialog/connector-dialog.atoms";
import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
import { membersAtom } from "@/atoms/members/members-query.atoms";
import {
globalModelConnectionsAtom,
modelConnectionsAtom,
modelRolesAtom,
} from "@/atoms/model-connections/model-connections-query.atoms";
import { llmSetupStatusAtomFamily } from "@/atoms/model-connections/model-connections-query.atoms";
import { currentUserAtom } from "@/atoms/user/user-query.atoms";
import { AssistantMessage } from "@/components/assistant-ui/assistant-message";
import { ChatSessionStatus } from "@/components/assistant-ui/chat-session-status";
@ -119,7 +115,7 @@ import {
} from "../new-chat/document-mention-picker";
const COMPOSER_PLACEHOLDER =
"Track competitors, scrape platforms, automate briefs. Use / for prompts, @ for docs";
"Research the live web, scrape platforms, automate briefs. Use / for prompts, @ for docs";
type ComposerSuggestionAnchorPoint = {
left: number;
@ -262,17 +258,17 @@ const ThreadWelcome: FC = () => {
const greeting = useMemo(() => getTimeBasedGreeting(user), [user]);
return (
<div className="aui-thread-welcome-root mx-auto flex w-full max-w-(--thread-max-width) grow flex-col items-center px-4 relative">
<div className="my-auto flex w-full flex-col items-center gap-6 py-6 sm:contents sm:my-0 sm:gap-0 sm:py-0">
<div className="aui-thread-welcome-message flex flex-col items-center text-center sm:absolute sm:bottom-[calc(50%+5rem)] sm:left-0 sm:right-0">
<div className="aui-thread-welcome-root flex min-h-0 flex-1">
<section className="mx-auto grid w-full max-w-(--thread-max-width) content-center gap-6 pt-8 pb-[clamp(5rem,16vh,12rem)]">
<div className="aui-thread-welcome-message flex flex-col items-center px-4 text-center">
<h1 className="aui-thread-welcome-message-inner text-3xl md:text-[2.625rem] select-none">
{greeting}
</h1>
</div>
<div className="w-full flex items-start justify-center sm:absolute sm:top-[calc(50%-3.5rem)] sm:left-0 sm:right-0">
<div className="flex w-full items-start justify-center">
<Composer />
</div>
</div>
</section>
</div>
);
};
@ -448,6 +444,52 @@ const ClipboardChip: FC<{ text: string; onDismiss: () => void }> = ({ text, onDi
);
};
/**
* Inline "this workspace can't chat yet" tray, mirrored on top of the composer
* (same visual treatment as the "Connect your tools" tray below it).
*
* Replaces the old full-screen onboarding redirect: a workspace with no usable
* chat model (fresh install, or an admin deleted the last model) renders the
* composer in place with this notice instead of navigating away. Configurable
* members get a CTA to connect one; everyone else is told to ask an admin. The
* send button stays disabled independently, and the backend rejects any send
* that slips through, so this is purely guidance.
*
* Presentational: visibility is decided by the parent so the composer can flatten
* its top shadow for a seamless join.
*/
const ChatUnavailableNotice: FC<{ workspaceId: number; canConfigure: boolean }> = ({
workspaceId,
canConfigure,
}) => {
const router = useRouter();
return (
<div className="relative z-0 -mb-5 flex min-w-0 items-center gap-2 rounded-t-3xl bg-popover px-4 pt-2 pb-6 shadow-sm shadow-black/5 dark:shadow-black/10">
<div className="flex min-w-0 items-center gap-2 text-[13px] font-normal text-muted-foreground select-none">
<AlertCircle className="size-4 shrink-0" />
<span className="truncate">
{canConfigure
? "Connect a chat model to start chatting."
: "No model available. Ask a workspace admin to connect a chat model."}
</span>
</div>
<div className="min-w-0 flex-1" />
{canConfigure ? (
<Button
type="button"
size="sm"
className="h-6 shrink-0 cursor-pointer gap-2 rounded-md px-2.5 text-xs font-medium select-none"
onClick={() => router.push(`/dashboard/${workspaceId}/workspace-settings/models`)}
>
<span className="sm:hidden">Connect</span>
<span className="hidden sm:inline">Connect a model</span>
</Button>
) : null}
</div>
);
};
const Composer: FC = () => {
const [mentionedDocuments, setMentionedDocuments] = useAtom(mentionedDocumentsAtom);
const setSubmittedMentions = useSetAtom(submittedMentionsAtom);
@ -487,6 +529,9 @@ const Composer: FC = () => {
const isThreadRunning = useAuiState(({ thread }) => thread.isRunning);
const [connectToolsTrayVisible, setConnectToolsTrayVisible] = useState(false);
const { data: chatSetupStatus } = useAtomValue(llmSetupStatusAtomFamily(workspaceId ?? 0));
const isChatUnavailable = !!chatSetupStatus && chatSetupStatus.status !== "ready";
const currentPlaceholder = COMPOSER_PLACEHOLDER;
const { data: currentUser } = useAtomValue(currentUserAtom);
@ -934,10 +979,17 @@ const Composer: FC = () => {
) : null}
</Popover>
<div className="relative flex w-full flex-col">
{isChatUnavailable ? (
<ChatUnavailableNotice
workspaceId={workspaceId ?? 0}
canConfigure={chatSetupStatus?.can_configure ?? false}
/>
) : null}
<div
className={cn(
"aui-composer-attachment-dropzone relative z-10 flex w-full flex-col overflow-hidden rounded-3xl border border-input/20 bg-muted pt-2 shadow-sm shadow-black/5 outline-none transition-[border-color,box-shadow] hover:border-input/60 focus-within:border-input/60 dark:shadow-black/10",
connectToolsTrayVisible && "rounded-b-3xl shadow-none dark:shadow-none"
connectToolsTrayVisible && "rounded-b-3xl shadow-none dark:shadow-none",
isChatUnavailable && "shadow-none dark:shadow-none"
)}
>
<PendingScreenImageStrip />
@ -1071,9 +1123,7 @@ const ComposerAction: FC<ComposerActionProps> = ({
if (url) setPendingScreenImages((prev) => [...prev, url]);
}, [electronAPI, setPendingScreenImages]);
const { data: globalModelConnections } = useAtomValue(globalModelConnectionsAtom);
const { data: modelConnections } = useAtomValue(modelConnectionsAtom);
const { data: modelRoles } = useAtomValue(modelRolesAtom);
const { data: setupStatus } = useAtomValue(llmSetupStatusAtomFamily(workspaceId));
const { data: agentTools } = useAtomValue(agentToolsAtom);
const disabledTools = useAtomValue(disabledToolsAtom);
@ -1154,21 +1204,13 @@ const ComposerAction: FC<ComposerActionProps> = ({
hydrateDisabled();
}, [hydrateDisabled]);
const hasModelConfigured = useMemo(() => {
const chatModelId = modelRoles?.chat_model_id ?? 0;
if (chatModelId === 0) {
return [...(globalModelConnections ?? []), ...(modelConnections ?? [])].some((connection) =>
connection.models.some((model) => model.enabled && Boolean(model.supports_chat))
);
}
return [...(globalModelConnections ?? []), ...(modelConnections ?? [])].some((connection) =>
connection.models.some(
(model) => model.id === chatModelId && model.enabled && Boolean(model.supports_chat)
)
);
}, [modelRoles?.chat_model_id, globalModelConnections, modelConnections]);
// A workspace with no usable chat model renders the composer with an inline
// notice (see ChatUnavailableNotice) rather than being redirected away, so
// the send button must stay disabled here. The backend also rejects any
// send that lacks a resolvable model, making this defense-in-depth.
const isWorkspaceChatReady = setupStatus?.status === "ready";
const isSendDisabled = isComposerEmpty || !hasModelConfigured || isBlockedByOtherUser;
const isSendDisabled = isComposerEmpty || !isWorkspaceChatReady || isBlockedByOtherUser;
return (
<div className="aui-composer-action-wrapper relative mx-3 mb-3 flex items-center justify-between">
@ -1194,7 +1236,7 @@ const ComposerAction: FC<ComposerActionProps> = ({
</DropdownMenuItem>
<DropdownMenuItem onSelect={() => setConnectorDialogOpen(true)}>
<Unplug className="size-4" />
Manage External MCP Connectors
MCP Connectors
</DropdownMenuItem>
<DropdownMenuItem onSelect={() => setToolsPopoverOpen(true)}>
<Settings2 className="size-4" />
@ -1414,7 +1456,7 @@ const ComposerAction: FC<ComposerActionProps> = ({
</DropdownMenuItem>
<DropdownMenuItem onSelect={() => setConnectorDialogOpen(true)}>
<Unplug className="h-4 w-4" />
Manage External MCP Connectors
MCP Connectors
</DropdownMenuItem>
<DropdownMenuSub
open={toolsPopoverOpen}
@ -1610,12 +1652,6 @@ const ComposerAction: FC<ComposerActionProps> = ({
)}
<ConnectedScraperIcons workspaceId={workspaceId} />
</div>
{!hasModelConfigured && (
<div className="flex items-center gap-1.5 text-amber-600 dark:text-amber-400 text-xs">
<AlertCircle className="size-3" />
<span>Select a model</span>
</div>
)}
<div className="ml-auto flex min-w-0 shrink-0 items-center gap-2">
<ChatHeader
workspaceId={workspaceId}
@ -1628,11 +1664,9 @@ const ComposerAction: FC<ComposerActionProps> = ({
tooltip={
isBlockedByOtherUser
? "Wait for AI to finish responding"
: !hasModelConfigured
? "Please select a model to start chatting"
: isComposerEmpty
? "Enter a message or add a screenshot to send"
: "Send message"
: isComposerEmpty
? "Enter a message or add a screenshot to send"
: "Send message"
}
side="bottom"
type="submit"

2
surfsense_web/components/connectors-marketing/connector-page.tsx
View file

@ -277,7 +277,7 @@ export function ConnectorPage({ content }: { content: ConnectorPageContent }) {
<p className="mx-auto mt-3 max-w-xl text-muted-foreground leading-relaxed">
The {content.name} connector is one of many in the SurfSense{" "}
<Link href="/" className="font-medium text-foreground underline underline-offset-4">
competitive intelligence platform
open web research platform
</Link>
. Start free, no credit card required.
</p>

88
surfsense_web/components/documents/DocumentNode.tsx
View file

@ -37,6 +37,7 @@ import {
import { Spinner } from "@/components/ui/spinner";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import type { DocumentTypeEnum } from "@/contracts/types/document.types";
import { useIsMobile } from "@/hooks/use-mobile";
import { cn } from "@/lib/utils";
import { SidebarListItem } from "../layout/ui/sidebar/SidebarListItem";
import { DND_TYPES } from "./FolderNode";
@ -92,6 +93,7 @@ export const DocumentNode = React.memo(function DocumentNode({
const isMemoryDocument =
doc.document_type === "USER_MEMORY" || doc.document_type === "TEAM_MEMORY";
const isSelectable = canMention && !isUnavailable;
const isMobile = useIsMobile();
const handleCheckChange = useCallback(() => {
if (isSelectable) {
@ -164,9 +166,9 @@ export const DocumentNode = React.memo(function DocumentNode({
<ContextMenuTrigger asChild>
<SidebarListItem
ref={attachRef}
active={isMentioned}
active={isMentioned || dropdownOpen}
dragging={isDragging}
className="gap-2.5 px-1"
className="group/item relative gap-2.5 px-1"
style={{ paddingLeft: `${depth * 16 + 4}px` }}
role="button"
tabIndex={isUnavailable ? -1 : 0}
@ -214,32 +216,34 @@ export const DocumentNode = React.memo(function DocumentNode({
);
}
return (
<>
{isMemoryDocument ? (
<span aria-disabled="true" className="h-3.5 w-3.5 shrink-0 cursor-default">
<Checkbox
checked={false}
disabled
aria-disabled
className="h-3.5 w-3.5 pointer-events-none"
/>
</span>
) : canMention ? (
<span className="relative flex h-3.5 w-3.5 shrink-0 items-center justify-center">
<span
className={cn(
"absolute inset-0 flex items-center justify-center transition-opacity",
canMention &&
(isMentioned ? "opacity-0" : "max-sm:opacity-0 group-hover/item:opacity-0")
)}
>
{getDocumentTypeIcon(
doc.document_type as DocumentTypeEnum,
"h-3.5 w-3.5 text-muted-foreground"
)}
</span>
{canMention ? (
<Checkbox
checked={isMentioned}
aria-label={`Select ${doc.title}`}
onCheckedChange={handleCheckChange}
onClick={(e) => e.stopPropagation()}
className="h-3.5 w-3.5 shrink-0"
/>
) : (
<span className="flex h-3.5 w-3.5 shrink-0 items-center justify-center">
{getDocumentTypeIcon(
doc.document_type as DocumentTypeEnum,
"h-3.5 w-3.5 text-muted-foreground"
className={cn(
"absolute h-3.5 w-3.5 transition-opacity max-sm:pointer-events-auto max-sm:opacity-100",
isMentioned
? "opacity-100"
: "pointer-events-none opacity-0 group-hover/item:pointer-events-auto group-hover/item:opacity-100 focus-visible:pointer-events-auto focus-visible:opacity-100"
)}
</span>
)}
</>
/>
) : null}
</span>
);
})()}
@ -260,36 +264,30 @@ export const DocumentNode = React.memo(function DocumentNode({
</TooltipContent>
</Tooltip>
<span className="relative shrink-0 flex items-center justify-center h-6 w-6">
{getDocumentTypeIcon(
doc.document_type as DocumentTypeEnum,
"h-3.5 w-3.5 text-muted-foreground"
) && (
<span
className={cn(
"absolute inset-0 flex items-center justify-center transition-opacity pointer-events-none",
dropdownOpen ? "opacity-0" : "group-hover:opacity-0"
)}
>
{getDocumentTypeIcon(
doc.document_type as DocumentTypeEnum,
"h-3.5 w-3.5 text-muted-foreground"
)}
</span>
<div
className={cn(
"pointer-events-none absolute top-0 right-0 bottom-0 flex items-center rounded-r-md pr-1 pl-6",
isMentioned || dropdownOpen
? "bg-gradient-to-l from-accent from-60% to-transparent"
: "bg-gradient-to-l from-sidebar from-60% to-transparent group-hover/item:from-accent",
isMobile
? "opacity-0"
: isMentioned || dropdownOpen
? "opacity-100"
: "opacity-0 group-hover/item:opacity-100"
)}
>
<DropdownMenu open={dropdownOpen} onOpenChange={setDropdownOpen}>
<DropdownMenuTrigger asChild>
<Button
variant="ghost"
size="icon"
className={cn(
"hidden sm:inline-flex h-6 w-6 shrink-0 hover:bg-transparent",
dropdownOpen
? "opacity-100 bg-accent hover:bg-accent"
: "opacity-0 group-hover:opacity-100"
"pointer-events-auto hidden h-6 w-6 shrink-0 hover:bg-transparent sm:inline-flex",
dropdownOpen && "bg-accent hover:bg-accent"
)}
onClick={(e) => e.stopPropagation()}
aria-label={`Document actions for ${doc.title}`}
>
<MoreHorizontal className="h-3.5 w-3.5 text-muted-foreground" />
</Button>
@ -341,7 +339,7 @@ export const DocumentNode = React.memo(function DocumentNode({
)}
</DropdownMenuContent>
</DropdownMenu>
</span>
</div>
</SidebarListItem>
</ContextMenuTrigger>

216
surfsense_web/components/documents/FolderNode.tsx
View file

@ -34,6 +34,7 @@ import {
} from "@/components/ui/dropdown-menu";
import { Spinner } from "@/components/ui/spinner";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { useIsMobile } from "@/hooks/use-mobile";
import { cn } from "@/lib/utils";
import type { FolderSelectionState } from "./FolderTreeView";
@ -124,6 +125,7 @@ export const FolderNode = React.memo(function FolderNode({
onStopWatching,
onExportFolder,
}: FolderNodeProps) {
const isMobile = useIsMobile();
const [renameValue, setRenameValue] = useState(folder.name);
const inputRef = useRef<HTMLInputElement>(null);
const rowRef = useRef<HTMLDivElement>(null);
@ -261,8 +263,9 @@ export const FolderNode = React.memo(function FolderNode({
tabIndex={0}
dragging={isDragging}
className={cn(
"relative gap-1 px-1",
"group/item relative gap-1 px-1",
isExpanded && "font-medium",
dropdownOpen && "bg-accent text-accent-foreground",
isOver && canDrop && dropZone === "middle" && "bg-accent ring-1 ring-primary/40",
isOver && canDrop && dropZone === "top" && "border-t-2 border-primary",
isOver && canDrop && dropZone === "bottom" && "border-b-2 border-primary",
@ -291,11 +294,11 @@ export const FolderNode = React.memo(function FolderNode({
)}
</span>
{processingState !== "idle" && selectionState === "none" ? (
<>
<span className="relative flex h-4 w-4 shrink-0 items-center justify-center">
{processingState !== "idle" && selectionState === "none" ? (
<Tooltip>
<TooltipTrigger asChild>
<span className="flex h-3.5 w-3.5 shrink-0 items-center justify-center group-hover:hidden">
<span className="flex h-3.5 w-3.5 items-center justify-center">
{processingState === "processing" ? (
<Spinner size="xs" className="text-primary" />
) : (
@ -309,29 +312,37 @@ export const FolderNode = React.memo(function FolderNode({
: "Some files failed to process"}
</TooltipContent>
</Tooltip>
<Checkbox
checked={false}
onCheckedChange={handleCheckChange}
onClick={(e) => e.stopPropagation()}
className="h-3.5 w-3.5 shrink-0 hidden group-hover:flex"
/>
</>
) : (
<Checkbox
checked={
selectionState === "all"
? true
: selectionState === "some"
? "indeterminate"
: false
}
onCheckedChange={handleCheckChange}
onClick={(e) => e.stopPropagation()}
className="h-3.5 w-3.5 shrink-0"
/>
)}
<FolderIcon className="h-4 w-4 shrink-0 text-muted-foreground" />
) : (
<>
<FolderIcon
className={cn(
"absolute h-4 w-4 text-muted-foreground transition-opacity",
selectionState === "none"
? "max-sm:opacity-0 group-hover/item:opacity-0"
: "opacity-0"
)}
/>
<Checkbox
checked={
selectionState === "all"
? true
: selectionState === "some"
? "indeterminate"
: false
}
aria-label={`Select ${folder.name}`}
onCheckedChange={handleCheckChange}
onClick={(e) => e.stopPropagation()}
className={cn(
"absolute h-3.5 w-3.5 transition-opacity max-sm:pointer-events-auto max-sm:opacity-100",
selectionState === "none"
? "pointer-events-none opacity-0 group-hover/item:pointer-events-auto group-hover/item:opacity-100 focus-visible:pointer-events-auto focus-visible:opacity-100"
: "opacity-100"
)}
/>
</>
)}
</span>
{isRenaming ? (
<input
@ -350,94 +361,107 @@ export const FolderNode = React.memo(function FolderNode({
)}
{!isRenaming && (
<DropdownMenu open={dropdownOpen} onOpenChange={setDropdownOpen}>
<DropdownMenuTrigger asChild>
<Button
variant="ghost"
size="icon"
className={cn(
"hidden sm:inline-flex h-6 w-6 shrink-0 hover:bg-transparent transition-opacity",
dropdownOpen
? "opacity-100 bg-accent hover:bg-accent"
: "opacity-0 group-hover:opacity-100"
<div
className={cn(
"pointer-events-none absolute top-0 right-0 bottom-0 flex items-center rounded-r-md pr-1 pl-6",
dropdownOpen
? "bg-gradient-to-l from-accent from-60% to-transparent"
: "bg-gradient-to-l from-sidebar from-60% to-transparent group-hover/item:from-accent",
isMobile
? "opacity-0"
: dropdownOpen
? "opacity-100"
: "opacity-0 group-hover/item:opacity-100"
)}
>
<DropdownMenu open={dropdownOpen} onOpenChange={setDropdownOpen}>
<DropdownMenuTrigger asChild>
<Button
variant="ghost"
size="icon"
className={cn(
"pointer-events-auto hidden h-6 w-6 shrink-0 hover:bg-transparent transition-opacity sm:inline-flex",
dropdownOpen && "bg-accent hover:bg-accent"
)}
onClick={(e) => e.stopPropagation()}
aria-label={`Folder actions for ${folder.name}`}
>
<MoreHorizontal className="h-3.5 w-3.5 text-muted-foreground" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-40">
{isWatched && onRescan && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
handleRescan();
}}
>
<RefreshCw className={cn("mr-2 h-4 w-4", isRescanning && "animate-spin")} />
Re-scan
</DropdownMenuItem>
)}
{isWatched && onStopWatching && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onStopWatching(folder);
}}
>
<EyeOff className="mr-2 h-4 w-4" />
Stop watching
</DropdownMenuItem>
)}
onClick={(e) => e.stopPropagation()}
>
<MoreHorizontal className="h-3.5 w-3.5 text-muted-foreground" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-40">
{isWatched && onRescan && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
handleRescan();
onCreateSubfolder(folder.id);
}}
>
<RefreshCw className={cn("mr-2 h-4 w-4", isRescanning && "animate-spin")} />
Re-scan
<FolderPlus className="mr-2 h-4 w-4" />
New subfolder
</DropdownMenuItem>
)}
{isWatched && onStopWatching && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onStopWatching(folder);
startRename();
}}
>
<EyeOff className="mr-2 h-4 w-4" />
Stop watching
<Pencil className="mr-2 h-4 w-4" />
Rename
</DropdownMenuItem>
)}
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onCreateSubfolder(folder.id);
}}
>
<FolderPlus className="mr-2 h-4 w-4" />
New subfolder
</DropdownMenuItem>
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
startRename();
}}
>
<Pencil className="mr-2 h-4 w-4" />
Rename
</DropdownMenuItem>
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onMove(folder);
}}
>
<Move className="mr-2 h-4 w-4" />
Move to...
</DropdownMenuItem>
{onExportFolder && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onExportFolder(folder);
onMove(folder);
}}
>
<Download className="mr-2 h-4 w-4" />
Export folder
<Move className="mr-2 h-4 w-4" />
Move to...
</DropdownMenuItem>
)}
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onDelete(folder);
}}
>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
{onExportFolder && (
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onExportFolder(folder);
}}
>
<Download className="mr-2 h-4 w-4" />
Export folder
</DropdownMenuItem>
)}
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
onDelete(folder);
}}
>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
)}
</SidebarListItem>
</ContextMenuTrigger>

98
surfsense_web/components/homepage/compare-table.tsx
View file

@ -1,36 +1,61 @@
import { Reveal } from "@/components/connectors-marketing/reveal";
import { MarketingSection } from "@/components/marketing/section";
const COLUMNS = [
{ label: "Browser agents", examples: "Browserbase, Browser Use" },
{ label: "Scraping APIs", examples: "Firecrawl" },
{ label: "Search APIs", examples: "Exa, Tavily, Parallel" },
{ label: "Scraper marketplaces", examples: "Apify" },
];
const ROWS = [
{
feature: "Who does the work",
suites: "Analysts read human dashboards",
scrapers: "You build everything on raw data",
surfsense: "Your AI agents gather, analyze, and alert",
feature: "Built for",
browser: "Web tasks that need clicking, logins, and forms",
scraping: "Turning individual pages into LLM-ready content",
search: "Finding and reading pages about a topic",
marketplace: "Thousands of community-built scrapers, one per site",
surfsense: "Live platform data as research primitives for agents",
},
{
feature: "Intelligence layer",
suites: "Curated but slow, human-in-the-loop",
scrapers: "None; data only",
surfsense: "Agent harness turns live data into briefs",
feature: "How retrieval works",
browser: "An LLM drives a real browser, page by page",
scraping: "Fetch a URL, get markdown or schema-extracted JSON",
search: "Query an index, get ranked results and page content",
marketplace: "Pick an actor per site, learn its input, run it",
surfsense: "One typed REST call per platform, no LLM in the retrieval loop",
},
{
feature: "Platform data (comment trees, transcripts, reviews)",
browser: "Whatever the LLM extracts from rendered pages",
scraping: "Page-level extraction; social platforms aren't the focus",
search: "Page text and snippets, not structured platform items",
marketplace: "Yes, but schema and quality vary per actor",
surfsense: "Native items: posts, comment trees, transcripts, reviews, SERPs",
},
{
feature: "Consistency",
browser: "Depends on the model and the page",
scraping: "One API, you define schemas per page type",
search: "One API, snippet and page-content shapes",
marketplace: "A different schema and quality bar per actor",
surfsense: "One API and one schema style across every connector",
},
{
feature: "Research workspace & knowledge base",
browser: "No",
scraping: "No",
search: "No",
marketplace: "No",
surfsense: "Cited briefs, knowledge base, scheduled automations, deliverables",
},
{
feature: "Pricing",
suites: "Enterprise contracts, annual quotes",
scrapers: "Usage-priced infrastructure",
surfsense: "Self-serve, pay per item, free tier",
},
{
feature: "Developer surface",
suites: "No agent surface",
scrapers: "APIs, but no MCP or harness",
surfsense: "REST API with your API key, plus an MCP server for agents",
},
{
feature: "Open source",
suites: "No",
scrapers: "No",
surfsense: "Yes, self-hostable",
browser: "Per browser minute (1-minute session minimum) plus the LLM tokens driving it",
scraping: "Credits per page; schema extraction costs extra credits",
search: "Per search request",
marketplace: "Per event or result, set by each actor",
surfsense: "Per item returned; failed calls never billed",
},
];
@ -40,24 +65,23 @@ export function CompareTable() {
<Reveal>
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">How SurfSense compares</h2>
<p className="mt-3 max-w-2xl text-muted-foreground leading-relaxed">
CI suites sell dashboards to analysts. Scraper APIs sell raw data to engineers. SurfSense
gives AI agents the data and the harness in one open-source platform.
SurfSense is the only open-source product that combines a NotebookLM-style research
workspace for people with live-data primitives for agents. Here is how that stacks up
against each class of tool.
</p>
</Reveal>
<Reveal>
<div className="mt-8 overflow-x-auto rounded-xl border bg-card">
<table className="w-full min-w-2xl text-sm">
<table className="w-full min-w-4xl text-sm">
<thead>
<tr className="border-b bg-muted/40 text-left">
<th className="p-4 font-medium">Feature</th>
<th className="p-4 font-medium text-muted-foreground">
CI suites
<span className="block text-xs font-normal">Crayon, Klue</span>
</th>
<th className="p-4 font-medium text-muted-foreground">
Scraper APIs
<span className="block text-xs font-normal">Bright Data, Apify</span>
</th>
{COLUMNS.map((col) => (
<th key={col.label} className="p-4 font-medium text-muted-foreground">
{col.label}
<span className="block text-xs font-normal">{col.examples}</span>
</th>
))}
<th className="p-4 font-medium text-brand">SurfSense</th>
</tr>
</thead>
@ -67,8 +91,10 @@ export function CompareTable() {
<th scope="row" className="p-4 text-left font-medium">
{row.feature}
</th>
<td className="p-4 text-muted-foreground">{row.suites}</td>
<td className="p-4 text-muted-foreground">{row.scrapers}</td>
<td className="p-4 text-muted-foreground">{row.browser}</td>
<td className="p-4 text-muted-foreground">{row.scraping}</td>
<td className="p-4 text-muted-foreground">{row.search}</td>
<td className="p-4 text-muted-foreground">{row.marketplace}</td>
<td className="bg-brand/5 p-4 text-foreground">{row.surfsense}</td>
</tr>
))}

2
surfsense_web/components/homepage/connector-grid.tsx
View file

@ -15,7 +15,7 @@ export function ConnectorGrid() {
<Reveal>
<div className="flex flex-wrap items-center gap-3">
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
Connectors for every platform your market uses
Connectors for every platform where answers live
</h2>
<Badge variant="outline" className="py-1">
{connectors.length} connectors and growing

2
surfsense_web/components/homepage/footer-new.tsx
View file

@ -106,7 +106,7 @@ export function FooterNew() {
];
return (
<div className="border-t border-neutral-100 dark:border-white/[0.1] px-8 py-20 bg-white dark:bg-neutral-950 w-full relative overflow-hidden">
<div className="max-w-7xl mx-auto text-sm text-neutral-500 flex sm:flex-row flex-col justify-between items-start md:px-8">
<div className="max-w-7xl mx-auto text-sm text-neutral-600 dark:text-neutral-400 flex sm:flex-row flex-col justify-between items-start md:px-8">
<div>
<div className="mr-0 md:mr-4 md:flex mb-4">
<Logo className="h-6 w-6 rounded-md mr-2" />

3
surfsense_web/components/homepage/hero-chat-demo.tsx
View file

@ -28,7 +28,7 @@ export type HeroChatDemoScript = {
type Stage = "typing" | "steps" | "answer" | "done";
const PLACEHOLDER =
"Track competitors, scrape platforms, automate briefs. Use / for prompts, @ for docs";
"Research the live web, scrape platforms, automate briefs. Use / for prompts, @ for docs";
/** Blinking caret for the typewriter (overlay only, never inside the real input). */
function Caret() {
@ -77,6 +77,7 @@ function DemoTimeline({
<Button
variant="ghost"
type="button"
tabIndex={-1}
onClick={() => setIsOpen((prev) => !prev)}
className="h-auto w-full justify-start gap-1.5 p-0 text-left text-sm font-normal text-muted-foreground transition-colors hover:bg-transparent hover:text-accent-foreground"
>

709
surfsense_web/components/homepage/hero-section.tsx
View file

@ -1,15 +1,27 @@
"use client";
import { ChevronDown, Download } from "lucide-react";
import { AnimatePresence, motion, useReducedMotion } from "motion/react";
import Link from "next/link";
import React, { memo, useCallback, useEffect, useRef, useState } from "react";
import Balancer from "react-wrap-balancer";
import { HeroChatDemo, type HeroChatDemoScript } from "@/components/homepage/hero-chat-demo";
import { Button } from "@/components/ui/button";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { ExpandedMediaOverlay, useExpandedMedia } from "@/components/ui/expanded-gif-overlay";
import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area";
import { Separator } from "@/components/ui/separator";
import { Skeleton } from "@/components/ui/skeleton";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import {
GITHUB_RELEASES_URL,
getAssetLabel,
usePrimaryDownload,
} from "@/lib/desktop-download-utils";
import { buildBackendUrl } from "@/lib/env-config";
import { trackLoginAttempt } from "@/lib/posthog/events";
import { cn } from "@/lib/utils";
@ -58,7 +70,6 @@ type HeroCategory = {
};
const HERO_TUTORIAL = "/homepage/hero_tutorial";
const HERO_REALTIME = "/homepage/hero_realtime";
/*
* Every scripted demo below mirrors a task the SurfSense agent has actually run
@ -67,8 +78,152 @@ const HERO_REALTIME = "/homepage/hero_realtime";
*/
const CATEGORIES: HeroCategory[] = [
{
id: "connector-workflows",
label: "Multi-Connector Workflows",
id: "live-research",
label: "Live Web Research",
useCases: [
{
id: "deep-research",
title: "Deep Research on the Live Web",
description:
"The agent crawls dozens of live sources on a question and synthesizes a cited answer, not a stale index.",
src: null,
demo: {
prompt: "Research the AI note-taking market and build a landscape brief with citations.",
steps: [
{
title: "Research",
items: ["Crawling 38 live sources", "Vendor sites, reviews, pricing pages"],
},
{
title: "Generate report",
items: ["Landscape brief · 24 inline citations"],
},
],
rows: [
{
primary: "23 vendors mapped across 4 segments",
secondary: "consumer, prosumer, team, developer-first",
},
{
primary: "Pricing clusters at $10 and $20/mo",
secondary: "3 vendors moved upmarket this quarter",
},
],
summary: "Landscape brief saved · 24 inline citations you can check",
},
},
{
id: "academic-research",
title: "Academic Literature Scan",
description:
"Sweep recent papers, preprints, and technical blogs on a topic and get the main approaches mapped with citations.",
src: null,
demo: {
prompt:
"Survey the last year of research on LLM hallucination detection and map the main approaches.",
steps: [
{
title: "Google Search",
items: ["12 SERPs · arXiv, ACL, technical blogs"],
},
{
title: "Web Crawler",
items: ["Reading 21 papers and posts", "Extracting methods and benchmarks"],
},
{
title: "Generate report",
items: ["Literature brief · grouped by approach"],
},
],
rows: [
{
primary: "4 approach families mapped",
secondary: "self-consistency, retrieval grounding, probes, uncertainty",
},
{
primary: "Benchmarks converge on 2 suites",
secondary: "used in 9 of the 21 papers reviewed",
},
],
summary: "Literature brief saved · every claim links to its paper",
},
},
{
id: "financial-research",
title: "Financial & Market Research",
description:
"Pull earnings coverage, analyst breakdowns, and retail sentiment on any company into one cited brief.",
src: null,
demo: {
prompt:
"Summarize the reaction to NVIDIA's latest earnings across news, YouTube, and Reddit.",
steps: [
{
title: "Google Search",
items: ["10 SERPs · earnings coverage and recaps"],
},
{
title: "Youtube",
items: ["8 analyst breakdowns · transcripts pulled"],
},
{
title: "Reddit",
items: ["r/investing + r/stocks · 31 threads"],
},
],
rows: [
{
primary: "Data-center revenue beat leads coverage",
secondary: "cited in 7 of 10 top results",
},
{
primary: "Analyst take split on guidance",
secondary: "transcripts: 5 bullish · 3 cautious",
},
{
primary: "Retail sentiment net positive",
secondary: "top threads focus on supply constraints",
},
],
summary: "Cited earnings brief saved to your workspace",
},
},
{
id: "geo-monitoring",
title: "AI Overview & GEO Tracking",
description:
"Capture when Google's AI Overviews answer the queries you care about, and exactly which sources they cite.",
src: null,
demo: {
prompt: "Which of our target keywords trigger an AI Overview, and who gets cited?",
steps: [
{
title: "Google Search",
items: ["Scraping 25 SERPs", "Capturing AI Overviews and citations"],
},
{
title: "Plan tasks",
items: ["Map citations to competitors", "Compute your citation gap"],
},
],
rows: [
{
primary: "9 of 25 keywords trigger an AI Overview",
secondary: "up from 6 last month",
},
{
primary: "Competitor A cited on 4 · you on 1",
secondary: "their listicle wins 3 of those citations",
},
],
summary: "Citation gap report saved · weekly re-check scheduled",
},
},
],
},
{
id: "ci-workflows",
label: "Competitive Intelligence Workflows",
useCases: [
{
id: "launch-impact",
@ -154,60 +309,6 @@ const CATEGORIES: HeroCategory[] = [
summary: "Maps + Crawler + Search in one run · teardown saved",
},
},
{
id: "competitor-360",
title: "Competitor 360, on a Schedule",
description:
"An automation chains four connectors every week: site changes, rank movements, Reddit sentiment, and YouTube reaction.",
src: null,
demo: {
prompt:
"Every Monday, build me a 360 on our top competitor: site changes, rankings, Reddit, and YouTube.",
steps: [
{
title: "Web Crawler",
items: ["pricing + changelog pages · 2 changes detected"],
},
{
title: "Google Search",
items: ["12 shared keywords · rank movements captured"],
},
{
title: "Reddit",
items: ["18 mentions this week · sentiment tagged"],
},
{
title: "Youtube",
items: ["2 new videos · comments and transcripts pulled"],
},
{
title: "Create automation",
items: ["Weekly 360 brief · Mondays 8:00"],
},
],
rows: [
{
primary: "Shipped: usage-based pricing page",
secondary: "pricing + changelog diff · detected Jul 3",
},
{
primary: 'Took #2 on "competitive intelligence api"',
secondary: "you hold #4 · gap widened two weeks in a row",
},
{
primary: "Reddit sentiment down 12 pts since the change",
secondary: "churn signals in 5 threads · quotes linked",
},
],
summary: "4 connectors, 1 automation · first brief lands Monday 8:00",
},
},
],
},
{
id: "competitor-monitoring",
label: "Competitor Monitoring",
useCases: [
{
id: "pricing-watch",
title: "Competitor Pricing Watch",
@ -290,7 +391,7 @@ const CATEGORIES: HeroCategory[] = [
id: "serp-watch",
title: "Rank & Ad Monitoring",
description:
"Automations track the Google rankings, paid ads, and AI Overview citations your market actually sees.",
"Automations track the Google rankings, paid ads, and AI Overview citations your audience actually sees.",
src: null,
demo: {
prompt: "Track who ranks and runs ads for our top 10 keywords in the US.",
@ -310,7 +411,7 @@ const CATEGORIES: HeroCategory[] = [
],
rows: [
{
primary: '"competitive intelligence tools" — you #4, ↓1',
primary: '"ai research tools" — you #4, ↓1',
secondary: "Competitor A took #3 · runs 2 sponsored ads",
},
{
@ -321,202 +422,6 @@ const CATEGORIES: HeroCategory[] = [
summary: "10 SERPs captured · 3 movements flagged · daily automation on",
},
},
],
},
{
id: "lead-generation",
label: "B2B Lead Generation",
useCases: [
{
id: "local-leads",
title: "Local Business Leads",
description:
"Turn a category and a territory into a lead list with phones, websites, ratings, and decision-maker contacts.",
src: null,
demo: {
prompt:
"Find the top 5 burger places in San Jose and pull staff contacts from their websites.",
steps: [
{
title: "Google Maps",
items: ['Searching "burger san jose" · top 5 by rating and reviews'],
},
{
title: "Web Crawler",
items: ["Visiting 5 business sites", "Extracting staff and contact pages"],
},
{
title: "Save document",
items: ["Lead list with phones, sites, contacts"],
},
],
rows: [
{
primary: "The Counter — 4.5★ (2,310 reviews)",
secondary: "+1 408-423-9200 · thecounter.com · 2 contacts found",
},
{
primary: "Paper Plane — 4.6★ (1,882 reviews)",
secondary: "site crawled · owner + events email found",
},
{
primary: "Smoking Pig BBQ — 4.4★ (3,041 reviews)",
secondary: "+1 408-380-4784 · catering contact found",
},
],
summary: "5 places · 9 contacts with provenance · saved as lead list",
},
},
{
id: "team-rosters",
title: "Team Rosters & Contacts",
description:
"Spider any company site and pull the full team with emails, socials, and provenance, exported to CSV.",
src: null,
demo: {
prompt: "Get the complete a16z team roster and save it as a CSV in my workspace.",
steps: [
{
title: "Web Crawler",
items: ["a16z.com/team · following profile links", "142 profiles found"],
},
{
title: "Plan tasks",
items: ["Enrich each profile with email or LinkedIn"],
},
{
title: "Write file",
items: ["a16z-team.csv · saved to workspace"],
},
],
rows: [
{
primary: "142 team profiles extracted",
secondary: "name, role, focus area, profile URL",
},
{
primary: "89 enriched with email or LinkedIn",
secondary: "pulled from bios and linked pages",
},
{ primary: "a16z-team.csv", secondary: "saved to your workspace · ready to export" },
],
summary: "Full roster in 94s · every row cites its source page",
},
},
{
id: "portfolio-mapping",
title: "Portfolio & Market Mapping",
description:
"Map an investor's portfolio or a whole category, then enrich every company with pricing and contacts.",
src: null,
demo: {
prompt: "Get the a16z team and their portfolio companies.",
steps: [
{
title: "Web Crawler",
items: ["a16z.com/team · 142 profiles", "a16z.com/portfolio · 312 companies"],
},
{
title: "Plan tasks",
items: [
"Match partners to their portfolio boards",
"Enrich your category with pricing",
],
},
],
rows: [
{
primary: "312 portfolio companies mapped",
secondary: "sector, stage, website, one-line pitch",
},
{
primary: "48 in your category",
secondary: "enriched with pricing page + team size",
},
],
summary: "Market map saved · new investments auto-added by automation",
},
},
],
},
{
id: "brand-listening",
label: "Brand & Market Listening",
useCases: [
{
id: "reddit-listening",
title: "Reddit Brand Monitoring",
description:
"Hear what your market says about you, your competitors, and your category in the threads where buyers speak candidly.",
src: null,
demo: {
prompt: "Give me 20 posts on Reddit where people ask for an alternative to NotebookLM.",
steps: [
{
title: "Reddit",
items: [
'Searching "notebooklm alternative" across Reddit',
"37 posts found · ranking by relevance and upvotes",
],
},
{
title: "Plan tasks",
items: ["Tag each post: buying intent, churn signal, question"],
},
],
rows: [
{
primary: "Best NotebookLM alternative that's open source?",
secondary: "r/selfhosted · 214 upvotes · buying intent",
},
{
primary: "NotebookLM keeps losing my sources, what else?",
secondary: "r/artificial · 96 upvotes · churn signal",
},
{
primary: "Anything like NotebookLM but with an API?",
secondary: "r/LocalLLaMA · 71 upvotes · developer intent",
},
],
summary: "20 posts · 8 with buying intent · daily tracking automation on",
},
},
{
id: "youtube-sentiment",
title: "YouTube Audience Sentiment",
description:
"Pull videos, transcripts, and comments at scale to mine what audiences praise and complain about.",
src: null,
demo: {
prompt:
"Analyze the comments on our competitor's last 10 videos and cluster the complaints.",
steps: [
{
title: "Youtube",
items: ["Fetching last 10 videos", "4,812 comments pulled"],
},
{
title: "Plan tasks",
items: ["Cluster complaints by theme", "Score sentiment per cluster"],
},
],
rows: [
{
primary: "Pricing complaints — 31% of negative comments",
secondary: '"went from free to $20/mo overnight"',
},
{
primary: "Export limits — 22%",
secondary: '"can\'t get my own data out"',
},
{
primary: "Praise: onboarding — 18% positive",
secondary: "worth copying in your messaging",
},
],
summary: "4,812 comments clustered · sentiment report saved",
},
},
{
id: "switcher-mining",
title: "Switcher & Intent Mining",
@ -554,79 +459,95 @@ const CATEGORIES: HeroCategory[] = [
],
},
{
id: "market-research",
label: "Market Research",
id: "artifacts",
label: "Artifacts (Podcasts, Videos & More)",
useCases: [
{
id: "deep-research",
title: "Deep Research on the Live Web",
description:
"The agent crawls dozens of live sources on a question and synthesizes a cited answer, not a stale index.",
src: null,
demo: {
prompt: "Research the AI note-taking market and build a landscape brief with citations.",
steps: [
{
title: "Research",
items: ["Crawling 38 live sources", "Vendor sites, reviews, pricing pages"],
},
{
title: "Generate report",
items: ["Landscape brief · 24 inline citations"],
},
],
rows: [
{
primary: "23 vendors mapped across 4 segments",
secondary: "consumer, prosumer, team, developer-first",
},
{
primary: "Pricing clusters at $10 and $20/mo",
secondary: "3 vendors moved upmarket this quarter",
},
],
summary: "Landscape brief saved · 24 inline citations you can check",
},
id: "report",
title: "AI Report Generator",
description: "Turn your research into cited reports, then export to PDF or Markdown.",
src: `${HERO_TUTORIAL}/ReportGenGif_compressed.mp4`,
},
{
id: "geo-monitoring",
title: "AI Overview & GEO Tracking",
id: "podcast",
title: "AI Podcast Generator",
description: "Turn any brief or folder into a two-host AI podcast in under 20 seconds.",
src: `${HERO_TUTORIAL}/PodcastGenGif.mp4`,
},
{
id: "presentation",
title: "AI Presentation & Video Maker",
description: "Create editable slide decks and narrated video overviews from your findings.",
src: `${HERO_TUTORIAL}/video_gen_surf.mp4`,
},
{
id: "image-gen",
title: "AI Image Generation",
description: "Generate images inside your workspace for decks, briefs, and posts.",
src: `${HERO_TUTORIAL}/ImageGenGif.mp4`,
},
],
},
{
id: "automations",
label: "Automations",
useCases: [
{
id: "competitor-360",
title: "Competitor 360, on a Schedule",
description:
"Capture when Google's AI Overviews answer your market's queries, and exactly which sources they cite.",
"An automation chains four connectors every week: site changes, rank movements, Reddit sentiment, and YouTube reaction.",
src: null,
demo: {
prompt: "Which of our target keywords trigger an AI Overview, and who gets cited?",
prompt:
"Every Monday, build me a 360 on our top competitor: site changes, rankings, Reddit, and YouTube.",
steps: [
{
title: "Google Search",
items: ["Scraping 25 SERPs", "Capturing AI Overviews and citations"],
title: "Web Crawler",
items: ["pricing + changelog pages · 2 changes detected"],
},
{
title: "Plan tasks",
items: ["Map citations to competitors", "Compute your citation gap"],
title: "Google Search",
items: ["12 shared keywords · rank movements captured"],
},
{
title: "Reddit",
items: ["18 mentions this week · sentiment tagged"],
},
{
title: "Youtube",
items: ["2 new videos · comments and transcripts pulled"],
},
{
title: "Create automation",
items: ["Weekly 360 brief · Mondays 8:00"],
},
],
rows: [
{
primary: "9 of 25 keywords trigger an AI Overview",
secondary: "up from 6 last month",
primary: "Shipped: usage-based pricing page",
secondary: "pricing + changelog diff · detected Jul 3",
},
{
primary: "Competitor A cited on 4 · you on 1",
secondary: "their listicle wins 3 of those citations",
primary: 'Took #2 on "reddit scraper api"',
secondary: "you hold #4 · gap widened two weeks in a row",
},
{
primary: "Reddit sentiment down 12 pts since the change",
secondary: "churn signals in 5 threads · quotes linked",
},
],
summary: "Citation gap report saved · weekly re-check scheduled",
summary: "4 connectors, 1 automation · first brief lands Monday 8:00",
},
},
{
id: "cited-briefs",
title: "Cited Briefs & Alerts",
title: "Scheduled Briefs & Alerts",
description:
"Everything the agents gather lands in your workspace as briefs and alerts with sources you can check.",
src: null,
demo: {
prompt: "Send me a Monday brief of every competitor change detected last week.",
prompt: "Send me a Monday brief of every change my agents detected last week.",
steps: [
{
title: "Plan tasks",
@ -650,50 +571,68 @@ const CATEGORIES: HeroCategory[] = [
summary: "Automation created · first brief lands Monday 8:00",
},
},
{
id: "event-triggers",
title: "Event-Triggered Workflows",
description:
"Automations can fire on events, not just schedules: a document landing in a folder kicks off the workflow.",
src: null,
demo: {
prompt:
"Whenever a new file lands in my Research folder, summarize it and post the summary to Slack.",
steps: [
{
title: "Create automation",
items: [
"Trigger: new document in Research folder",
"Action: summarize → post to #research",
],
},
],
rows: [
{
primary: "Automation armed on the Research folder",
secondary: "fires the moment a document lands",
},
{
primary: "First run: competitor-teardown.pdf",
secondary: "summary posted to #research · 42s after upload",
},
],
summary: "Event-triggered automation live · no schedule needed",
},
},
],
},
{
id: "platform-agent",
label: "SurfSense Agent",
id: "desktop-app",
label: "Desktop App",
useCases: [
{
id: "chat-workspace",
title: "Chat With Everything You Gather",
id: "general-assist",
title: "General Assist",
description:
"Ask questions across every crawl, mention, and document in your workspace and get answers with inline citations.",
src: `${HERO_TUTORIAL}/BQnaGif_compressed.mp4`,
"Launch SurfSense from any application on your computer with a global shortcut.",
src: `${HERO_TUTORIAL}/general_assist.mp4`,
},
{
id: "report",
title: "AI Report Generator",
id: "quick-assist",
title: "Quick Assist",
description: "Select text anywhere, then ask AI to explain, rewrite, or act on it.",
src: `${HERO_TUTORIAL}/quick_assist.mp4`,
},
{
id: "screenshot-assist",
title: "Screenshot Assist",
description: "Capture any region of your screen and ask AI about it.",
src: `${HERO_TUTORIAL}/screenshot_assist.mp4`,
},
{
id: "folder-watch",
title: "Watch Local Folder",
description:
"Turn your intelligence into cited research reports, then export to PDF or Markdown.",
src: `${HERO_TUTORIAL}/ReportGenGif_compressed.mp4`,
},
{
id: "podcast",
title: "AI Podcast Generator",
description: "Turn any brief or folder into a two-host AI podcast in under 20 seconds.",
src: `${HERO_TUTORIAL}/PodcastGenGif.mp4`,
},
{
id: "presentation",
title: "AI Presentation & Video Maker",
description: "Create editable slide decks and narrated video overviews from your findings.",
src: `${HERO_TUTORIAL}/video_gen_surf.mp4`,
},
{
id: "connect",
title: "Build Your Knowledge Base",
description:
"Upload files or sync Google Drive, OneDrive, and Dropbox into one searchable knowledge base alongside everything your agents gather.",
src: `${HERO_TUTORIAL}/ConnectorFlowGif.mp4`,
},
{
id: "collab",
title: "Collaborative AI Chat",
description: "Work on AI conversations with your team in real time.",
src: `${HERO_REALTIME}/RealTimeChatGif.mp4`,
"Auto-sync a local folder to your knowledge base. Point it at your Obsidian vault to keep your notes searchable.",
src: `${HERO_TUTORIAL}/folder_watch.mp4`,
},
],
},
@ -708,7 +647,7 @@ export function HeroSection() {
"relative mt-4 max-w-4xl text-left text-4xl font-bold tracking-tight text-balance text-neutral-900 sm:text-5xl md:text-6xl dark:text-neutral-50"
)}
>
<Balancer>NotebookLM for competitive intelligence research.</Balancer>
<Balancer>The open-source NotebookLM alternative for open web research.</Balancer>
</h1>
<div className="mt-4 flex w-full flex-col items-start justify-between gap-4 md:mt-8 md:flex-row md:items-end md:gap-10">
<div>
@ -717,14 +656,15 @@ export function HeroSection() {
"relative mb-8 max-w-2xl text-left text-sm text-neutral-600 antialiased sm:text-base md:text-lg dark:text-neutral-400"
)}
>
SurfSense is an open-source competitive intelligence platform, like NotebookLM but
with live scraping connectors. Your AI agents monitor competitors, track rankings, and
listen to your market with live data from platforms like Reddit, YouTube, Instagram,
TikTok, Google Maps, Google Search, and the open web.
SurfSense is the open-source NotebookLM alternative for AI agents, an open web
research platform with live data connectors. Your AI agents research the live web with
structured data from Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google
Search, and any page on the open web.
</p>
<div className="relative mb-4 flex w-full flex-col justify-center gap-y-2 sm:flex-row sm:justify-start sm:space-y-0 sm:space-x-4">
<GetStartedButton />
<DownloadButton />
</div>
</div>
</div>
@ -767,6 +707,89 @@ function GetStartedButton() {
);
}
function DownloadButton() {
const { os, primary, alternatives, isMobileOS } = usePrimaryDownload();
const fallbackUrl = GITHUB_RELEASES_URL;
const mobileDisabledLabel = "Desktop app unavailable on mobile";
if (isMobileOS) {
return (
<Button
type="button"
variant="ghost"
disabled
className="h-14 w-full gap-2 rounded-lg border border-neutral-200 bg-white text-center text-base font-medium text-neutral-700 shadow-sm transition duration-150 sm:w-auto sm:px-6 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-200"
>
<Download className="size-4" />
{mobileDisabledLabel}
</Button>
);
}
if (!primary) {
return (
<Button
asChild
variant="ghost"
className="h-14 w-full gap-2 rounded-lg border border-neutral-200 bg-white text-center text-base font-medium text-neutral-700 shadow-sm transition duration-150 active:scale-98 hover:bg-neutral-50 sm:w-auto sm:px-6 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-200 dark:hover:bg-neutral-800"
>
<a href={fallbackUrl} target="_blank" rel="noopener noreferrer">
<Download className="size-4" />
Download for {os}
</a>
</Button>
);
}
return (
<div className="flex h-14 w-full items-stretch sm:w-auto">
<Button
asChild
variant="ghost"
className="h-auto flex-1 gap-2 rounded-l-lg rounded-r-none border border-r-0 border-neutral-200 bg-white px-5 text-base font-medium text-neutral-700 shadow-sm transition duration-150 active:scale-[0.99] hover:bg-neutral-50 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-200 dark:hover:bg-neutral-800"
>
<a href={primary.url}>
<Download className="size-4 shrink-0" />
Download for {os}
</a>
</Button>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button
type="button"
variant="ghost"
aria-label="More download options"
className="h-auto rounded-l-none rounded-r-lg border border-neutral-200 bg-white px-2.5 text-neutral-500 shadow-sm transition duration-150 hover:bg-neutral-50 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-400 dark:hover:bg-neutral-800"
>
<ChevronDown className="size-4" aria-hidden />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-64">
{alternatives.map((asset) => (
<DropdownMenuItem key={asset.name} asChild>
<a href={asset.url} className="cursor-pointer">
<Download className="mr-2 size-3.5" />
{getAssetLabel(asset.name)}
</a>
</DropdownMenuItem>
))}
<DropdownMenuItem asChild>
<a
href={fallbackUrl}
target="_blank"
rel="noopener noreferrer"
className="cursor-pointer"
>
All downloads
</a>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
);
}
const TabVideo = memo(function TabVideo({
src,
title,
@ -847,9 +870,9 @@ const UseCasePane = memo(function UseCasePane({
<div className="relative overflow-hidden rounded-tl-xl rounded-tr-xl bg-white shadow-sm ring-1 shadow-black/10 ring-black/10 dark:bg-neutral-950">
<div className="flex items-center gap-3 border-b border-neutral-200/60 px-4 py-3 sm:px-6 sm:py-4 dark:border-neutral-700/60">
<div className="min-w-0">
<h3 className="truncate text-base font-semibold text-neutral-900 sm:text-lg dark:text-white">
<h2 className="truncate text-base font-semibold text-neutral-900 sm:text-lg dark:text-white">
{useCase.title}
</h3>
</h2>
<p className="text-sm text-neutral-500 text-pretty dark:text-neutral-400">
{useCase.description}
</p>

6
surfsense_web/components/homepage/home-faq.tsx
View file

@ -6,9 +6,9 @@ import { FAQJsonLd } from "@/components/seo/json-ld";
/** Answers are 40-60 words, written as quotable definitions for AI Overviews. */
export const HOME_FAQ = [
{
question: "What is competitive intelligence?",
question: "What is open web research?",
answer:
"Competitive intelligence is the practice of gathering and analyzing public information about competitors and your market to make better decisions. It covers pricing, product moves, rankings, reviews, and what customers say online. SurfSense automates it: AI agents collect the live data and turn it into briefs and alerts.",
"Open web research is gathering and analyzing live public information from across the web: search results, community discussions, reviews, videos, and any page. Unlike asking a chatbot that reasons over a stale index, it works from what the web says right now. SurfSense automates it: AI agents collect the live data and turn it into cited briefs and alerts.",
},
{
question: "What is an MCP server?",
@ -28,7 +28,7 @@ export const HOME_FAQ = [
{
question: "Can I self-host SurfSense?",
answer:
"Yes. SurfSense is open source and self-hostable, so you can run the entire platform on your own infrastructure and keep sensitive competitive research in-house. Use the cloud version to start in minutes, or deploy from the GitHub repository when you need full control.",
"Yes. SurfSense is open source and self-hostable, so you can run the entire platform on your own infrastructure and keep sensitive research in-house. Use the cloud version to start in minutes, or deploy from the GitHub repository when you need full control.",
},
];

111
surfsense_web/components/homepage/logo-cloud.tsx Normal file
View file

@ -0,0 +1,111 @@
"use client";
import { AnimatePresence, motion } from "motion/react";
import { useEffect, useState } from "react";
import { Reveal } from "@/components/connectors-marketing/reveal";
import { MarketingSection } from "@/components/marketing/section";
/**
* Real signups pulled from prod (Google-auth), curated to the most recognizable
* names. These are self-serve users, not signed enterprise accounts, so the
* heading stays honest ("Used by people at") rather than implying contracts.
*
* Logos live in `public/logos/` — official marks from Wikimedia Commons /
* Wikipedia (universities use their seals). Bosta, Devoteam, and Leverage Edu
* have no clean Wikimedia logo, so they fall back to a brand favicon. Each item
* also falls back to a text wordmark on image error.
*/
const COMPANIES: { title: string; file: string }[] = [
{ title: "UC Berkeley", file: "berkeley.svg" },
{ title: "USC", file: "usc.svg" },
{ title: "Texas A&M", file: "tamu.svg" },
{ title: "UW–Madison", file: "wisc.svg" },
{ title: "Pitt", file: "pitt.svg" },
{ title: "Korean Air", file: "koreanair.svg" },
{ title: "Iron Mountain", file: "ironmountain.svg" },
{ title: "Globant", file: "globant.svg" },
{ title: "Devoteam", file: "devoteam.png" },
{ title: "VNG", file: "vng.svg" },
{ title: "TPBank", file: "tpbank.svg" },
{ title: "OpenGov", file: "opengov.png" },
{ title: "WeLab", file: "welab.png" },
{ title: "Leverage Edu", file: "leverageedu.png" },
{ title: "Zopper", file: "zopper.png" },
{ title: "Tec de Monterrey", file: "tec.svg" },
{ title: "Chulalongkorn", file: "chula.svg" },
{ title: "Univ. of Bristol", file: "bristol.svg" },
{ title: "Nutresa", file: "nutresa.svg" },
{ title: "Bosta", file: "bosta.png" },
];
const LOGOS_PER_SET = 10;
const TOTAL_SETS = Math.ceil(COMPANIES.length / LOGOS_PER_SET);
function LogoItem({ title, file }: { title: string; file: string }) {
const [failed, setFailed] = useState(false);
if (failed) {
return (
<span className="text-sm font-semibold text-neutral-500 dark:text-neutral-400">{title}</span>
);
}
return (
// biome-ignore lint/performance/noImgElement: swapped in/out by the cycling animation, next/image adds no value here
<img
src={`/logos/${file}`}
alt={title}
title={title}
width={130}
height={40}
loading="lazy"
onError={() => setFailed(true)}
// dark mode: dark-on-transparent marks would vanish, so render every logo as a
// uniform light silhouette (brightness-0 + invert) instead of relying on its own color
className="h-10 w-auto max-w-[130px] object-contain opacity-60 grayscale transition duration-300 hover:opacity-100 hover:grayscale-0 dark:opacity-70 dark:brightness-0 dark:invert dark:hover:opacity-100"
/>
);
}
export function LogoCloud() {
const [currentSet, setCurrentSet] = useState(0);
useEffect(() => {
const interval = setInterval(() => {
setCurrentSet((prev) => (prev + 1) % TOTAL_SETS);
}, 3000);
return () => clearInterval(interval);
}, []);
const startIndex = currentSet * LOGOS_PER_SET;
const currentLogos = Array.from(
{ length: LOGOS_PER_SET },
(_, i) => COMPANIES[(startIndex + i) % COMPANIES.length]
);
return (
<MarketingSection>
<Reveal>
<h2 className="mx-auto max-w-xl text-center text-lg font-medium text-neutral-600 dark:text-neutral-400">
Used by people at
</h2>
</Reveal>
<div className="mx-auto mt-10 grid max-w-4xl grid-cols-3 gap-8 sm:grid-cols-5">
<AnimatePresence mode="popLayout">
{currentLogos.map((logo, index) => (
<motion.div
key={`${logo.title}-${currentSet}-${index}`}
initial={{ x: 40, opacity: 0, filter: "blur(8px)" }}
animate={{ x: 0, opacity: 1, filter: "blur(0px)" }}
exit={{ x: -40, opacity: 0, filter: "blur(8px)" }}
transition={{ duration: 0.4, ease: "easeOut", delay: index * 0.05 }}
className="flex items-center justify-center"
>
<LogoItem title={logo.title} file={logo.file} />
</motion.div>
))}
</AnimatePresence>
</div>
</MarketingSection>
);
}

21
surfsense_web/components/homepage/navbar.tsx
View file

@ -250,17 +250,22 @@ const DesktopNav = ({ navItems, isScrolled, scrolledBgClassName }: DesktopNavPro
href="https://discord.gg/ejRNvftDp9"
target="_blank"
rel="noopener noreferrer"
aria-label="SurfSense on Discord"
className="hidden rounded-full p-2 hover:bg-gray-100 dark:hover:bg-neutral-800 transition-colors md:flex items-center justify-center"
>
<IconBrandDiscord className="h-5 w-5 text-neutral-600 dark:text-neutral-300" />
<IconBrandDiscord
className="h-5 w-5 text-neutral-600 dark:text-neutral-300"
aria-hidden
/>
</Link>
<Link
href="https://www.reddit.com/r/SurfSense/"
target="_blank"
rel="noopener noreferrer"
aria-label="SurfSense on Reddit"
className="hidden rounded-full p-2 hover:bg-gray-100 dark:hover:bg-neutral-800 transition-colors md:flex items-center justify-center"
>
<IconBrandReddit className="h-5 w-5 text-neutral-600 dark:text-neutral-300" />
<IconBrandReddit className="h-5 w-5 text-neutral-600 dark:text-neutral-300" aria-hidden />
</Link>
<NavbarGitHubStars className="hidden md:flex" />
<ThemeTogglerComponent />
@ -362,17 +367,25 @@ const MobileNav = ({ navItems, isScrolled, scrolledBgClassName }: MobileNavProps
href="https://discord.gg/ejRNvftDp9"
target="_blank"
rel="noopener noreferrer"
aria-label="SurfSense on Discord"
className="flex items-center justify-center rounded-lg p-2 hover:bg-gray-100 dark:hover:bg-neutral-800 transition-colors touch-manipulation"
>
<IconBrandDiscord className="h-5 w-5 text-neutral-600 dark:text-neutral-300" />
<IconBrandDiscord
className="h-5 w-5 text-neutral-600 dark:text-neutral-300"
aria-hidden
/>
</Link>
<Link
href="https://www.reddit.com/r/SurfSense/"
target="_blank"
rel="noopener noreferrer"
aria-label="SurfSense on Reddit"
className="flex items-center justify-center rounded-lg p-2 hover:bg-gray-100 dark:hover:bg-neutral-800 transition-colors touch-manipulation"
>
<IconBrandReddit className="h-5 w-5 text-neutral-600 dark:text-neutral-300" />
<IconBrandReddit
className="h-5 w-5 text-neutral-600 dark:text-neutral-300"
aria-hidden
/>
</Link>
<NavbarGitHubStars className="rounded-lg" />
<ThemeTogglerComponent />

6
surfsense_web/components/homepage/persona-paths.tsx
View file

@ -21,9 +21,9 @@ const PATHS: {
icon: Megaphone,
art: "chat",
eyebrow: "For founders & marketing teams",
title: "Competitor and market intelligence without the enterprise price tag",
title: "Live web research without the enterprise price tag",
description:
"Ask for a competitor teardown, a lead list, or a weekly market brief in plain English. The agent gathers live data, cites its sources, and automations keep watch so you hear about changes first. Start free, pay only for what you use.",
"Ask for a research brief, a lead list, or a competitor teardown in plain English. The agent gathers live data, cites its sources, and automations keep watch so you hear about changes first. Start free, pay only for what you use.",
links: [
{ label: "See what teams build", href: "/connectors" },
{ label: "Pricing", href: "/pricing" },
@ -35,7 +35,7 @@ const PATHS: {
eyebrow: "For developers & agents",
title: "The whole platform is programmable",
description:
"Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.",
"Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.",
links: [
{ label: "Read the docs", href: "/docs" },
{ label: "SurfSense MCP server", href: "/mcp-server" },

298
surfsense_web/components/homepage/social-proof.tsx Normal file
View file

@ -0,0 +1,298 @@
"use client";
import {
IconBrandLinkedin,
IconBrandTiktok,
IconBrandX,
IconBrandYoutube,
} from "@tabler/icons-react";
import { Component, type ReactNode, useEffect, useRef, useState } from "react";
import { LinkedInEmbed, TikTokEmbed, XEmbed, YouTubeEmbed } from "react-social-media-embed";
import { Reveal } from "@/components/connectors-marketing/reveal";
type Post =
| { kind: "youtube"; url: string; title: string; channel: string }
| { kind: "x"; url: string }
| { kind: "linkedin"; url: string; postUrl: string }
| { kind: "tiktok"; url: string };
/**
* Organic SurfSense coverage — real posts embedded live with react-social-media-embed,
* grouped into three platform-uniform marquee rows (heights match within a row).
*
* Note: LinkedIn only renders when the author enabled embedding on the post; if
* disabled, the EmbedBoundary swaps in a link card to the original.
*
* ponytail: three rows, each list duplicated once for a seamless CSS loop. Ceiling:
* heavy third-party embeds. Mitigated by lazy-mounting the whole section on scroll
* (IntersectionObserver). Upgrade path: per-card virtualization, or swap YouTube
* players for thumbnail links.
*/
const YOUTUBE: Post[] = [
{
kind: "youtube",
url: "https://www.youtube.com/watch?v=i9AJ7PHGSGg",
title: "SurfSense vs NotebookLM",
channel: "rezasaad plus",
},
{
kind: "youtube",
url: "https://www.youtube.com/watch?v=VBOwuD6xVK0",
title: "NotebookLM Is Great… Until You See SurfSense",
channel: "Thomas AI",
},
{
kind: "youtube",
url: "https://www.youtube.com/watch?v=UaekqjhUiJM",
title: "NotebookLM vs SurfSense en 6 pruebas reales (sorprendente)",
channel: "NextGen IA Hub",
},
{
kind: "youtube",
url: "https://www.youtube.com/watch?v=QGjKpZJJ9aw",
title: "Gana DINERO configurando “Cerebros de IA” privados con SurfSense",
channel: "Creando Con La IA",
},
{
kind: "youtube",
url: "https://www.youtube.com/watch?v=cfNAIQtNbKY",
title: "¿Adiós NotebookLM? Probé SurfSense y es BRUTAL (IA Gratis)",
channel: "NextGen IA Hub",
},
{
kind: "youtube",
url: "https://www.youtube.com/watch?v=pIWOKSHhf38",
title: "¿Superaron a NotebookLM? SurfSense es Open Source, privada y GRATIS",
channel: "academIArtificial",
},
{
kind: "youtube",
url: "https://www.youtube.com/watch?v=K5xx-J_mQZ8",
title: "¿Mejor que NotebookLM? IA GRATIS con modo local",
channel: "Migue Baena IA",
},
{
kind: "youtube",
url: "https://www.youtube.com/watch?v=AKxM3RUBFsc",
title: "¿Nueva IA GRATIS destroza a NotebookLM de Google? (OPEN SOURCE)",
channel: "Inteligencia Artificial Top",
},
{
kind: "youtube",
url: "https://www.youtube.com/watch?v=jCAgeaVgPDA",
title: "¿Nueva Herramienta IA GRATIS Destroza a NotebookLM? (OPEN SOURCE)",
channel: "Joaquín Barberá",
},
];
const TWEETS: Post[] = [
{ kind: "x", url: "https://x.com/LangChain/status/1853133037019562434" },
{ kind: "x", url: "https://x.com/MoureDev/status/1976279289780740448" },
{ kind: "x", url: "https://x.com/GithubProjects/status/2004892541590929490" },
{ kind: "x", url: "https://x.com/GitHub_Daily/status/1920418408736436438" },
{ kind: "x", url: "https://x.com/tom_doerr/status/2066062170173977088" },
{ kind: "x", url: "https://x.com/itsharmanjot/status/2066118517905354816" },
{ kind: "x", url: "https://x.com/JulianGoldieSEO/status/2011085275133604095" },
{ kind: "x", url: "https://x.com/L_go_mrk/status/2066482853232115847" },
{ kind: "x", url: "https://x.com/semihdev/status/2006275500952736028" },
{ kind: "x", url: "https://x.com/shao__meng/status/1919912860957999494" },
{ kind: "x", url: "https://x.com/LangChain/status/1840406184316342561" },
];
const SOCIAL: Post[] = [
{
kind: "linkedin",
url: "https://www.linkedin.com/embed/feed/update/urn:li:ugcPost:7448203908834938881",
postUrl:
"https://www.linkedin.com/posts/vikas-singh-546643206_most-ai-tools-live-in-your-browser-and-ugcPost-7448203908834938881-gR6y",
},
{
kind: "linkedin",
url: "https://www.linkedin.com/embed/feed/update/urn:li:share:7448351685409701889",
postUrl:
"https://www.linkedin.com/posts/neha-jain-279b80118_ive-been-using-a-lot-of-ai-tools-daily-share-7448351685409701889-JvFP",
},
{
kind: "tiktok",
url: "https://www.tiktok.com/@alejavirivera/video/7603064928114625814",
},
];
const CARD = "mr-4 shrink-0 overflow-hidden rounded-xl border bg-card";
const SIZE: Record<Post["kind"], string> = {
youtube: "h-[262px] w-[340px]",
x: "h-[440px] w-[340px]",
linkedin: "h-[540px] w-[340px]",
tiktok: "h-[540px] w-[340px]",
};
const META: Record<Post["kind"], { Icon: typeof IconBrandX; label: string }> = {
youtube: { Icon: IconBrandYoutube, label: "YouTube" },
x: { Icon: IconBrandX, label: "X" },
linkedin: { Icon: IconBrandLinkedin, label: "LinkedIn" },
tiktok: { Icon: IconBrandTiktok, label: "TikTok" },
};
/**
* Some embeds (notably Facebook, and LinkedIn when the author disabled embedding)
* throw at render/mount instead of degrading gracefully. This boundary stops one
* bad embed from taking down the whole marquee — it swaps in a link card instead.
*/
class EmbedBoundary extends Component<
{ fallback: ReactNode; children: ReactNode },
{ failed: boolean }
> {
state = { failed: false };
static getDerivedStateFromError() {
return { failed: true };
}
render() {
return this.state.failed ? this.props.fallback : this.props.children;
}
}
function FallbackCard({ post }: { post: Post }) {
const { Icon, label } = META[post.kind];
const href = post.kind === "linkedin" ? post.postUrl : post.url;
return (
<div className={`${CARD} ${SIZE[post.kind]}`}>
<a
href={href}
target="_blank"
rel="noopener noreferrer"
className="flex h-full w-full flex-col items-center justify-center gap-3 p-6 text-center text-sm font-medium text-muted-foreground transition-colors hover:text-brand"
>
<Icon className="size-8" aria-hidden />
<span>View this post on {label}</span>
</a>
</div>
);
}
function Card({ post }: { post: Post }) {
switch (post.kind) {
case "youtube":
return (
<div className={`${CARD} ${SIZE.youtube} flex flex-col`}>
<YouTubeEmbed url={post.url} width={340} height={191} />
<div className="flex flex-1 flex-col gap-1.5 p-3">
<div className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
<IconBrandYoutube className="size-4 shrink-0 text-red-500" aria-hidden />
<span className="truncate">{post.channel}</span>
</div>
<a
href={post.url}
target="_blank"
rel="noopener noreferrer"
className="line-clamp-2 text-sm font-medium leading-snug hover:text-brand"
>
{post.title}
</a>
</div>
</div>
);
case "x":
return (
<div className={`${CARD} ${SIZE.x}`}>
<XEmbed url={post.url} width={340} />
</div>
);
case "linkedin":
return (
<div className={`${CARD} ${SIZE.linkedin}`}>
<LinkedInEmbed url={post.url} postUrl={post.postUrl} width={340} height={540} />
</div>
);
case "tiktok":
return (
<div className={`${CARD} ${SIZE.tiktok}`}>
<TikTokEmbed url={post.url} width={340} />
</div>
);
}
}
function Row({
posts,
animation,
duration,
}: {
posts: Post[];
animation: "ss-marquee-l" | "ss-marquee-r";
duration: number;
}) {
return (
<div className="group flex overflow-hidden">
{/* Track holds the list twice; the -50% shift wraps seamlessly (margins, not gap). */}
<div
className="flex w-max shrink-0 group-hover:paused motion-reduce:paused"
style={{ animation: `${animation} ${duration}s linear infinite` }}
>
{[...posts, ...posts].map((post, i) => (
<EmbedBoundary
key={`${post.kind}-${post.url}-${i}`}
fallback={<FallbackCard post={post} />}
>
<Card post={post} />
</EmbedBoundary>
))}
</div>
</div>
);
}
export function SocialProof() {
// Third-party embeds are heavy; only mount them once the section scrolls near view.
const ref = useRef<HTMLDivElement>(null);
const [visible, setVisible] = useState(false);
useEffect(() => {
const el = ref.current;
if (!el) return;
const io = new IntersectionObserver(
(entries) => {
if (entries[0]?.isIntersecting) {
setVisible(true);
io.disconnect();
}
},
{ rootMargin: "300px" }
);
io.observe(el);
return () => io.disconnect();
}, []);
return (
<section className="overflow-hidden py-12 sm:py-16">
<Reveal>
<div className="mx-auto max-w-2xl px-4 text-center">
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
Loved across the internet
</h2>
</div>
</Reveal>
<div
ref={ref}
className="mt-10 flex min-h-[1360px] flex-col gap-4"
style={{
maskImage: "linear-gradient(to right, transparent, black 6%, black 94%, transparent)",
WebkitMaskImage:
"linear-gradient(to right, transparent, black 6%, black 94%, transparent)",
}}
>
{visible ? (
<>
<Row posts={YOUTUBE} animation="ss-marquee-l" duration={60} />
<Row posts={TWEETS} animation="ss-marquee-r" duration={75} />
{/* Only 3 social cards — pre-double so one set spans wide viewports (no loop gap). */}
<Row posts={[...SOCIAL, ...SOCIAL]} animation="ss-marquee-l" duration={70} />
</>
) : null}
</div>
<style>{`
@keyframes ss-marquee-l { from { transform: translateX(0); } to { transform: translateX(-50%); } }
@keyframes ss-marquee-r { from { transform: translateX(-50%); } to { transform: translateX(0); } }
`}</style>
</section>
);
}

26
surfsense_web/components/homepage/use-cases.tsx
View file

@ -13,17 +13,17 @@ const USE_CASES: {
art: UseCaseArtVariant;
}[] = [
{
title: "Competitor price monitoring",
title: "Search & AI answer research",
description:
"Crawl competitor pricing and product pages on a schedule and get an alert the day something changes, not the quarter after.",
href: "/web-crawl",
anchor: "Web Crawl API",
art: "price",
"Watch the rankings, ads, and AI Overviews people actually see for the queries you care about, and know the moment they change.",
href: "/google-search",
anchor: "SERP API",
art: "serp",
},
{
title: "Brand monitoring",
title: "Community & brand listening",
description:
"Track every mention of your brand, your competitors, and your category across the communities where buyers speak candidly.",
"Track every mention of your brand, your competitors, and your category across the communities where people speak candidly.",
href: "/reddit",
anchor: "Reddit API",
art: "brand",
@ -31,7 +31,7 @@ const USE_CASES: {
{
title: "Social sentiment mining",
description:
"Pull public posts, reels, and full comment threads from any creator or competitor, then score how audiences actually react to launches and campaigns.",
"Pull public posts, reels, and full comment threads from any creator or brand, then score how audiences actually react to launches and campaigns.",
href: "/instagram",
anchor: "Instagram API",
art: "chat",
@ -45,12 +45,12 @@ const USE_CASES: {
art: "leads",
},
{
title: "Market research",
title: "Competitor price monitoring",
description:
"Watch the rankings, ads, and AI answers your market actually sees, and mine audience sentiment at scale.",
href: "/google-search",
anchor: "SERP API",
art: "serp",
"Crawl competitor pricing and product pages on a schedule and get an alert the day something changes, not the quarter after.",
href: "/web-crawl",
anchor: "Web Crawl API",
art: "price",
},
];

12
surfsense_web/components/layout/providers/LayoutDataProvider.tsx
View file

@ -2,7 +2,7 @@
import { useQuery } from "@tanstack/react-query";
import { useAtomValue, useSetAtom } from "jotai";
import { AlarmClock, AlertTriangle, Boxes, SquareTerminal } from "lucide-react";
import { AlarmClock, AlertTriangle, Shapes, SquareTerminal } from "lucide-react";
import { useParams, usePathname, useRouter } from "next/navigation";
import { useTranslations } from "next-intl";
import { useTheme } from "next-themes";
@ -57,10 +57,15 @@ import { LayoutShell } from "../ui/shell";
interface LayoutDataProviderProps {
workspaceId: string;
initialPlaygroundSidebarCollapsed: boolean;
children: React.ReactNode;
}
export function LayoutDataProvider({ workspaceId, children }: LayoutDataProviderProps) {
export function LayoutDataProvider({
workspaceId,
initialPlaygroundSidebarCollapsed,
children,
}: LayoutDataProviderProps) {
const t = useTranslations("dashboard");
const tCommon = useTranslations("common");
const tSidebar = useTranslations("sidebar");
@ -307,7 +312,7 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
{
title: "Artifacts",
url: `/dashboard/${workspaceId}/artifacts`,
icon: Boxes,
icon: Shapes,
isActive: isArtifactsActive,
},
{
@ -724,6 +729,7 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
onTabSwitch={handleTabSwitch}
onTabPrefetch={handleTabPrefetch}
playgroundSidebar={<PlaygroundSidebar workspaceId={workspaceId} />}
initialPlaygroundSidebarCollapsed={initialPlaygroundSidebarCollapsed}
>
<Fragment key={chatResetKey}>{children}</Fragment>
</LayoutShell>

17
surfsense_web/components/layout/ui/dialogs/CreateWorkspaceDialog.tsx
View file

@ -28,6 +28,8 @@ import {
import { Input } from "@/components/ui/input";
import { Spinner } from "@/components/ui/spinner";
import { trackWorkspaceCreated } from "@/lib/posthog/events";
import { cacheKeys } from "@/lib/query-client/cache-keys";
import { queryClient } from "@/lib/query-client/client";
const formSchema = z.object({
name: z.string().min(1, "Name is required"),
@ -67,7 +69,20 @@ export function CreateWorkspaceDialog({ open, onOpenChange }: CreateWorkspaceDia
trackWorkspaceCreated(result.id, values.name);
router.push(`/dashboard/${result.id}/new-chat`);
// Seed the gate's query so it resolves without a loader flash, and
// route straight to onboarding vs. new-chat on the first hop.
if (result.llm_setup) {
queryClient.setQueryData(
cacheKeys.modelConnections.setupStatus(result.id),
result.llm_setup
);
}
// A fresh workspace can never be recovery, so this matches the gate,
// which is the authoritative net regardless.
const isInitialSetup = result.llm_setup?.stage === "initial_setup";
router.push(
isInitialSetup ? `/dashboard/${result.id}/onboard` : `/dashboard/${result.id}/new-chat`
);
} catch (error) {
console.error("Failed to create workspace:", error);
setIsSubmitting(false);

16
surfsense_web/components/layout/ui/shell/LayoutShell.tsx
View file

@ -41,6 +41,9 @@ const DocumentTabContent = dynamic(
}
);
const PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE = "surfsense_playground_sidebar_collapsed";
const PLAYGROUND_SIDEBAR_COOKIE_MAX_AGE = 60 * 60 * 24 * 365;
function MacDesktopTitleBar({
isSidebarCollapsed,
onToggleSidebar,
@ -113,6 +116,7 @@ interface LayoutShellProps {
onTabSwitch?: (tab: Tab) => void;
onTabPrefetch?: (tab: Tab) => void;
playgroundSidebar?: React.ReactNode;
initialPlaygroundSidebarCollapsed?: boolean;
}
function MainContentPanel({
@ -214,12 +218,15 @@ export function LayoutShell({
onTabSwitch,
onTabPrefetch,
playgroundSidebar,
initialPlaygroundSidebarCollapsed = false,
}: LayoutShellProps) {
const isMobile = useIsMobile();
const electronAPI = useElectronAPI();
const isMacDesktop = electronAPI?.versions.platform === "darwin";
const [mobileMenuOpen, setMobileMenuOpen] = useState(false);
const [isPlaygroundSidebarCollapsed, setIsPlaygroundSidebarCollapsed] = useState(false);
const [isPlaygroundSidebarCollapsed, setIsPlaygroundSidebarCollapsed] = useState(
initialPlaygroundSidebarCollapsed
);
const { isCollapsed, setIsCollapsed, toggleCollapsed } = useSidebarState(defaultCollapsed);
const {
sidebarWidth,
@ -233,7 +240,12 @@ export function LayoutShell({
[isCollapsed, setIsCollapsed, toggleCollapsed]
);
const handlePlaygroundSidebarToggle = () => {
setIsPlaygroundSidebarCollapsed((collapsed) => !collapsed);
setIsPlaygroundSidebarCollapsed((collapsed) => {
const nextCollapsed = !collapsed;
const secureAttribute = window.location.protocol === "https:" ? "; Secure" : "";
document.cookie = `${PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE}=${nextCollapsed}; Path=/; Max-Age=${PLAYGROUND_SIDEBAR_COOKIE_MAX_AGE}; SameSite=Lax${secureAttribute}`;
return nextCollapsed;
});
};
// Mobile layout

31
surfsense_web/components/mcp/agent-setup-tabs.tsx
View file

@ -30,18 +30,17 @@ function CopyButton({ text }: { text: string }) {
<Button
variant="ghost"
size="sm"
className="absolute top-2 right-2 h-7 gap-1.5 px-2 text-xs"
className="absolute top-2 right-2 size-7 p-0"
onClick={handleCopy}
aria-label="Copy configuration"
aria-label={copied ? "Configuration copied" : "Copy configuration"}
>
{copied ? <Check className="size-3.5 text-brand" /> : <Copy className="size-3.5" />}
{copied ? "Copied" : "Copy"}
{copied ? <Check className="size-3.5" /> : <Copy className="size-3.5" />}
</Button>
);
}
const TRANSPORTS: { id: McpTransport; label: string; hint: string }[] = [
{ id: "remote", label: "Hosted", hint: "mcp.surfsense.com — nothing to install" },
{ id: "remote", label: "Hosted", hint: "mcp.surfsense.com, nothing to install" },
{ id: "stdio", label: "Self-host", hint: "run the server against your own backend" },
];
@ -64,7 +63,7 @@ export function AgentSetupTabs({ options }: { options?: Partial<McpSnippetOption
const active = TRANSPORTS.find((t) => t.id === transport) ?? TRANSPORTS[0];
return (
<div className="space-y-4">
<div className="min-w-0 space-y-4">
<div className="flex flex-wrap items-center gap-3">
<div className="inline-flex rounded-lg border bg-muted/40 p-0.5">
{TRANSPORTS.map((t) => (
@ -83,19 +82,21 @@ export function AgentSetupTabs({ options }: { options?: Partial<McpSnippetOption
<span className="text-xs text-muted-foreground">{active.hint}</span>
</div>
<Tabs defaultValue={MCP_CLIENTS[0].id} className="w-full">
<TabsList className="flex h-auto flex-wrap justify-start gap-1">
{MCP_CLIENTS.map((client) => (
<TabsTrigger key={client.id} value={client.id}>
{client.label}
</TabsTrigger>
))}
</TabsList>
<Tabs defaultValue={MCP_CLIENTS[0].id} className="min-w-0 w-full">
<div className="w-full max-w-full overflow-x-auto overscroll-x-contain">
<TabsList className="flex h-auto w-max min-w-full flex-nowrap justify-start gap-1">
{MCP_CLIENTS.map((client) => (
<TabsTrigger key={client.id} value={client.id}>
{client.label}
</TabsTrigger>
))}
</TabsList>
</div>
{MCP_CLIENTS.map((client) => {
const snippet = client[transport];
const config = snippet.build(resolved);
return (
<TabsContent key={client.id} value={client.id} className="space-y-3">
<TabsContent key={client.id} value={client.id} className="min-w-0 space-y-3">
<ol className="list-decimal space-y-1 pl-5 text-sm leading-relaxed text-muted-foreground">
{snippet.steps.map((step) => (
<li key={step}>{step}</li>

7
surfsense_web/components/mcp/connect-agent-dialog.tsx
View file

@ -41,13 +41,12 @@ export function ConnectAgentDialog({ className }: { className?: string }) {
<SidebarButtonBadge>New</SidebarButtonBadge>
</span>
</DialogTrigger>
<DialogContent className="max-h-[85vh] overflow-y-auto sm:max-w-2xl">
<DialogContent className="max-h-[85vh] min-w-0 overflow-x-hidden overflow-y-auto sm:max-w-2xl">
<DialogHeader>
<DialogTitle>Connect to Claude Code, Codex, OpenCode…</DialogTitle>
<DialogDescription>
The SurfSense MCP server gives any coding agent these scrapers and your knowledge base
as native tools. You need an API key (create one under API Keys) — then pick your agent
and paste its config.
Give your coding agent access to SurfSense scrapers and your knowledge base. Create an
API key under API Keys, choose your agent, then paste the config.
</DialogDescription>
</DialogHeader>
<AgentSetupTabs options={{ baseUrl: BACKEND_URL || undefined }} />

6
surfsense_web/components/new-chat/composer-suggestion-popup.tsx
View file

@ -181,13 +181,13 @@ function ComposerSuggestionSkeleton({
}
export {
ComposerSuggestionPopoverContent,
ComposerSuggestionList,
ComposerSuggestionGroup,
ComposerSuggestionGroupHeading,
ComposerSuggestionHeader,
ComposerSuggestionItem,
ComposerSuggestionSeparator,
ComposerSuggestionList,
ComposerSuggestionMessage,
ComposerSuggestionPopoverContent,
ComposerSuggestionSeparator,
ComposerSuggestionSkeleton,
};

8
surfsense_web/components/pricing/pricing-section.tsx
View file

@ -35,7 +35,7 @@ const demoPlans = [
billingText: "Your first $5 of credit is free. No subscription, ever",
features: [
"$5 of free credit to start, one balance for everything",
"Platform connectors: Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web",
"Platform connectors: Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, and the open web",
"Call every connector as a REST API with your key or through the MCP server",
"Pay per item returned and per page crawled. Failed calls are never billed",
"Premium models like GPT-5.5, Claude Sonnet 5, Gemini 3.1 Pro billed at provider cost",
@ -145,12 +145,12 @@ const faqData: FAQSection[] = [
{
question: "Do agents and automations cost extra?",
answer:
"No. There is no add-on fee for agents or automations. A scheduled competitor brief or an event-triggered alert draws from the same credit balance: connector items and crawled pages at their per-unit rates, and model usage at provider cost. A workflow that uses free models and no scraping costs nothing.",
"No. There is no add-on fee for agents or automations. A scheduled research brief or an event-triggered alert draws from the same credit balance: connector items and crawled pages at their per-unit rates, and model usage at provider cost. A workflow that uses free models and no scraping costs nothing.",
},
{
question: "What can the agents actually do?",
answer:
"You describe the job in plain English and SurfSense sets up the agent, no code needed. Agents can watch competitor pricing pages, track brand mentions on Reddit and YouTube, monitor Google rankings and Maps reviews, then turn what they find into briefs and alerts, and write results back to Notion, Slack, Linear, and Jira.",
"You describe the job in plain English and SurfSense sets up the agent, no code needed. Agents can watch any page for changes, track mentions on Reddit and YouTube, monitor Google rankings and Maps reviews, then turn what they find into briefs and alerts, and write results back to Notion, Slack, Linear, and Jira.",
},
],
},
@ -363,7 +363,7 @@ function PricingBasic() {
<Pricing
plans={demoPlans}
title="SurfSense Pricing"
description="Give your agents competitive intelligence. Self-host for free, or start with $5 of credit and pay as you go. No subscriptions."
description="Give your agents the live web. Self-host for free, or start with $5 of credit and pay as you go. No subscriptions."
/>
<PricingFAQ />
</>

10
surfsense_web/components/seo/json-ld.tsx
View file

@ -19,7 +19,7 @@ export function OrganizationJsonLd() {
url: "https://www.surfsense.com",
logo: "https://www.surfsense.com/logo.png",
description:
"SurfSense is an open-source competitive intelligence platform. AI agents monitor competitors, track rankings, and listen to your market through one API or MCP server.",
"SurfSense is an open-source NotebookLM alternative for AI agents. It researches the live web with structured data through one API or MCP server.",
sameAs: [
"https://github.com/MODSetter/SurfSense",
"https://discord.gg/ejRNvftDp9",
@ -45,7 +45,7 @@ export function WebSiteJsonLd() {
name: "SurfSense",
url: "https://www.surfsense.com",
description:
"SurfSense is an open-source competitive intelligence platform for AI agents, with live data connectors served through one API or MCP server.",
"SurfSense is an open-source NotebookLM alternative for AI agents, an open web research platform with live data connectors served through one API or MCP server.",
potentialAction: {
"@type": "SearchAction",
target: {
@ -76,14 +76,14 @@ export function SoftwareApplicationJsonLd() {
"Free self-hosted from the open-source repo; cloud starts with $5 of free credit, then pay as you go",
},
description:
"SurfSense is an open-source competitive intelligence platform. AI agents monitor competitors, track rankings, and listen to your market with platform-native connectors for Reddit, YouTube, TikTok, Google Maps, Google Search, and the open web, through one API or MCP server.",
"SurfSense is an open-source NotebookLM alternative for AI agents. It researches the live web with platform-native connectors for Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, and any page on the open web, through one API or MCP server.",
url: "https://www.surfsense.com",
downloadUrl: "https://github.com/MODSetter/SurfSense/releases",
featureList: [
"Platform-native connectors: Reddit, YouTube, TikTok, Google Maps, Google Search, Web Crawl",
"Platform-native connectors: Reddit, YouTube, TikTok, Amazon, Google Maps, Google Search, Web Crawl",
"MCP server that exposes every connector as a native agent tool",
"Agent harness with retries, structured output, and credit metering",
"Competitor, brand, and rank monitoring with briefs and alerts",
"Live web research with cited briefs and alerts",
"AI automations and agents (scheduled and event-triggered workflows)",
"AI-powered semantic search across connected tools and documents",
"Knowledge base with file uploads and Google Drive, OneDrive, and Dropbox sync",

4
surfsense_web/components/tool-ui/generate-image.tsx
View file

@ -135,8 +135,8 @@ export const GenerateImageToolUI = ({
};
export {
GenerateImageArgsSchema,
GenerateImageResultSchema,
type GenerateImageArgs,
GenerateImageArgsSchema,
type GenerateImageResult,
GenerateImageResultSchema,
};

2
surfsense_web/components/tool-ui/sandbox-execute.tsx
View file

@ -418,4 +418,4 @@ export const SandboxExecuteToolUI = ({
return <ExecuteCompleted command={command} parsed={parsed} threadId={threadId} />;
};
export { ExecuteArgsSchema, ExecuteResultSchema, type ExecuteArgs, type ExecuteResult };
export { type ExecuteArgs, ExecuteArgsSchema, type ExecuteResult, ExecuteResultSchema };

4
surfsense_web/components/tool-ui/user-memory.tsx
View file

@ -90,8 +90,8 @@ export const UpdateMemoryToolUI = ({
// ============================================================================
export {
UpdateMemoryArgsSchema,
UpdateMemoryResultSchema,
type UpdateMemoryArgs,
UpdateMemoryArgsSchema,
type UpdateMemoryResult,
UpdateMemoryResultSchema,
};

2
surfsense_web/components/tool-ui/write-todos.tsx
View file

@ -157,4 +157,4 @@ export const WriteTodosToolUI = ({
);
};
export { WriteTodosSchema, type WriteTodosData };
export { type WriteTodosData, WriteTodosSchema };

2
surfsense_web/components/ui/accordion.tsx
View file

@ -61,4 +61,4 @@ function AccordionContent({
);
}
export { Accordion, AccordionItem, AccordionTrigger, AccordionContent };
export { Accordion, AccordionContent, AccordionItem, AccordionTrigger };

16
surfsense_web/components/ui/alert-dialog.tsx
View file

@ -121,14 +121,14 @@ function AlertDialogCancel({
export {
AlertDialog,
AlertDialogPortal,
AlertDialogOverlay,
AlertDialogTrigger,
AlertDialogContent,
AlertDialogHeader,
AlertDialogFooter,
AlertDialogTitle,
AlertDialogDescription,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogOverlay,
AlertDialogPortal,
AlertDialogTitle,
AlertDialogTrigger,
};

2
surfsense_web/components/ui/alert.tsx
View file

@ -59,4 +59,4 @@ function AlertDescription({ className, ...props }: React.ComponentProps<"div">)
);
}
export { Alert, AlertTitle, AlertDescription };
export { Alert, AlertDescription, AlertTitle };

2
surfsense_web/components/ui/animated-tabs.tsx
View file

@ -556,4 +556,4 @@ const TabsContent = forwardRef<
});
TabsContent.displayName = "TabsContent";
export { Tabs, TabsList, TabsTrigger, TabsContent };
export { Tabs, TabsContent, TabsList, TabsTrigger };

2
surfsense_web/components/ui/avatar.tsx
View file

@ -55,4 +55,4 @@ function AvatarGroupCount({ className, ...props }: React.ComponentProps<"span">)
);
}
export { Avatar, AvatarImage, AvatarFallback, AvatarGroup, AvatarGroupCount };
export { Avatar, AvatarFallback, AvatarGroup, AvatarGroupCount, AvatarImage };

2
surfsense_web/components/ui/card.tsx
View file

@ -52,4 +52,4 @@ const CardFooter = React.forwardRef<HTMLDivElement, React.HTMLAttributes<HTMLDiv
);
CardFooter.displayName = "CardFooter";
export { Card, CardHeader, CardTitle, CardDescription, CardContent, CardFooter };
export { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle };

2
surfsense_web/components/ui/collapsible.tsx
View file

@ -18,4 +18,4 @@ function CollapsibleContent({
return <CollapsiblePrimitive.CollapsibleContent data-slot="collapsible-content" {...props} />;
}
export { Collapsible, CollapsibleTrigger, CollapsibleContent };
export { Collapsible, CollapsibleContent, CollapsibleTrigger };

6
surfsense_web/components/ui/command.tsx
View file

@ -150,11 +150,11 @@ function CommandShortcut({ className, ...props }: React.ComponentProps<"span">)
export {
Command,
CommandDialog,
CommandInput,
CommandList,
CommandEmpty,
CommandGroup,
CommandInput,
CommandItem,
CommandShortcut,
CommandList,
CommandSeparator,
CommandShortcut,
};

18
surfsense_web/components/ui/context-menu.tsx
View file

@ -66,7 +66,7 @@ function ContextMenuSubContent({
<ContextMenuPrimitive.SubContent
data-slot="context-menu-sub-content"
className={cn(
"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 min-w-[8rem] overflow-hidden rounded-md p-1 shadow-lg",
"bg-popover text-popover-foreground border border-popover-border data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 min-w-[8rem] overflow-hidden rounded-md p-1 shadow-lg",
className
)}
{...props}
@ -83,7 +83,7 @@ function ContextMenuContent({
<ContextMenuPrimitive.Content
data-slot="context-menu-content"
className={cn(
"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 min-w-[8rem] overflow-hidden rounded-md p-1 shadow-md",
"bg-popover text-popover-foreground border border-popover-border data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 min-w-[8rem] overflow-hidden rounded-md p-1 shadow-md",
className
)}
{...props}
@ -207,18 +207,18 @@ function ContextMenuShortcut({ className, ...props }: React.ComponentProps<"span
export {
ContextMenu,
ContextMenuTrigger,
ContextMenuContent,
ContextMenuItem,
ContextMenuCheckboxItem,
ContextMenuRadioItem,
ContextMenuContent,
ContextMenuGroup,
ContextMenuItem,
ContextMenuLabel,
ContextMenuPortal,
ContextMenuRadioGroup,
ContextMenuRadioItem,
ContextMenuSeparator,
ContextMenuShortcut,
ContextMenuGroup,
ContextMenuPortal,
ContextMenuSub,
ContextMenuSubContent,
ContextMenuSubTrigger,
ContextMenuRadioGroup,
ContextMenuTrigger,
};

12
surfsense_web/components/ui/dialog.tsx
View file

@ -92,13 +92,13 @@ DialogDescription.displayName = DialogPrimitive.Description.displayName;
export {
Dialog,
DialogPortal,
DialogOverlay,
DialogClose,
DialogTrigger,
DialogContent,
DialogHeader,
DialogFooter,
DialogTitle,
DialogDescription,
DialogFooter,
DialogHeader,
DialogOverlay,
DialogPortal,
DialogTitle,
DialogTrigger,
};

12
surfsense_web/components/ui/drawer.tsx
View file

@ -102,14 +102,14 @@ DrawerHandle.displayName = "DrawerHandle";
export {
Drawer,
DrawerPortal,
DrawerOverlay,
DrawerTrigger,
DrawerClose,
DrawerContent,
DrawerHeader,
DrawerFooter,
DrawerTitle,
DrawerDescription,
DrawerFooter,
DrawerHandle,
DrawerHeader,
DrawerOverlay,
DrawerPortal,
DrawerTitle,
DrawerTrigger,
};

10
surfsense_web/components/ui/dropdown-menu.tsx
View file

@ -211,18 +211,18 @@ function DropdownMenuSubContent({
export {
DropdownMenu,
DropdownMenuPortal,
DropdownMenuTrigger,
DropdownMenuCheckboxItem,
DropdownMenuContent,
DropdownMenuGroup,
DropdownMenuLabel,
DropdownMenuItem,
DropdownMenuCheckboxItem,
DropdownMenuLabel,
DropdownMenuPortal,
DropdownMenuRadioGroup,
DropdownMenuRadioItem,
DropdownMenuSeparator,
DropdownMenuShortcut,
DropdownMenuSub,
DropdownMenuSubTrigger,
DropdownMenuSubContent,
DropdownMenuSubTrigger,
DropdownMenuTrigger,
};

2
surfsense_web/components/ui/empty.tsx
View file

@ -91,4 +91,4 @@ function EmptyContent({ className, ...props }: React.ComponentProps<"div">) {
);
}
export { Empty, EmptyHeader, EmptyTitle, EmptyDescription, EmptyContent, EmptyMedia };
export { Empty, EmptyContent, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle };

2
surfsense_web/components/ui/expanded-gif-overlay.tsx
View file

@ -91,4 +91,4 @@ const ExpandedGifOverlay = ExpandedMediaOverlay;
/** @deprecated Use useExpandedMedia instead */
const useExpandedGif = useExpandedMedia;
export { ExpandedMediaOverlay, useExpandedMedia, ExpandedGifOverlay, useExpandedGif };
export { ExpandedGifOverlay, ExpandedMediaOverlay, useExpandedGif, useExpandedMedia };

8
surfsense_web/components/ui/form.tsx
View file

@ -140,12 +140,12 @@ function FormMessage({ className, ...props }: React.ComponentProps<"p">) {
}
export {
useFormField,
Form,
FormItem,
FormLabel,
FormControl,
FormDescription,
FormMessage,
FormField,
FormItem,
FormLabel,
FormMessage,
useFormField,
};

8
surfsense_web/components/ui/pagination.tsx
View file

@ -96,9 +96,9 @@ function PaginationEllipsis({ className, ...props }: React.ComponentProps<"span"
export {
Pagination,
PaginationContent,
PaginationLink,
PaginationItem,
PaginationPrevious,
PaginationNext,
PaginationEllipsis,
PaginationItem,
PaginationLink,
PaginationNext,
PaginationPrevious,
};

2
surfsense_web/components/ui/popover.tsx
View file

@ -39,4 +39,4 @@ function PopoverAnchor({ ...props }: React.ComponentProps<typeof PopoverPrimitiv
return <PopoverPrimitive.Anchor data-slot="popover-anchor" {...props} />;
}
export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor };
export { Popover, PopoverAnchor, PopoverContent, PopoverTrigger };

8
surfsense_web/components/ui/sheet.tsx
View file

@ -122,11 +122,11 @@ function SheetDescription({
export {
Sheet,
SheetTrigger,
SheetClose,
SheetContent,
SheetHeader,
SheetFooter,
SheetTitle,
SheetDescription,
SheetFooter,
SheetHeader,
SheetTitle,
SheetTrigger,
};

2
surfsense_web/components/ui/table.tsx
View file

@ -89,4 +89,4 @@ function TableCaption({ className, ...props }: React.ComponentProps<"caption">)
);
}
export { Table, TableHeader, TableBody, TableFooter, TableHead, TableRow, TableCell, TableCaption };
export { Table, TableBody, TableCaption, TableCell, TableFooter, TableHead, TableHeader, TableRow };

2
surfsense_web/components/ui/tabs.tsx
View file

@ -52,4 +52,4 @@ const TabsContent = React.forwardRef<
));
TabsContent.displayName = TabsPrimitive.Content.displayName;
export { Tabs, TabsList, TabsTrigger, TabsContent };
export { Tabs, TabsContent, TabsList, TabsTrigger };

2
surfsense_web/components/ui/tooltip.tsx
View file

@ -82,4 +82,4 @@ function TooltipContent({
);
}
export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider };
export { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger };

2
surfsense_web/content/docs/connectors/index.mdx
View file

@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li
<Card
icon={<Zap />}
title="Native Connectors"
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, and Web Crawl — usable in chat, the API Playground, or via REST"
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, and Web Crawl — usable in chat, the API Playground, or via REST"
href="/docs/connectors/native"
/>
<Card

67
surfsense_web/content/docs/connectors/native/amazon.mdx Normal file
View file

@ -0,0 +1,67 @@
---
title: Amazon
description: Scrape public Amazon product data as structured JSON
---
The Amazon scraper returns public product data as structured JSON: price and list price, rating and review breakdown, availability, marketplace offers, sellers, best-seller ranks, product variants, and on-page reviews. It only uses public, anonymous data — no login or seller account. Give it search terms or Amazon product, search, category, or best-seller URLs.
Marketplace support is inferred from the Amazon URL or `domain` you provide. Supported public marketplaces include US (`amazon.com`), UK (`amazon.co.uk`), Germany (`amazon.de`), Italy (`amazon.it`), and Spain (`amazon.es`). France (`amazon.fr`) is supported in the scraper but remains best-effort because Amazon France is more sensitive to proxy-pool WAF challenges.
## Endpoint
```bash
POST /api/v1/workspaces/{workspace_id}/scrapers/amazon/scrape
```
## Inputs
| Field | Default | Description |
|-------|---------|-------------|
| `urls` | — | Amazon product, search, category, best-seller, or short (`a.co` / `amzn.to`) URLs. Marketplace is inferred from the URL, e.g. `amazon.co.uk`, `amazon.de`, `amazon.it`, or `amazon.es`. Provide `urls` or `search_terms` |
| `search_terms` | — | Search phrases run on the selected `domain`, e.g. `wireless earbuds`. Provide `search_terms` or `urls` |
| `max_items` | `10` | Max products per search term or category/best-seller URL (1–100) |
| `domain` | `www.amazon.com` | Amazon marketplace domain for `search_terms`, e.g. `www.amazon.co.uk`, `www.amazon.de`, `www.amazon.it`, or `www.amazon.es`. `www.amazon.fr` is best-effort due to Amazon WAF sensitivity |
| `include_details` | `true` | Fetch full product detail pages; `false` returns faster card-only results |
| `max_offers` | `0` | Extra marketplace offers per product (0–100); `0` = featured offer only |
| `include_sellers` | `false` | Enrich the product and each offer with the seller's public profile |
| `max_variants` | `0` | Product variants to return as separate results (0–100) |
| `include_variant_prices` | `false` | Attach per-variant prices (one extra request per variant) |
| `country_code` | — | Two-letter delivery country for localized pricing, e.g. `us` |
| `zip_code` | — | Delivery ZIP/postal code for localized availability, e.g. `10001` |
| `language` | — | Content language for the domain, e.g. `en` |
At least one of `urls` or `search_terms` is required, with up to 20 combined sources per call.
## Example
```bash
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/amazon/scrape" \
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"search_terms": ["mechanical keyboard"],
"max_items": 5,
"max_offers": 3
}'
```
## Marketplace examples
```json
{
"urls": [
"https://www.amazon.co.uk/s?k=usb+c+cable",
"https://www.amazon.de/s?k=mechanische+tastatur",
"https://www.amazon.it/s?k=cavo+usb+c",
"https://www.amazon.es/s?k=cable+usb+c"
],
"max_items": 5,
"include_details": false
}
```
For `search_terms`, set `domain` to the marketplace you want to search. For direct URLs, the scraper infers the marketplace from each URL.
The response is `{ "items": [...] }` — one item per product, plus structured error items for any input that could not be resolved. Billing is per returned product; error items are never billed.
For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Amazon → Scrape** in your workspace.

7
surfsense_web/content/docs/connectors/native/index.mdx
View file

@ -1,6 +1,6 @@
---
title: Native Connectors
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, and the web
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, and the web
---
import { Card, Cards } from 'fumadocs-ui/components/card';
@ -43,6 +43,11 @@ Native connectors are SurfSense's own scraper APIs — built into the platform,
description="Job postings by search, company, or URL — with salary and full description"
href="/docs/connectors/native/indeed"
/>
<Card
title="Amazon"
description="Public product data: prices, ratings, offers, sellers, and best-seller ranks"
href="/docs/connectors/native/amazon"
/>
<Card
title="Web Crawl"
description="Scrape any page or spider a whole site into clean markdown"

1
surfsense_web/content/docs/connectors/native/meta.json
View file

@ -8,6 +8,7 @@
"google-maps",
"google-search",
"indeed",
"amazon",
"web-crawl"
],
"defaultOpen": false

2
surfsense_web/content/docs/connectors/native/web-crawl.mdx
View file

@ -51,7 +51,7 @@ curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/web/crawl" \
One item per fetched page (in crawl order) with its markdown, metadata (title, description), crawl provenance (depth, referrer), links classified as internal/external/social/email/tel, and page-level contacts. The response also includes a site-wide `contacts` summary that deduplicates every email, phone, and social profile found — `siteWide: true` marks header/footer values (the company's own contacts) versus page-local finds like individual team members.
Contact details often live on about/contact/privacy pages, so crawl with `maxCrawlDepth >= 1` to surface them — useful for lead generation and competitive intelligence.
Contact details often live on about/contact/privacy pages, so crawl with `maxCrawlDepth >= 1` to surface them — useful for lead generation and company research.
Billing is per successfully fetched page.

2
surfsense_web/content/docs/how-to/mcp-server.mdx
View file

@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
# SurfSense MCP Server
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 25 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 26 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted.

8
surfsense_web/contracts/types/model-connections.types.ts
View file

@ -111,6 +111,13 @@ export const globalLlmConfigStatus = z.object({
exists: z.boolean(),
});
export const llmSetupStatus = z.object({
status: z.enum(["ready", "needs_setup"]),
source: z.enum(["global_config", "models", "none"]),
can_configure: z.boolean(),
stage: z.enum(["initial_setup", "recovery", "ready"]),
});
export const modelProviderRead = z.object({
provider: z.string(),
transport: z.string(),
@ -140,5 +147,6 @@ export type ModelUpdateRequest = z.infer<typeof modelUpdateRequest>;
export type ModelsBulkUpdateRequest = z.infer<typeof modelsBulkUpdateRequest>;
export type ModelRoles = z.infer<typeof modelRoles>;
export type GlobalLlmConfigStatus = z.infer<typeof globalLlmConfigStatus>;
export type LlmSetupStatus = z.infer<typeof llmSetupStatus>;
export type VerifyConnectionResponse = z.infer<typeof verifyConnectionResponse>;
export type ModelProviderRead = z.infer<typeof modelProviderRead>;

5
surfsense_web/contracts/types/workspace.types.ts
View file

@ -1,5 +1,6 @@
import { z } from "zod";
import { paginationQueryParams } from ".";
import { llmSetupStatus } from "./model-connections.types";
export const workspace = z.object({
id: z.number(),
@ -36,7 +37,9 @@ export const createWorkspaceRequest = workspace.pick({ name: true, description:
qna_custom_instructions: z.string().nullable().optional(),
});
export const createWorkspaceResponse = workspace.omit({ member_count: true, is_owner: true });
export const createWorkspaceResponse = workspace
.omit({ member_count: true, is_owner: true })
.extend({ llm_setup: llmSetupStatus.nullable().optional() });
/**
* Get workspace

24
surfsense_web/hooks/use-announcements.ts
View file

@ -11,6 +11,7 @@ import {
} from "@/lib/announcements/announcements-storage";
import {
getActiveAnnouncements,
getPublishedAnnouncements,
msUntilNextTransition,
} from "@/lib/announcements/announcements-utils";
import { isAuthenticated } from "@/lib/auth-utils";
@ -56,10 +57,16 @@ export interface AnnouncementWithState extends Announcement {
interface UseAnnouncementsOptions {
/** Filter by category */
category?: AnnouncementCategory;
/**
* Include announcements whose visibility window has expired (archive
* views). Defaults to false so spotlights, toasts, and unread badges
* only consider currently-active announcements.
*/
includeExpired?: boolean;
}
export function useAnnouncements(options: UseAnnouncementsOptions = {}) {
const { category } = options;
const { category, includeExpired = false } = options;
// Subscribe to state changes (re-renders when localStorage state is bumped)
useSyncExternalStore(subscribe, getSnapshot, getServerSnapshot);
@ -71,12 +78,13 @@ export function useAnnouncements(options: UseAnnouncementsOptions = {}) {
const enriched: AnnouncementWithState[] = useMemo(() => {
const authed = isAuthenticated();
const now = new Date();
let items: AnnouncementWithState[] = getActiveAnnouncements(announcements, authed, now).map(
(a) => ({
...a,
isRead: isAnnouncementRead(a.id),
})
);
const visible = includeExpired
? getPublishedAnnouncements(announcements, authed, now)
: getActiveAnnouncements(announcements, authed, now);
let items: AnnouncementWithState[] = visible.map((a) => ({
...a,
isRead: isAnnouncementRead(a.id),
}));
if (category) {
items = items.filter((a) => a.category === category);
@ -86,7 +94,7 @@ export function useAnnouncements(options: UseAnnouncementsOptions = {}) {
return items;
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [category, stateVersion, tick]);
}, [category, includeExpired, stateVersion, tick]);
// Schedule a re-render when the next announcement starts or expires
useEffect(() => {

39
surfsense_web/lib/announcements/announcements-data.ts
View file

@ -15,9 +15,9 @@ import type { Announcement } from "@/contracts/types/announcement.types";
export const announcements: Announcement[] = [
{
id: "2026-07-05-competitive-intelligence-direction",
title: "SurfSense's Next Chapter: Competitive Intelligence for AI Agents",
title: "SurfSense's Next Chapter: Open Web Research for AI Agents",
description:
"Happy Independence Day to everyone celebrating in the United States! We picked this week to share some big news: SurfSense is now the open-source competitive intelligence agent platform. Your agents monitor competitors, track rankings, and listen to your market with live data from Reddit, YouTube, Google Maps, Google Search, and the open web, through one REST API or MCP server. Everything you rely on today keeps working, and self-hosting stays free.",
"Happy Independence Day to everyone celebrating in the United States! We picked this week to share some big news: SurfSense is now the open-source open web research platform for AI agents. Your agents research the live web with structured data from Reddit, YouTube, Google Maps, Google Search, and any page on the open web, through one REST API or MCP server. Everything you rely on today keeps working, and self-hosting stays free.",
category: "update",
date: "2026-07-05T00:00:00Z",
startTime: "2026-07-05T00:00:00Z",
@ -27,7 +27,7 @@ export const announcements: Announcement[] = [
spotlight: true,
image: {
src: "/announcements/competitive-intelligence.png",
alt: "Platform data tiles for social, video, maps, search, and the web flowing into a central AI core that outputs market charts and alerts.",
alt: "Platform data tiles for social, video, maps, search, and the web flowing into a central AI core that outputs research briefs and alerts.",
},
link: {
label: "Read the full announcement",
@ -54,28 +54,17 @@ export const announcements: Announcement[] = [
url: "/changelog",
},
},
{
id: "announcement-1",
title: "Introducing What's New",
description: "All major product updates will be posted here.",
category: "feature",
date: "2026-02-17T00:00:00Z",
startTime: "2026-02-17T00:00:00Z",
endTime: "2026-02-20T00:00:00Z",
audience: "all",
isImportant: false,
},
{
id: "announcement-6",
title: "Past Test Announcement",
description: "This should be seen by nobody, because it's in the past.",
category: "maintenance",
date: "2026-02-17T00:00:00Z",
startTime: "2026-02-15T23:23:00Z",
endTime: "2026-02-16T00:00:00Z",
audience: "users",
isImportant: true,
},
// {
// id: "announcement-1",
// title: "Introducing What's New",
// description: "All major product updates will be posted here.",
// category: "feature",
// date: "2026-02-17T00:00:00Z",
// startTime: "2026-02-17T00:00:00Z",
// endTime: "2026-02-20T00:00:00Z",
// audience: "all",
// isImportant: false,
// },
// {
// id: "2026-02-10-podcast-improvements",
// title: "Podcast Generation Improvements",

26
surfsense_web/lib/announcements/announcements-utils.ts
View file

@ -37,6 +37,32 @@ export function announcementMatchesAudience(
}
}
/**
* Returns true when the announcement has been published (startTime has
* passed), regardless of whether its visibility window has expired.
* Used for archive views like the announcements page.
*/
export function isAnnouncementPublished(announcement: Announcement, now = new Date()): boolean {
const start = new Date(announcement.startTime).getTime();
if (Number.isNaN(start)) return false;
return now.getTime() >= start;
}
/**
* Filter announcements to all published ones (including expired) targeted
* at the given audience. Powers archive views; use getActiveAnnouncements
* for toasts, spotlights, and unread badges.
*/
export function getPublishedAnnouncements(
announcements: Announcement[],
isAuthenticated: boolean,
now = new Date()
): Announcement[] {
return announcements.filter(
(a) => isAnnouncementPublished(a, now) && announcementMatchesAudience(a, isAuthenticated)
);
}
/**
* Filter announcements to only those that are currently active and
* targeted at the given audience.

2
surfsense_web/lib/apis/connectors-api.service.ts
View file

@ -426,6 +426,6 @@ export interface ObsidianStats {
last_sync_at: string | null;
}
export type { SlackChannel, DiscordChannel };
export type { DiscordChannel, SlackChannel };
export const connectorsApiService = new ConnectorsApiService();

6
surfsense_web/lib/apis/model-connections-api.service.ts
View file

@ -8,6 +8,8 @@ import {
connectionUpdateRequest,
type GlobalLlmConfigStatus,
globalLlmConfigStatus,
type LlmSetupStatus,
llmSetupStatus,
type ModelCreateRequest,
type ModelPreviewRead,
type ModelProviderRead,
@ -40,6 +42,10 @@ class ModelConnectionsApiService {
return baseApiService.get(`/api/v1/global-llm-config-status`, globalLlmConfigStatus);
};
getLlmSetupStatus = async (workspaceId: number): Promise<LlmSetupStatus> => {
return baseApiService.get(`/api/v1/workspaces/${workspaceId}/llm-setup-status`, llmSetupStatus);
};
getModelProviders = async (): Promise<ModelProviderRead[]> => {
return baseApiService.get(`/api/v1/model-providers`, modelProviderListResponse);
};

1
surfsense_web/lib/auth-utils.ts
View file

@ -46,6 +46,7 @@ const PUBLIC_ROUTE_PREFIXES = [
"/google-search",
"/indeed",
"/web-crawl",
"/amazon",
];
/**

31
surfsense_web/lib/chat/example-prompts.ts
View file

@ -20,43 +20,44 @@ export interface ChatExampleCategory {
export const CHAT_EXAMPLE_CATEGORIES: ChatExampleCategory[] = [
{
id: "monitor",
label: "Monitor Competitors",
id: "research",
label: "Research the Web",
prompts: [
"Extract every plan, price, and limit from [competitor]'s pricing page",
"Crawl [competitor]'s changelog and brief me on what they shipped this month",
"Track who ranks and runs ads for [keyword] in the US",
"Which of our target keywords trigger an AI Overview, and who gets cited?",
"Research [topic] across the live web and give me a cited brief",
"Map who ranks for [keyword], crawl each result, and compare their claims in one table",
"Pull the Google Maps reviews for [business] and summarize the top complaints",
"Which queries about [topic] trigger an AI Overview, and who gets cited?",
],
},
{
id: "listen",
label: "Market Listening",
label: "Community Listening",
prompts: [
"Find 20 Reddit posts where people ask for an alternative to [product]",
"Analyze the comments on [channel]'s last 10 videos and cluster the complaints",
"Pull the Google Maps reviews for [business] and summarize the top complaints",
"Find people actively looking to switch away from [competitor] this month",
"What is Reddit saying about [topic] this week?",
"Pull the top TikTok videos for [hashtag] and summarize the trend",
],
},
{
id: "workflows",
label: "Multi-Connector",
id: "monitor",
label: "Monitor Competitors",
prompts: [
"Extract every plan, price, and limit from [competitor]'s pricing page",
"Track the Amazon price, rating, and offers for [product] and its top rivals",
"Crawl [competitor]'s changelog and brief me on what they shipped this month",
"Measure the reaction to [competitor]'s launch across search, Reddit, and YouTube",
"Find the top-rated [category] businesses in [city], crawl their sites, and build a lead list with contacts",
"Map who ranks for [keyword], crawl each result, and compare their pricing in one table",
"Build a 360 on [competitor]: site changes, rank movements, Reddit sentiment, and YouTube reaction",
],
},
{
id: "automate",
label: "Automate",
prompts: [
"Re-check [competitor]'s pricing page daily and alert me on any change",
"Every Monday, crawl our competitors' changelogs and send me a brief",
"Watch [url] daily and alert me on any change",
"Track our brand mentions on Reddit daily and tag buying intent",
"Send me a weekly report on [keyword] rankings and AI Overview citations",
"Every Monday, crawl [site]'s changelog and send me a brief",
],
},
{

320
surfsense_web/lib/connectors-marketing/amazon.tsx Normal file
View file

@ -0,0 +1,320 @@
import { IconBrandAmazon } from "@tabler/icons-react";
import type { ConnectorPageContent } from "./types";
export const amazon: ConnectorPageContent = {
slug: "amazon",
name: "Amazon",
cardTitle: "Amazon Product API",
icon: IconBrandAmazon,
metaTitle: "Amazon Product API for Price and Review Data | SurfSense",
metaDescription:
"Scrape public Amazon product data as structured JSON across US, UK, Germany, Italy, and Spain: prices, ratings, offers, sellers, and ranks. Start free.",
keywords: [
"amazon product api",
"amazon scraper api",
"amazon price scraper",
"scrape amazon product data",
"amazon product data api",
"amazon review scraper",
"amazon best sellers api",
"amazon asin lookup",
"amazon price tracking api",
"amazon competitor monitoring",
"amazon offers scraper",
"ecommerce product api",
],
h1: "Amazon Product API for Price, Review, and Offer Data",
heroLede:
"The SurfSense Amazon Product API scrapes public Amazon listings as structured JSON: price and list price, rating and review breakdown, availability, marketplace offers, sellers, best-seller ranks, and on-page reviews. Point your AI agents at a search term or product URL and track prices across US, UK, German, Italian, and Spanish marketplaces — no login, only public data.",
transcript: {
prompt: "Track the price and rating of the top mechanical keyboards on Amazon",
toolCall:
'amazon.scrape({ search_terms: ["mechanical keyboard"],\n max_items: 5, max_offers: 3 })',
rows: [
{
primary: "Keychron K8 Pro — $89.99 (was $99.99)",
secondary: "4.7 stars · 12,431 ratings · In Stock",
tag: "-10%",
},
{
primary: "3 marketplace offers, cheapest $84.50 used",
secondary: "sold by 2 third-party sellers",
tag: "offers",
},
{
primary: "#2 in Computer Keyboards best-sellers",
secondary: "Electronics › Accessories › Keyboards",
tag: "rank",
},
],
resultSummary: "5 products · 14 offers · 5 best-seller ranks · surfaced in 3.1s",
},
extractIntro:
"Give the API a list of product/search/category/best-seller URLs or search terms and a domain. Marketplace is inferred from each Amazon URL, or from the domain used with search terms. Supported marketplaces include US, UK, Germany, Italy, and Spain; France is best-effort because Amazon France is more WAF-sensitive.",
extractFields: [
{
label: "Product core",
description:
"Title, ASIN, brand, price, list price, availability, condition, and canonical URL for every product.",
},
{
label: "Ratings & reviews",
description:
"Star rating, review count, the 5-to-1-star histogram, and on-page customer reviews with text, author, and date.",
},
{
label: "Marketplace offers",
description:
"Additional buy-box offers with price, condition, delivery, and the third-party seller behind each.",
},
{
label: "Sellers",
description:
"Public seller profile summaries — name, rating, and feedback count — for the featured and offer sellers.",
},
{
label: "Best-seller ranks",
description:
"Category rank positions and best-seller list placements, for demand and category monitoring.",
},
{
label: "Variants & media",
description:
"Variant ASINs and attributes (color, size), per-variant prices, gallery and high-resolution images.",
},
],
useCasesHeading: "What teams do with the Amazon Product API",
useCases: [
{
title: "Price and buy-box tracking",
description:
"Watch your own and competitors' prices, list prices, and marketplace offers across US, UK, German, Italian, and Spanish marketplaces. Feed each run to an agent that diffs prices and alerts you when a competitor undercuts or the buy box changes hands.",
},
{
title: "Review mining and product research",
description:
"Pull ratings, the star histogram, and on-page reviews to understand what customers love and complain about, then brief an agent to summarize themes and inform your roadmap or listing copy.",
},
{
title: "Best-seller and category monitoring",
description:
"Track best-seller ranks in the categories you care about to spot rising products and demand shifts before they show up anywhere else.",
},
{
title: "Catalog and seller intelligence",
description:
"Enrich a catalog by ASIN with structured attributes, variants, and images, and see which third-party sellers are winning offers on the products that matter to you.",
},
],
comparison: {
heading: "An Amazon scraper API built for agents",
intro:
"Most Amazon data APIs bill per request, meter add-ons separately, and leave the discovery-to-detail logic to you. Here is how SurfSense compares.",
columnLabel: "Typical Amazon data API",
rows: [
{
feature: "Pricing",
official: "Per-request pricing that climbs fast at scale",
surfsense: "Pay per product returned, with a free tier to start",
},
{
feature: "Discovery",
official: "Separate search and detail endpoints you stitch together",
surfsense: "One verb takes search terms, product, category, or best-seller URLs",
},
{
feature: "Offers & sellers",
official: "Often separate paid endpoints",
surfsense: "Offers and seller profiles enriched inline on the product",
},
{
feature: "Data access",
official: "Requires accounts, keys, or approval for many providers",
surfsense: "Public, anonymous data only — no login or seller account",
},
{
feature: "Agent-ready",
official: "No; you wire the harness yourself",
surfsense: "MCP server exposes amazon.scrape as a native tool",
},
],
},
api: {
platform: "amazon",
verb: "scrape",
mcpTool: "amazon.scrape",
requestBody: {
search_terms: ["mechanical keyboard"],
max_items: 5,
max_offers: 3,
},
},
schema: {
requestNote:
"Provide urls or search_terms (at least one). Marketplace is inferred from Amazon URLs, or from domain for search terms. Up to 20 combined sources per call.",
request: [
{
name: "urls",
type: "string[]",
description:
"Amazon product, search, category, best-seller, or short (a.co / amzn.to) URLs. Supports amazon.com, amazon.co.uk, amazon.de, amazon.it, and amazon.es; amazon.fr is best-effort. Provide urls or search_terms.",
},
{
name: "search_terms",
type: "string[]",
description:
"Search phrases run on the Amazon domain, e.g. 'wireless earbuds'. Provide search_terms or urls.",
},
{
name: "max_items",
type: "integer",
defaultValue: "10",
description: "Max products per search term or category/best-seller URL, 1 to 100.",
},
{
name: "domain",
type: "string",
defaultValue: '"www.amazon.com"',
description:
"Amazon marketplace domain for search_terms, e.g. 'www.amazon.co.uk', 'www.amazon.de', 'www.amazon.it', or 'www.amazon.es'. 'www.amazon.fr' is best-effort due to WAF sensitivity.",
},
{
name: "include_details",
type: "boolean",
defaultValue: "true",
description: "Fetch full product detail pages. false returns faster card-only results.",
},
{
name: "max_offers",
type: "integer",
defaultValue: "0",
description:
"Extra marketplace offers to fetch per product, 0 to 100. 0 returns the featured offer only.",
},
{
name: "include_sellers",
type: "boolean",
defaultValue: "false",
description: "Enrich the product and each offer with the seller's public profile.",
},
{
name: "max_variants",
type: "integer",
defaultValue: "0",
description: "Product variants to return as separate results, 0 to 100.",
},
{
name: "include_variant_prices",
type: "boolean",
defaultValue: "false",
description: "Attach per-variant prices (one extra request per variant).",
},
{
name: "country_code",
type: "string",
description: "Two-letter delivery country for localized pricing, e.g. 'us'.",
},
{
name: "zip_code",
type: "string",
description: "Delivery ZIP/postal code for localized availability, e.g. '10001'.",
},
{
name: "language",
type: "string",
description: "Content language for the domain, e.g. 'en'.",
},
],
responseNote:
"The response is { items: [...] } with one item per product. One returned product is one billable unit; error items are never billed.",
response: [
{
name: "title / asin / brand",
type: "string",
description: "Product identity: display title, ASIN, and brand.",
},
{
name: "price / listPrice",
type: "object",
description: "Current price and strike-through list price, each with value and currency.",
},
{
name: "stars / reviewsCount / starsBreakdown",
type: "object",
description:
"Average rating, total review count, and the 5-to-1-star distribution as fractions.",
},
{
name: "offers",
type: "object[]",
description:
"Marketplace offers with price, condition, delivery, seller, and pinned-offer flag.",
},
{
name: "seller",
type: "object",
description: "Featured seller profile: id, name, url, rating, and feedback count.",
},
{
name: "bestsellerRanks",
type: "object[]",
description: "Category rank positions and best-seller list placements.",
},
{
name: "variantAsins / variantDetails / variantAttributes",
type: "object[]",
description: "Related variant ASINs and their attributes and per-variant prices.",
},
{
name: "productPageReviews",
type: "object[]",
description: "On-page customer reviews with text, author, rating, and date.",
},
],
},
faq: [
{
question: "What is an Amazon product API?",
answer:
"An Amazon product API returns a listing's data as structured JSON instead of raw HTML. The SurfSense Amazon API scrapes public product pages and gives you price, rating, review breakdown, offers, sellers, and best-seller ranks as clean JSON your agents can read.",
},
{
question: "Can I track Amazon prices with it?",
answer:
"Yes. Each product returns its current price, list price, and marketplace offers. Run the same URLs or search terms on a schedule and diff the prices to build price and buy-box tracking, without maintaining scrapers or proxies yourself.",
},
{
question: "Does it only use public data?",
answer:
"It does. The scraper collects public, anonymous product data only — no login, seller account, or authenticated APIs. That covers product details, ratings, on-page reviews, offers, public seller profiles, and best-seller rankings.",
},
{
question: "How do I look up a product by ASIN or search term?",
answer:
"Pass a product URL (which contains the ASIN) in urls, or a phrase in search_terms to discover products on the domain. You can also pass search, category, and best-seller URLs. One verb, amazon.scrape, handles all of them.",
},
{
question: "Which Amazon marketplaces are supported?",
answer:
"Marketplace routing is inferred from the Amazon URL or domain. SurfSense supports public Amazon data from US, UK, Germany, Italy, and Spain. France is wired in and retried like the others, but remains best-effort because Amazon France is more sensitive to proxy-pool WAF challenges.",
},
],
related: [
{ label: "Google Search API", href: "/google-search" },
{ label: "Google Maps API", href: "/google-maps" },
{ label: "Web Crawl API", href: "/web-crawl" },
{ label: "Reddit API", href: "/reddit" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
{ label: "Read the docs", href: "/docs" },
],
};

4
surfsense_web/lib/connectors-marketing/google-maps.tsx
View file

@ -21,9 +21,9 @@ export const googleMaps: ConnectorPageContent = {
"google maps scraper python",
],
h1: "Google Maps Scraper API for Lead Generation and Local Market Intelligence",
h1: "Google Maps Scraper API for Lead Generation and Local Business Research",
heroLede:
"The SurfSense Google Maps API turns Maps into a B2B lead engine. Extract business names, categories, phones, websites, ratings, and reviews at scale, then point your AI agents at any territory or category to build lead lists and map a local market in minutes.",
"The SurfSense Google Maps API turns Maps into a B2B lead engine. Extract business names, categories, phones, websites, ratings, and reviews at scale, then point your AI agents at any territory or category to build lead lists and map any local category in minutes.",
transcript: {
prompt: "Find every dentist in Austin without a website",

11
surfsense_web/lib/connectors-marketing/google-search.tsx
View file

@ -7,9 +7,9 @@ export const googleSearch: ConnectorPageContent = {
cardTitle: "SERP API",
icon: IconBrandGoogle,
metaTitle: "SERP API for Rank Tracking and Competitor Intel | SurfSense",
metaTitle: "SERP API for Rank Tracking and AI Overview Monitoring | SurfSense",
metaDescription:
"Track rankings, competitor ads, and Google AI Overviews with the SurfSense SERP API. Scrape Google Search results as structured JSON via API or MCP. Start free.",
"Track rankings, ads, and Google AI Overviews with the SurfSense SERP API. Scrape Google Search results as structured JSON via API or MCP. Start free.",
keywords: [
"serp api",
"serp api alternative",
@ -25,9 +25,9 @@ export const googleSearch: ConnectorPageContent = {
"serp analysis",
],
h1: "SERP API for Rank Tracking and Competitor SERP Intelligence",
h1: "SERP API for Rank Tracking and AI Overview Monitoring",
heroLede:
"The SurfSense SERP API scrapes Google Search results as structured JSON: organic rankings, paid ads, AI Overviews, and People Also Ask. Point your AI agents at the queries that matter to your market and know the moment a rank moves, an ad appears, or an AI answer starts citing a competitor.",
"The SurfSense SERP API scrapes Google Search results as structured JSON: organic rankings, paid ads, AI Overviews, and People Also Ask. Point your AI agents at the queries you care about and know the moment a rank moves, an ad appears, or an AI answer changes its sources.",
transcript: {
prompt: "Track who's ranking and running ads for 'competitor pricing' in the US",
@ -77,8 +77,7 @@ export const googleSearch: ConnectorPageContent = {
},
{
label: "Related searches",
description:
"Suggested and related queries, to map how your market actually phrases its search.",
description: "Suggested and related queries, to map how people actually phrase the search.",
},
{
label: "SERP metadata",

2
surfsense_web/lib/connectors-marketing/index.ts
View file

@ -1,3 +1,4 @@
import { amazon } from "./amazon";
import { googleMaps } from "./google-maps";
import { googleSearch } from "./google-search";
import { indeed } from "./indeed";
@ -19,6 +20,7 @@ const CONNECTOR_LIST: ConnectorPageContent[] = [
googleMaps,
googleSearch,
indeed,
amazon,
webCrawl,
];

4
surfsense_web/lib/connectors-marketing/instagram.tsx
View file

@ -6,7 +6,7 @@ export const instagram: ConnectorPageContent = {
name: "Instagram",
icon: IconBrandInstagram,
metaTitle: "Instagram Scraper API for Profiles and Reels | SurfSense",
metaTitle: "Instagram API for Profiles, Posts, and Reels | SurfSense",
metaDescription:
"Instagram scraper API for public profiles, posts, and reels. No login or Graph API review. Structured data for AI agents, plus a free tier. Start now.",
keywords: [
@ -29,7 +29,7 @@ export const instagram: ConnectorPageContent = {
"social listening",
],
h1: "Instagram Scraper API for Profiles, Posts, and Reels",
h1: "Instagram API for Profiles, Posts, and Reels",
heroLede:
"The SurfSense Instagram scraper extracts public profiles, posts, and reels without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post, so you spot trends and shifts in engagement first.",

22
surfsense_web/lib/connectors-marketing/reddit.tsx
View file

@ -6,9 +6,9 @@ export const reddit: ConnectorPageContent = {
name: "Reddit",
icon: IconBrandReddit,
metaTitle: "Reddit Scraper API for Brand Monitoring | SurfSense",
metaTitle: "Reddit API for Brand Monitoring | SurfSense",
metaDescription:
"Track every brand and competitor mention across Reddit with the SurfSense Reddit Scraper API. No official API rate limits or pricing, plus a free tier. Start now.",
"Track every mention of any brand, product, or topic across Reddit with the SurfSense Reddit Scraper API. No official API rate limits or pricing, plus a free tier. Start now.",
keywords: [
"reddit scraper",
"reddit scraper api",
@ -24,28 +24,28 @@ export const reddit: ConnectorPageContent = {
"brand monitoring tool",
],
h1: "Reddit Scraper API for Brand Monitoring and Market Research",
h1: "Reddit API for Brand Monitoring and Community Research",
heroLede:
"The SurfSense Reddit API extracts posts, comments, subreddits, and user data without the official Reddit API's rate limits or per-call pricing. Give your AI agents a live feed of what your market says about your brand, your competitors, and your category, so you hear it first.",
"The SurfSense Reddit API extracts posts, comments, subreddits, and user data without the official Reddit API's rate limits or per-call pricing. Give your AI agents a live feed of what Reddit actually says about any brand, product, or topic, so you hear it first.",
transcript: {
prompt: "See what r/webscraping is saying about our competitor this week",
prompt: "See what r/SaaS is saying about our competitor this week",
toolCall:
'reddit.scrape({ search_queries: ["competitor"], community: "webscraping",\n sort: "top", time_filter: "week", max_comments: 50 })',
'reddit.scrape({ search_queries: ["competitor"], community: "SaaS",\n sort: "top", time_filter: "week", max_comments: 50 })',
rows: [
{
primary: "Switched off Competitor after the API price hike",
secondary: "r/webscraping · 312 upvotes · 0.94 ratio",
secondary: "r/SaaS · 312 upvotes · 0.94 ratio",
tag: "pricing pain",
},
{
primary: "Anyone got a Competitor alternative that does comments?",
secondary: "r/webscraping · 148 upvotes · 61 comments",
secondary: "r/SaaS · 148 upvotes · 61 comments",
tag: "buying intent",
},
{
primary: "Competitor keeps rate-limiting my agent, help",
secondary: "r/webscraping · 89 upvotes · 44 comments",
secondary: "r/SaaS · 89 upvotes · 44 comments",
tag: "churn signal",
},
],
@ -97,7 +97,7 @@ export const reddit: ConnectorPageContent = {
{
title: "Reddit sentiment analysis",
description:
"Pull comment trees at depth and score them for sentiment, so you can measure how a launch, a price change, or a competitor's move actually landed with your market.",
"Pull comment trees at depth and score them for sentiment, so you can measure how a launch, a price change, or an announcement actually landed with the community.",
},
{
title: "Community research for go-to-market",
@ -146,7 +146,7 @@ export const reddit: ConnectorPageContent = {
mcpTool: "reddit.scrape",
requestBody: {
search_queries: ["your brand"],
community: "webscraping",
community: "SaaS",
sort: "top",
time_filter: "week",
max_comments: 50,

6
surfsense_web/lib/connectors-marketing/tiktok.tsx
View file

@ -6,7 +6,7 @@ export const tiktok: ConnectorPageContent = {
name: "TikTok",
icon: IconBrandTiktok,
metaTitle: "TikTok Scraper API for Videos and Comments | SurfSense",
metaTitle: "TikTok API for Videos, Comments, and Trends | SurfSense",
metaDescription:
"TikTok scraper API for public videos, comments, hashtags, and profiles. No Research API approval. Structured data for AI agents, plus a free tier. Start now.",
keywords: [
@ -30,9 +30,9 @@ export const tiktok: ConnectorPageContent = {
"influencer research tool",
],
h1: "TikTok Scraper API for Videos, Comments, and Trend Research",
h1: "TikTok API for Videos, Comments, and Trend Research",
heroLede:
"The SurfSense TikTok scraper extracts public videos by hashtag, creator profile, or URL, plus comment threads and trending feeds, without TikTok's approval-gated Research API. Give your AI agents a live feed of what your market watches and shares, so you catch a trend while it is still rising.",
"The SurfSense TikTok scraper extracts public videos by hashtag, creator profile, or URL, plus comment threads and trending feeds, without TikTok's approval-gated Research API. Give your AI agents a live feed of what people watch and share, so you catch a trend while it is still rising.",
transcript: {
prompt: "Find trending TikToks about meal prep this week",

4
surfsense_web/lib/connectors-marketing/web-crawl.tsx
View file

@ -9,7 +9,7 @@ export const webCrawl: ConnectorPageContent = {
metaTitle: "Web Scraping and Crawl API for AI Agents | SurfSense",
metaDescription:
"Point your agents at any URL and get clean, LLM-ready markdown, links, and contacts back. The SurfSense Web Crawl API for deep research and competitor monitoring. Start free.",
"Point your agents at any URL and get clean, LLM-ready markdown, links, and contacts back. The SurfSense Web Crawl API for deep research and site monitoring. Start free.",
keywords: [
"web crawl api",
"web scraping api",
@ -20,7 +20,7 @@ export const webCrawl: ConnectorPageContent = {
"agentic web scraping",
"deep research api",
"web scraping mcp",
"competitor price monitoring",
"website change monitoring",
"crawl website for ai",
"website to markdown",
"web data for ai",

10
surfsense_web/lib/connectors-marketing/youtube.tsx
View file

@ -6,7 +6,7 @@ export const youtube: ConnectorPageContent = {
name: "YouTube",
icon: IconBrandYoutube,
metaTitle: "YouTube Scraper API for Audience Sentiment | SurfSense",
metaTitle: "YouTube Data API for Comments and Transcripts | SurfSense",
metaDescription:
"Scrape YouTube videos, comments, and transcripts at scale with the SurfSense YouTube Scraper API. No Data API quotas, plus a free tier for AI agents. Start now.",
keywords: [
@ -23,9 +23,9 @@ export const youtube: ConnectorPageContent = {
"audience research tools",
],
h1: "YouTube Scraper API for Audience Sentiment and Content Intelligence",
h1: "YouTube Data API for Comments, Transcripts, and Audience Sentiment",
heroLede:
"The SurfSense YouTube API pulls videos, channel stats, comments, and transcripts at scale, without YouTube Data API quotas. Feed your AI agents every comment and transcript that matters to your market, so you know what people actually think about your product and your competitors' content.",
"The SurfSense YouTube API pulls videos, channel stats, comments, and transcripts at scale, without YouTube Data API quotas. Feed your AI agents every comment and transcript that matters to your research, so you know what audiences actually think about any product, topic, or channel.",
transcript: {
prompt: "Summarize how people reacted in the comments on our launch video",
@ -88,9 +88,9 @@ export const youtube: ConnectorPageContent = {
"Read every comment on your launch video or a competitor's ad, not a sample. Score product feedback and ad reaction so your team knows what actually resonated with the audience.",
},
{
title: "Competitor channel analysis",
title: "Channel analysis",
description:
"Pull a rival channel's full catalog with views, likes, and cadence to see which content wins in your niche, then brief an agent to spot the patterns you should copy.",
"Pull any channel's full catalog with views, likes, and cadence to see which content wins in a niche, then brief an agent to spot the patterns worth copying.",
},
{
title: "Influencer vetting",

28
surfsense_web/lib/onboarding.ts
View file

@ -1,28 +0,0 @@
import type { ConnectionRead } from "@/contracts/types/model-connections.types";
export function hasEnabledChatModel(connections: ConnectionRead[]): boolean {
return connections.some(
(connection) =>
connection.enabled &&
connection.models.some((model) => model.enabled && Boolean(model.supports_chat))
);
}
export function isLlmOnboardingComplete(
chatModelId: number | null | undefined,
globalConnections: ConnectionRead[],
workspaceConnections: ConnectionRead[]
): boolean {
const connections = [...globalConnections, ...workspaceConnections];
const resolvedChatModelId = chatModelId ?? 0;
if (resolvedChatModelId === 0) {
return hasEnabledChatModel(connections);
}
return connections.some((connection) =>
connection.models.some(
(model) => model.id === resolvedChatModelId && model.enabled && Boolean(model.supports_chat)
)
);
}

7
surfsense_web/lib/playground/catalog.ts
View file

@ -1,5 +1,6 @@
import type { ComponentType } from "react";
import {
AmazonIcon,
GoogleMapsIcon,
GoogleSearchIcon,
IndeedIcon,
@ -96,6 +97,12 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [
icon: IndeedIcon,
verbs: [{ name: "indeed.scrape", verb: "scrape", label: "Scrape" }],
},
{
id: "amazon",
label: "Amazon",
icon: AmazonIcon,
verbs: [{ name: "amazon.scrape", verb: "scrape", label: "Scrape" }],
},
{
id: "web",
label: "Web",

1
surfsense_web/lib/playground/platform-icons.tsx
View file

@ -22,6 +22,7 @@ function brandIcon(src: string, alt: string) {
};
}
export const AmazonIcon = brandIcon("/connectors/amazon.svg", "Amazon");
export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit");
export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube");
export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram");

38
surfsense_web/lib/playground/platform-overrides/amazon.tsx Normal file
View file

@ -0,0 +1,38 @@
import { Info } from "lucide-react";
import type { FieldOption } from "@/app/dashboard/[workspace_id]/playground/components/schema-form";
import { Alert, AlertDescription } from "@/components/ui/alert";
import type { FormField } from "@/lib/playground/json-schema";
export const AMAZON_DOMAIN_OPTIONS: FieldOption[] = [
{ label: "US", value: "www.amazon.com" },
{ label: "UK", value: "www.amazon.co.uk" },
{ label: "Germany", value: "www.amazon.de" },
{ label: "Italy", value: "www.amazon.it" },
{ label: "Spain", value: "www.amazon.es" },
{ label: "France, best effort", value: "www.amazon.fr" },
];
const AMAZON_SUPPORTED_COUNTRIES = "US, UK, Germany, Italy, Spain, France";
export function getAmazonFieldOptions(field: FormField): FieldOption[] | undefined {
return field.name === "domain" ? AMAZON_DOMAIN_OPTIONS : undefined;
}
export function hasAmazonFranceValue(values: Record<string, unknown>): boolean {
return JSON.stringify(values).toLowerCase().includes("amazon.fr");
}
export function AmazonMarketplaceHint({ showFranceWarning }: { showFranceWarning: boolean }) {
return (
<Alert>
<Info />
<AlertDescription className="flex flex-wrap items-baseline gap-x-1">
<span className="font-medium text-foreground">Supported countries: </span>
<span>{AMAZON_SUPPORTED_COUNTRIES}</span>
{showFranceWarning
? " France is more WAF-sensitive. If this run returns no results, retry later or use another marketplace."
: null}
</AlertDescription>
</Alert>
);
}

1
surfsense_web/lib/query-client/cache-keys.ts
View file

@ -42,6 +42,7 @@ export const cacheKeys = {
globalConfigStatus: () => ["model-connections", "global-config-status"] as const,
providers: () => ["model-connections", "providers"] as const,
roles: (workspaceId: number) => ["model-roles", workspaceId] as const,
setupStatus: (workspaceId: number) => ["llm-setup-status", workspaceId] as const,
},
auth: {
user: ["auth", "user"] as const,

Some files were not shown because too many files have changed in this diff Show more