mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-10 22:32:16 +02:00
feat: api playground, other tweaks
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
7bf559cd5b
commit
50f2d095aa
104 changed files with 5482 additions and 244 deletions
|
|
@ -1,4 +1,4 @@
|
|||
import { ArrowRight, Plug } from "lucide-react";
|
||||
import { ArrowRight, Plug, Server } from "lucide-react";
|
||||
import type { Metadata } from "next";
|
||||
import Link from "next/link";
|
||||
import { getAllConnectors } from "@/lib/connectors-marketing";
|
||||
|
|
@ -75,15 +75,32 @@ export default function ConnectorsIndexPage() {
|
|||
</Link>
|
||||
);
|
||||
})}
|
||||
{/* Bespoke page (not in the scrape-API registry): SurfSense as an MCP client. */}
|
||||
{/* Bespoke pages (not in the scrape-API registry): the two MCP directions. */}
|
||||
<Link
|
||||
href="/mcp-connector"
|
||||
href="/mcp-server"
|
||||
className="group flex flex-col rounded-xl border bg-card p-6 transition-colors hover:border-brand/40"
|
||||
>
|
||||
<span className="flex size-11 items-center justify-center rounded-lg border bg-muted/40 transition-transform duration-200 ease-out group-hover:scale-110 motion-reduce:transition-none motion-reduce:group-hover:scale-100">
|
||||
<Server className="size-5 text-foreground" />
|
||||
</span>
|
||||
<h2 className="mt-4 text-lg font-semibold">SurfSense MCP Server</h2>
|
||||
<p className="mt-2 flex-1 text-sm leading-relaxed text-muted-foreground line-clamp-4">
|
||||
Give Claude, Cursor, or any MCP client native tools for your workspace: every scraper
|
||||
API plus knowledge base search, reads, and writes. One API key.
|
||||
</p>
|
||||
<span className="mt-4 inline-flex items-center gap-1 text-sm font-medium text-foreground">
|
||||
Explore
|
||||
<ArrowRight className="size-4 transition-transform group-hover:translate-x-0.5" />
|
||||
</span>
|
||||
</Link>
|
||||
<Link
|
||||
href="/external-mcp-connectors"
|
||||
className="group flex flex-col rounded-xl border bg-card p-6 transition-colors hover:border-brand/40"
|
||||
>
|
||||
<span className="flex size-11 items-center justify-center rounded-lg border bg-muted/40 transition-transform duration-200 ease-out group-hover:scale-110 motion-reduce:transition-none motion-reduce:group-hover:scale-100">
|
||||
<Plug className="size-5 text-foreground" />
|
||||
</span>
|
||||
<h2 className="mt-4 text-lg font-semibold">MCP Connector</h2>
|
||||
<h2 className="mt-4 text-lg font-semibold">External MCP Connectors</h2>
|
||||
<p className="mt-2 flex-1 text-sm leading-relaxed text-muted-foreground line-clamp-4">
|
||||
Bring any MCP server to your agents. Paste a config like you would in Cursor, tools
|
||||
are auto-discovered, and Notion, Slack, Jira, and more connect with one-click OAuth.
|
||||
|
|
|
|||
|
|
@ -12,35 +12,37 @@ import { Button } from "@/components/ui/button";
|
|||
import { Separator } from "@/components/ui/separator";
|
||||
import type { FaqItem } from "@/lib/connectors-marketing/types";
|
||||
|
||||
const canonicalUrl = "https://www.surfsense.com/mcp-connector";
|
||||
const canonicalUrl = "https://www.surfsense.com/external-mcp-connectors";
|
||||
|
||||
const metaDescription =
|
||||
"The SurfSense MCP connector lets your AI agents use any MCP server. Paste a config, tools are auto-discovered, and every call runs with per-tool approval. Try it free.";
|
||||
"External MCP connectors let your SurfSense agents use any MCP server. Paste a config, tools are auto-discovered, and every call runs with per-tool approval. Try it free.";
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "MCP Connector for AI Agents: Add Any MCP Server | SurfSense",
|
||||
title: "External MCP Connectors: Add Any MCP Server | SurfSense",
|
||||
description: metaDescription,
|
||||
keywords: [
|
||||
"mcp connector",
|
||||
"external mcp connectors",
|
||||
"what is an mcp connector",
|
||||
"mcp client",
|
||||
"add mcp server",
|
||||
"connect mcp server",
|
||||
"mcp integrations",
|
||||
"mcp server for ai agents",
|
||||
],
|
||||
alternates: { canonical: canonicalUrl },
|
||||
openGraph: {
|
||||
title: "MCP Connector for AI Agents: Add Any MCP Server | SurfSense",
|
||||
title: "External MCP Connectors: Add Any MCP Server | SurfSense",
|
||||
description: metaDescription,
|
||||
url: canonicalUrl,
|
||||
siteName: "SurfSense",
|
||||
type: "website",
|
||||
images: [{ url: "/og-image.png", width: 1200, height: 630, alt: "SurfSense MCP connector" }],
|
||||
images: [
|
||||
{ url: "/og-image.png", width: 1200, height: 630, alt: "SurfSense external MCP connectors" },
|
||||
],
|
||||
},
|
||||
twitter: {
|
||||
card: "summary_large_image",
|
||||
title: "MCP Connector for AI Agents: Add Any MCP Server | SurfSense",
|
||||
title: "External MCP Connectors: Add Any MCP Server | SurfSense",
|
||||
description: metaDescription,
|
||||
images: ["/og-image.png"],
|
||||
},
|
||||
|
|
@ -94,14 +96,14 @@ const ONE_CLICK_APPS = [
|
|||
|
||||
const FAQ: FaqItem[] = [
|
||||
{
|
||||
question: "What is an MCP connector?",
|
||||
question: "What is an external MCP connector?",
|
||||
answer:
|
||||
"An MCP connector links an AI application to an MCP (Model Context Protocol) server, so the app's agents can call the server's tools. In SurfSense, you add a server config once, its tools are auto-discovered, and every agent in your workspace can use them with per-tool approval.",
|
||||
"An external MCP connector links SurfSense to an outside MCP (Model Context Protocol) server, so your agents can call that server's tools. You add a server config once, its tools are auto-discovered, and every agent in your workspace can use them with per-tool approval.",
|
||||
},
|
||||
{
|
||||
question: "How is an MCP connector different from an MCP server?",
|
||||
question: "How is this different from the SurfSense MCP server?",
|
||||
answer:
|
||||
"An MCP server exposes tools; an MCP connector consumes them. This page covers SurfSense acting as the client: plugging outside MCP servers into your agents. SurfSense also ships its own MCP server, which exposes connectors like Reddit and Google Maps as tools inside Claude, Cursor, or any MCP client.",
|
||||
"Direction. External MCP connectors make SurfSense the client: outside tools flow into your SurfSense agents. The SurfSense MCP server is the reverse: it exposes your workspace and the scraper APIs as tools inside Claude, Cursor, or any MCP client you already run.",
|
||||
},
|
||||
{
|
||||
question: "Which MCP transports are supported?",
|
||||
|
|
@ -139,14 +141,14 @@ function ConfigCard() {
|
|||
);
|
||||
}
|
||||
|
||||
export default function McpConnectorPage() {
|
||||
export default function ExternalMcpConnectorsPage() {
|
||||
return (
|
||||
<>
|
||||
<JsonLd
|
||||
data={{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "SoftwareApplication",
|
||||
name: "SurfSense MCP Connector",
|
||||
name: "SurfSense External MCP Connectors",
|
||||
applicationCategory: "DeveloperApplication",
|
||||
operatingSystem: "Web",
|
||||
description: metaDescription,
|
||||
|
|
@ -176,21 +178,21 @@ export default function McpConnectorPage() {
|
|||
className="mb-6"
|
||||
items={[
|
||||
{ name: "Connectors", href: "/connectors" },
|
||||
{ name: "MCP Connector", href: "/mcp-connector" },
|
||||
{ name: "External MCP Connectors", href: "/external-mcp-connectors" },
|
||||
]}
|
||||
/>
|
||||
<Badge variant="outline" className="mb-5 gap-1.5 py-1">
|
||||
<Plug className="size-3.5" />
|
||||
MCP connector
|
||||
External MCP connectors
|
||||
</Badge>
|
||||
<h1 className="text-3xl font-bold tracking-tight text-balance sm:text-4xl lg:text-5xl">
|
||||
Bring any MCP server to your AI agents
|
||||
Bring any external MCP server to your agents
|
||||
</h1>
|
||||
<p className="mt-5 max-w-xl text-base leading-relaxed text-muted-foreground sm:text-lg">
|
||||
The SurfSense MCP connector turns your workspace into an MCP client. Add any MCP
|
||||
server with the same config you'd use in Cursor, and its tools are auto-discovered
|
||||
and handed to your agents, guarded by per-tool approval. Notion, Slack, Jira, and
|
||||
more connect with one-click OAuth.
|
||||
External MCP connectors turn your SurfSense workspace into an MCP client. Add any
|
||||
MCP server with the same config you'd use in Cursor, and its tools are
|
||||
auto-discovered and handed to your agents, guarded by per-tool approval. Notion,
|
||||
Slack, Jira, and more connect with one-click OAuth.
|
||||
</p>
|
||||
<div className="mt-8 flex flex-wrap items-center gap-3">
|
||||
<Button asChild size="lg">
|
||||
|
|
@ -273,12 +275,18 @@ export default function McpConnectorPage() {
|
|||
<MarketingSection>
|
||||
<Reveal>
|
||||
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
|
||||
MCP connector vs MCP server
|
||||
External MCP connectors vs the SurfSense MCP server
|
||||
</h2>
|
||||
<p className="mt-3 max-w-2xl text-muted-foreground leading-relaxed">
|
||||
They are two sides of the same protocol. The MCP connector on this page makes
|
||||
SurfSense a <em>client</em>: it consumes tools from outside MCP servers. The SurfSense
|
||||
MCP <em>server</em> does the reverse, exposing platform connectors like{" "}
|
||||
They are two sides of the same protocol. The external MCP connectors on this page make
|
||||
SurfSense a <em>client</em>: they consume tools from outside MCP servers. The{" "}
|
||||
<Link
|
||||
href="/mcp-server"
|
||||
className="font-medium text-foreground underline underline-offset-4"
|
||||
>
|
||||
SurfSense MCP server
|
||||
</Link>{" "}
|
||||
does the reverse, exposing your workspace and scraper APIs like{" "}
|
||||
<Link
|
||||
href="/reddit"
|
||||
className="font-medium text-foreground underline underline-offset-4"
|
||||
|
|
@ -302,7 +310,7 @@ export default function McpConnectorPage() {
|
|||
<MarketingSection>
|
||||
<Reveal>
|
||||
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
|
||||
MCP connector: frequently asked questions
|
||||
External MCP connectors: frequently asked questions
|
||||
</h2>
|
||||
</Reveal>
|
||||
<Reveal>
|
||||
|
|
@ -320,7 +328,7 @@ export default function McpConnectorPage() {
|
|||
Give your agents every tool they need
|
||||
</h2>
|
||||
<p className="mx-auto mt-3 max-w-xl text-muted-foreground leading-relaxed">
|
||||
The MCP connector is part of the SurfSense{" "}
|
||||
External MCP connectors are part of the SurfSense{" "}
|
||||
<Link href="/" className="font-medium text-foreground underline underline-offset-4">
|
||||
competitive intelligence platform
|
||||
</Link>
|
||||
|
|
@ -344,6 +352,9 @@ export default function McpConnectorPage() {
|
|||
<Button asChild variant="ghost" size="sm">
|
||||
<Link href="/connectors">All connectors</Link>
|
||||
</Button>
|
||||
<Button asChild variant="ghost" size="sm">
|
||||
<Link href="/mcp-server">SurfSense MCP Server</Link>
|
||||
</Button>
|
||||
<Button asChild variant="ghost" size="sm">
|
||||
<Link href="/reddit">Reddit API</Link>
|
||||
</Button>
|
||||
400
surfsense_web/app/(home)/mcp-server/page.tsx
Normal file
400
surfsense_web/app/(home)/mcp-server/page.tsx
Normal file
|
|
@ -0,0 +1,400 @@
|
|||
import { IconBrandGithub } from "@tabler/icons-react";
|
||||
import { ArrowRight, Check, Database, KeyRound, Server, TerminalSquare } from "lucide-react";
|
||||
import type { Metadata } from "next";
|
||||
import Link from "next/link";
|
||||
import { ConnectorFaq } from "@/components/connectors-marketing/connector-faq";
|
||||
import { Reveal } from "@/components/connectors-marketing/reveal";
|
||||
import { MarketingSection } from "@/components/marketing/section";
|
||||
import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav";
|
||||
import { FAQJsonLd, JsonLd } from "@/components/seo/json-ld";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Separator } from "@/components/ui/separator";
|
||||
import type { FaqItem } from "@/lib/connectors-marketing/types";
|
||||
|
||||
const canonicalUrl = "https://www.surfsense.com/mcp-server";
|
||||
|
||||
const metaDescription =
|
||||
"The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Google Maps, Google Search, and the web, plus full knowledge base access. One API key.";
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools",
|
||||
description: metaDescription,
|
||||
keywords: [
|
||||
"surfsense mcp server",
|
||||
"mcp server",
|
||||
"mcp server for web scraping",
|
||||
"reddit mcp server",
|
||||
"youtube mcp server",
|
||||
"google maps mcp server",
|
||||
"serp mcp server",
|
||||
"mcp server for claude",
|
||||
"mcp server for cursor",
|
||||
"knowledge base mcp server",
|
||||
],
|
||||
alternates: { canonical: canonicalUrl },
|
||||
openGraph: {
|
||||
title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools",
|
||||
description: metaDescription,
|
||||
url: canonicalUrl,
|
||||
siteName: "SurfSense",
|
||||
type: "website",
|
||||
images: [{ url: "/og-image.png", width: 1200, height: 630, alt: "SurfSense MCP server" }],
|
||||
},
|
||||
twitter: {
|
||||
card: "summary_large_image",
|
||||
title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools",
|
||||
description: metaDescription,
|
||||
images: ["/og-image.png"],
|
||||
},
|
||||
};
|
||||
|
||||
/* Mirrors surfsense_mcp/README.md: the real Cursor config. */
|
||||
const CURSOR_CONFIG = `{
|
||||
"mcpServers": {
|
||||
"surfsense": {
|
||||
"command": "uv",
|
||||
"args": ["run", "--directory", ".../surfsense_mcp",
|
||||
"python", "-m", "surfsense_mcp"],
|
||||
"env": {
|
||||
"SURFSENSE_BASE_URL": "https://api.surfsense.com",
|
||||
"SURFSENSE_API_KEY": "ss_pat_..."
|
||||
}
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
const STEPS = [
|
||||
{
|
||||
icon: KeyRound,
|
||||
title: "Create an API key",
|
||||
description:
|
||||
"In SurfSense, go to Settings, then API, and create a key. Enable API access on the workspaces you want your agents to reach. That key is all the server needs.",
|
||||
},
|
||||
{
|
||||
icon: TerminalSquare,
|
||||
title: "Add the server to your client",
|
||||
description:
|
||||
"Drop the config into Cursor's mcp.json, run claude mcp add for Claude Code, or paste it into Claude Desktop. Point it at the cloud or your own self-hosted instance.",
|
||||
},
|
||||
{
|
||||
icon: Server,
|
||||
title: "Your agent has the tools",
|
||||
description:
|
||||
"Every scraper and knowledge base operation shows up as a native, typed MCP tool. Your agent picks a workspace once and the server carries the context between calls.",
|
||||
},
|
||||
] as const;
|
||||
|
||||
/** Mirrors the tool registry in surfsense_mcp (see its README). */
|
||||
const TOOL_GROUPS = [
|
||||
{
|
||||
icon: Server,
|
||||
title: "Live scrapers",
|
||||
description: "Structured, current platform data. One returned item is one billable unit.",
|
||||
tools: [
|
||||
"surfsense_reddit_scrape",
|
||||
"surfsense_youtube_scrape",
|
||||
"surfsense_youtube_comments",
|
||||
"surfsense_google_maps_scrape",
|
||||
"surfsense_google_maps_reviews",
|
||||
"surfsense_google_search",
|
||||
"surfsense_web_crawl",
|
||||
"surfsense_list_scraper_runs",
|
||||
"surfsense_get_scraper_run",
|
||||
],
|
||||
},
|
||||
{
|
||||
icon: Database,
|
||||
title: "Knowledge base",
|
||||
description: "Read and write the same knowledge base your SurfSense agents use.",
|
||||
tools: [
|
||||
"surfsense_search_knowledge_base",
|
||||
"surfsense_list_documents",
|
||||
"surfsense_get_document",
|
||||
"surfsense_add_document",
|
||||
"surfsense_upload_file",
|
||||
"surfsense_update_document",
|
||||
"surfsense_delete_document",
|
||||
],
|
||||
},
|
||||
{
|
||||
icon: KeyRound,
|
||||
title: "Workspace selector",
|
||||
description: "Pick a workspace once; every later call defaults to it.",
|
||||
tools: ["surfsense_list_workspaces", "surfsense_select_workspace"],
|
||||
},
|
||||
] as const;
|
||||
|
||||
const FAQ: FaqItem[] = [
|
||||
{
|
||||
question: "What is the SurfSense MCP server?",
|
||||
answer:
|
||||
"It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.",
|
||||
},
|
||||
{
|
||||
question: "Which MCP clients does it work with?",
|
||||
answer:
|
||||
"Any MCP client that supports stdio servers. Claude Code, Cursor, and Claude Desktop are documented with copy-paste configs, and the same command works in custom agent harnesses built on the MCP SDK.",
|
||||
},
|
||||
{
|
||||
question: "How is usage billed?",
|
||||
answer:
|
||||
"Exactly like the REST API, because the server is a thin layer over it. Scraper tools consume the same pay-as-you-go credits, priced per returned item, and knowledge base operations work within your plan. New accounts start with $5 of free credit.",
|
||||
},
|
||||
{
|
||||
question: "Does it work with a self-hosted SurfSense?",
|
||||
answer:
|
||||
"Yes. The server talks to SurfSense purely over its REST API and imports no backend code, so pointing SURFSENSE_BASE_URL at your own instance is all it takes. It works with the cloud at api.surfsense.com the same way.",
|
||||
},
|
||||
{
|
||||
question: "How does the agent know which workspace to use?",
|
||||
answer:
|
||||
"The server ships a workspace selector: the agent lists the workspaces your API key can access, selects one by name, and every later call defaults to it. Any tool also accepts a workspace override for a single call, and ids never need to be typed by hand.",
|
||||
},
|
||||
];
|
||||
|
||||
function ConfigCard() {
|
||||
return (
|
||||
<div className="rounded-xl border bg-card p-5 shadow-sm">
|
||||
<p className="font-mono text-xs text-muted-foreground">.cursor/mcp.json</p>
|
||||
<pre className="mt-2 overflow-x-auto rounded-lg bg-muted/50 p-4 font-mono text-xs leading-relaxed">
|
||||
{CURSOR_CONFIG}
|
||||
</pre>
|
||||
<p className="mt-3 flex items-center gap-1.5 text-xs text-muted-foreground">
|
||||
<Check className="size-3.5 text-brand" aria-hidden />
|
||||
Works with Claude Code, Cursor, Claude Desktop, and any MCP client
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function McpServerPage() {
|
||||
return (
|
||||
<>
|
||||
<JsonLd
|
||||
data={{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "SoftwareApplication",
|
||||
name: "SurfSense MCP Server",
|
||||
applicationCategory: "DeveloperApplication",
|
||||
operatingSystem: "Web",
|
||||
description: metaDescription,
|
||||
url: canonicalUrl,
|
||||
offers: {
|
||||
"@type": "Offer",
|
||||
price: "0",
|
||||
priceCurrency: "USD",
|
||||
description: "$5 free credit included, pay as you go",
|
||||
},
|
||||
provider: {
|
||||
"@type": "Organization",
|
||||
name: "SurfSense",
|
||||
url: "https://www.surfsense.com",
|
||||
},
|
||||
isPartOf: { "@type": "WebSite", name: "SurfSense", url: "https://www.surfsense.com" },
|
||||
}}
|
||||
/>
|
||||
<FAQJsonLd questions={FAQ} />
|
||||
|
||||
<div className="pb-4">
|
||||
{/* Hero */}
|
||||
<MarketingSection className="pt-28 pb-12 sm:pt-32 sm:pb-16">
|
||||
<div className="grid items-center gap-10 lg:grid-cols-2 lg:gap-14">
|
||||
<div>
|
||||
<BreadcrumbNav
|
||||
className="mb-6"
|
||||
items={[
|
||||
{ name: "Connectors", href: "/connectors" },
|
||||
{ name: "SurfSense MCP Server", href: "/mcp-server" },
|
||||
]}
|
||||
/>
|
||||
<Badge variant="outline" className="mb-5 gap-1.5 py-1">
|
||||
<Server className="size-3.5" />
|
||||
SurfSense MCP server
|
||||
</Badge>
|
||||
<h1 className="text-3xl font-bold tracking-tight text-balance sm:text-4xl lg:text-5xl">
|
||||
Give your agents SurfSense as native tools
|
||||
</h1>
|
||||
<p className="mt-5 max-w-xl text-base leading-relaxed text-muted-foreground sm:text-lg">
|
||||
The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform:
|
||||
scrape Reddit, YouTube, Google Maps, Google Search, and the open web, and search,
|
||||
read, and write your knowledge base. One API key, typed tools, pay as you go.
|
||||
</p>
|
||||
<div className="mt-8 flex flex-wrap items-center gap-3">
|
||||
<Button asChild size="lg">
|
||||
<Link href="/register">
|
||||
Get your API key
|
||||
<ArrowRight className="size-4" />
|
||||
</Link>
|
||||
</Button>
|
||||
<Button asChild variant="outline" size="lg">
|
||||
<Link href="/docs">Read the docs</Link>
|
||||
</Button>
|
||||
<Button asChild variant="ghost" size="lg">
|
||||
<Link
|
||||
href="https://github.com/MODSetter/SurfSense"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
<IconBrandGithub className="size-4" />
|
||||
GitHub
|
||||
</Link>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
<ConfigCard />
|
||||
</div>
|
||||
</MarketingSection>
|
||||
|
||||
{/* How it works */}
|
||||
<MarketingSection>
|
||||
<Reveal>
|
||||
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
|
||||
From API key to agent tools in three steps
|
||||
</h2>
|
||||
</Reveal>
|
||||
<div className="mt-8 grid gap-4 sm:grid-cols-3">
|
||||
{STEPS.map((step) => (
|
||||
<Reveal key={step.title}>
|
||||
<div className="h-full rounded-xl border bg-card p-6 transition-colors hover:border-brand/40">
|
||||
<span className="flex size-10 items-center justify-center rounded-lg border bg-muted/40">
|
||||
<step.icon className="size-5 text-foreground" aria-hidden />
|
||||
</span>
|
||||
<h3 className="mt-4 font-semibold">{step.title}</h3>
|
||||
<p className="mt-2 text-sm leading-relaxed text-muted-foreground">
|
||||
{step.description}
|
||||
</p>
|
||||
</div>
|
||||
</Reveal>
|
||||
))}
|
||||
</div>
|
||||
</MarketingSection>
|
||||
|
||||
{/* Tools */}
|
||||
<MarketingSection>
|
||||
<Reveal>
|
||||
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
|
||||
Every tool the server exposes
|
||||
</h2>
|
||||
<p className="mt-3 max-w-2xl text-muted-foreground leading-relaxed">
|
||||
The server is a thin layer over the SurfSense REST API: the same endpoints, the same
|
||||
billing, no backend code imported. Whatever ships in the API shows up here.
|
||||
</p>
|
||||
</Reveal>
|
||||
<div className="mt-8 grid gap-4 lg:grid-cols-3">
|
||||
{TOOL_GROUPS.map((group) => (
|
||||
<Reveal key={group.title}>
|
||||
<div className="h-full rounded-xl border bg-card p-6">
|
||||
<span className="flex size-10 items-center justify-center rounded-lg border bg-muted/40">
|
||||
<group.icon className="size-5 text-foreground" aria-hidden />
|
||||
</span>
|
||||
<h3 className="mt-4 font-semibold">{group.title}</h3>
|
||||
<p className="mt-2 text-sm leading-relaxed text-muted-foreground">
|
||||
{group.description}
|
||||
</p>
|
||||
<ul className="mt-4 space-y-1.5">
|
||||
{group.tools.map((tool) => (
|
||||
<li key={tool} className="truncate font-mono text-xs text-muted-foreground">
|
||||
{tool}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
</Reveal>
|
||||
))}
|
||||
</div>
|
||||
</MarketingSection>
|
||||
|
||||
{/* Server vs external connectors */}
|
||||
<MarketingSection>
|
||||
<Reveal>
|
||||
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
|
||||
The SurfSense MCP server vs external MCP connectors
|
||||
</h2>
|
||||
<p className="mt-3 max-w-2xl text-muted-foreground leading-relaxed">
|
||||
They are two sides of the same protocol. The MCP <em>server</em> on this page pushes
|
||||
SurfSense tools out to agents you already run in Claude, Cursor, or your own harness.{" "}
|
||||
<Link
|
||||
href="/external-mcp-connectors"
|
||||
className="font-medium text-foreground underline underline-offset-4"
|
||||
>
|
||||
External MCP connectors
|
||||
</Link>{" "}
|
||||
do the reverse: they pull outside tools like Notion, Slack, and Jira into your
|
||||
SurfSense agents. Use both and data flows in either direction.
|
||||
</p>
|
||||
</Reveal>
|
||||
</MarketingSection>
|
||||
|
||||
{/* FAQ */}
|
||||
<MarketingSection>
|
||||
<Reveal>
|
||||
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
|
||||
SurfSense MCP server: frequently asked questions
|
||||
</h2>
|
||||
</Reveal>
|
||||
<Reveal>
|
||||
<div className="mt-6 max-w-3xl">
|
||||
<ConnectorFaq items={FAQ} />
|
||||
</div>
|
||||
</Reveal>
|
||||
</MarketingSection>
|
||||
|
||||
{/* Closing CTA + related */}
|
||||
<MarketingSection>
|
||||
<Reveal>
|
||||
<div className="rounded-2xl border bg-card p-8 text-center sm:p-12">
|
||||
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
|
||||
Put live market data inside your agents
|
||||
</h2>
|
||||
<p className="mx-auto mt-3 max-w-xl text-muted-foreground leading-relaxed">
|
||||
The MCP server is part of the SurfSense{" "}
|
||||
<Link href="/" className="font-medium text-foreground underline underline-offset-4">
|
||||
competitive intelligence platform
|
||||
</Link>
|
||||
. Start with $5 of free credit, no credit card required.
|
||||
</p>
|
||||
<div className="mt-7 flex flex-wrap justify-center gap-3">
|
||||
<Button asChild size="lg">
|
||||
<Link href="/register">
|
||||
Start for free
|
||||
<ArrowRight className="size-4" />
|
||||
</Link>
|
||||
</Button>
|
||||
<Button asChild variant="outline" size="lg">
|
||||
<Link href="/pricing">See pricing</Link>
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<Separator className="my-8" />
|
||||
|
||||
<nav aria-label="Other connectors" className="flex flex-wrap justify-center gap-2">
|
||||
<Button asChild variant="ghost" size="sm">
|
||||
<Link href="/connectors">All connectors</Link>
|
||||
</Button>
|
||||
<Button asChild variant="ghost" size="sm">
|
||||
<Link href="/external-mcp-connectors">External MCP Connectors</Link>
|
||||
</Button>
|
||||
<Button asChild variant="ghost" size="sm">
|
||||
<Link href="/reddit">Reddit API</Link>
|
||||
</Button>
|
||||
<Button asChild variant="ghost" size="sm">
|
||||
<Link href="/youtube">YouTube API</Link>
|
||||
</Button>
|
||||
<Button asChild variant="ghost" size="sm">
|
||||
<Link href="/google-maps">Google Maps API</Link>
|
||||
</Button>
|
||||
<Button asChild variant="ghost" size="sm">
|
||||
<Link href="/google-search">SERP API</Link>
|
||||
</Button>
|
||||
<Button asChild variant="ghost" size="sm">
|
||||
<Link href="/web-crawl">Web Crawl API</Link>
|
||||
</Button>
|
||||
</nav>
|
||||
</div>
|
||||
</Reveal>
|
||||
</MarketingSection>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ import { ConnectorGrid } from "@/components/homepage/connector-grid";
|
|||
import { HeroSection } from "@/components/homepage/hero-section";
|
||||
import { HomeFaq } from "@/components/homepage/home-faq";
|
||||
import { HowItWorks } from "@/components/homepage/how-it-works";
|
||||
import { PersonaPaths } from "@/components/homepage/persona-paths";
|
||||
import { UseCasesRow } from "@/components/homepage/use-cases";
|
||||
|
||||
export default function HomePage() {
|
||||
|
|
@ -12,6 +13,7 @@ export default function HomePage() {
|
|||
<div className="min-h-screen bg-linear-to-b from-gray-50 to-gray-100 text-gray-900 dark:from-black dark:to-gray-900 dark:text-white">
|
||||
<AuthRedirect />
|
||||
<HeroSection />
|
||||
<PersonaPaths />
|
||||
<ConnectorGrid />
|
||||
<HowItWorks />
|
||||
<UseCasesRow />
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
import { PlaygroundRunner } from "../../components/playground-runner";
|
||||
|
||||
export default async function PlaygroundVerbPage({
|
||||
params,
|
||||
}: {
|
||||
params: Promise<{ workspace_id: string; platform: string; verb: string }>;
|
||||
}) {
|
||||
const { workspace_id, platform, verb } = await params;
|
||||
|
||||
return (
|
||||
<div className="mx-auto w-full max-w-6xl">
|
||||
<PlaygroundRunner workspaceId={Number(workspace_id)} platform={platform} verb={verb} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import { ApiKeysSection } from "../components/api-keys-section";
|
||||
|
||||
export default async function PlaygroundApiKeysPage({
|
||||
params,
|
||||
}: {
|
||||
params: Promise<{ workspace_id: string }>;
|
||||
}) {
|
||||
const { workspace_id } = await params;
|
||||
|
||||
return (
|
||||
<div className="mx-auto w-full max-w-3xl">
|
||||
<ApiKeysSection workspaceId={Number(workspace_id)} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
"use client";
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { useAtomValue } from "jotai";
|
||||
import { useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { updateWorkspaceApiAccessMutationAtom } from "@/atoms/workspaces/workspace-mutation.atoms";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { workspacesApiService } from "@/lib/apis/workspaces-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
import { ApiKeyContent } from "../../user-settings/components/ApiKeyContent";
|
||||
|
||||
/**
|
||||
* One-stop API key management for the playground: the workspace API-access
|
||||
* toggle (otherwise buried in workspace settings) plus the personal API key
|
||||
* manager (otherwise buried in user settings).
|
||||
*/
|
||||
export function ApiKeysSection({ workspaceId }: { workspaceId: number }) {
|
||||
const {
|
||||
data: workspace,
|
||||
isLoading,
|
||||
refetch,
|
||||
} = useQuery({
|
||||
queryKey: cacheKeys.workspaces.detail(workspaceId.toString()),
|
||||
queryFn: () => workspacesApiService.getWorkspace({ id: workspaceId }),
|
||||
enabled: !!workspaceId,
|
||||
});
|
||||
const { mutateAsync: updateWorkspaceApiAccess } = useAtomValue(
|
||||
updateWorkspaceApiAccessMutationAtom
|
||||
);
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
const handleToggle = async (enabled: boolean) => {
|
||||
try {
|
||||
setSaving(true);
|
||||
await updateWorkspaceApiAccess({ id: workspaceId, api_access_enabled: enabled });
|
||||
await refetch();
|
||||
} catch (error) {
|
||||
console.error("Error updating API access:", error);
|
||||
toast.error(error instanceof Error ? error.message : "Failed to update API access");
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
const apiAccessEnabled = !!workspace?.api_access_enabled;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div>
|
||||
<h1 className="text-xl font-semibold">API Keys</h1>
|
||||
<p className="mt-1 text-sm text-muted-foreground">
|
||||
Enable API access for this workspace and manage the keys that use it.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between gap-4 rounded-lg border border-border/60 px-4 py-3">
|
||||
<div className="space-y-1">
|
||||
<Label htmlFor="playground-api-access">API key access</Label>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Allow API keys to access this workspace.
|
||||
{!isLoading && !apiAccessEnabled && " Currently disabled — keys won't work here."}
|
||||
</p>
|
||||
</div>
|
||||
{isLoading ? (
|
||||
<Skeleton className="h-5 w-9 rounded-full" />
|
||||
) : (
|
||||
<Switch
|
||||
id="playground-api-access"
|
||||
checked={apiAccessEnabled}
|
||||
disabled={saving}
|
||||
onCheckedChange={handleToggle}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<ApiKeyContent />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
"use client";
|
||||
|
||||
import { Check, Copy } from "lucide-react";
|
||||
import { useMemo, useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { BACKEND_URL } from "@/lib/env-config";
|
||||
import { buildExamplePayload, buildSnippets } from "@/lib/playground/code-snippets";
|
||||
import type { FormField } from "@/lib/playground/json-schema";
|
||||
|
||||
function CopyButton({ text }: { text: string }) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
const copy = () => {
|
||||
navigator.clipboard.writeText(text).then(() => {
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 1500);
|
||||
});
|
||||
};
|
||||
return (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={copy}
|
||||
className="absolute right-2 top-2 h-7 gap-1.5 px-2 text-xs"
|
||||
>
|
||||
{copied ? <Check className="h-3.5 w-3.5" /> : <Copy className="h-3.5 w-3.5" />}
|
||||
{copied ? "Copied" : "Copy"}
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
|
||||
function CodeBlock({ code }: { code: string }) {
|
||||
return (
|
||||
<div className="relative">
|
||||
<CopyButton text={code} />
|
||||
<pre className="max-h-[420px] overflow-auto rounded-md border border-border/60 bg-muted/20 p-3 pr-20 text-xs">
|
||||
<code>{code}</code>
|
||||
</pre>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SchemaBlock({ title, schema }: { title: string; schema: Record<string, unknown> }) {
|
||||
const json = useMemo(() => JSON.stringify(schema, null, 2), [schema]);
|
||||
return (
|
||||
<details className="group rounded-md border border-border/60">
|
||||
<summary className="cursor-pointer select-none px-3 py-2 text-sm font-medium text-muted-foreground transition-colors hover:text-foreground">
|
||||
{title}
|
||||
</summary>
|
||||
<div className="relative border-t border-border/60">
|
||||
<CopyButton text={json} />
|
||||
<pre className="max-h-[360px] overflow-auto p-3 pr-20 text-xs">
|
||||
<code>{json}</code>
|
||||
</pre>
|
||||
</div>
|
||||
</details>
|
||||
);
|
||||
}
|
||||
|
||||
export function ApiReference({
|
||||
workspaceId,
|
||||
platform,
|
||||
verb,
|
||||
fields,
|
||||
inputSchema,
|
||||
outputSchema,
|
||||
}: {
|
||||
workspaceId: number;
|
||||
platform: string;
|
||||
verb: string;
|
||||
fields: FormField[];
|
||||
inputSchema: Record<string, unknown>;
|
||||
/** Absent only when talking to a backend that predates output schemas. */
|
||||
outputSchema?: Record<string, unknown>;
|
||||
}) {
|
||||
const path = `/api/v1/workspaces/${workspaceId}/scrapers/${platform}/${verb}`;
|
||||
|
||||
// In proxy mode BACKEND_URL is intentionally empty (same-origin), so external
|
||||
// callers use this app's origin. Client component, so window is available.
|
||||
const baseUrl = BACKEND_URL || (typeof window !== "undefined" ? window.location.origin : "");
|
||||
|
||||
const snippets = useMemo(() => {
|
||||
const payload = buildExamplePayload(fields);
|
||||
return buildSnippets(baseUrl, path, payload);
|
||||
}, [fields, baseUrl, path]);
|
||||
|
||||
return (
|
||||
<section className="space-y-4">
|
||||
<div>
|
||||
<h2 className="text-base font-semibold">API reference</h2>
|
||||
<p className="mt-1 text-sm text-muted-foreground">
|
||||
Call this API from your own project. Create a key under{" "}
|
||||
<span className="font-medium text-foreground">API Keys</span> (and enable API access for
|
||||
this workspace), then send it as a{" "}
|
||||
<code className="rounded bg-muted/40 px-1 py-0.5 text-xs">Authorization: Bearer</code>{" "}
|
||||
header.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<Tabs defaultValue="curl">
|
||||
<TabsList className="h-auto flex-wrap">
|
||||
{snippets.map((snippet) => (
|
||||
<TabsTrigger key={snippet.id} value={snippet.id}>
|
||||
{snippet.label}
|
||||
</TabsTrigger>
|
||||
))}
|
||||
</TabsList>
|
||||
{snippets.map((snippet) => (
|
||||
<TabsContent key={snippet.id} value={snippet.id}>
|
||||
<CodeBlock code={snippet.code} />
|
||||
</TabsContent>
|
||||
))}
|
||||
</Tabs>
|
||||
|
||||
<div className="space-y-2">
|
||||
<SchemaBlock title="Input schema (JSON)" schema={inputSchema} />
|
||||
{outputSchema && <SchemaBlock title="Output schema (JSON)" schema={outputSchema} />}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,183 @@
|
|||
"use client";
|
||||
|
||||
import { Check, Copy, Download } from "lucide-react";
|
||||
import { useMemo, useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
import { downloadCsv, rowsToCsv } from "@/lib/playground/csv";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
const MAX_TABLE_ROWS = 200;
|
||||
|
||||
function extractItems(data: unknown): Record<string, unknown>[] | null {
|
||||
if (
|
||||
typeof data === "object" &&
|
||||
data !== null &&
|
||||
"items" in data &&
|
||||
Array.isArray((data as { items: unknown }).items)
|
||||
) {
|
||||
const items = (data as { items: unknown[] }).items;
|
||||
if (items.every((item) => typeof item === "object" && item !== null)) {
|
||||
return items as Record<string, unknown>[];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function cellText(value: unknown): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
if (typeof value === "object") return JSON.stringify(value);
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function ResultTable({ items }: { items: Record<string, unknown>[] }) {
|
||||
const columns = useMemo(() => {
|
||||
const keys = new Set<string>();
|
||||
for (const item of items.slice(0, MAX_TABLE_ROWS)) {
|
||||
for (const key of Object.keys(item)) keys.add(key);
|
||||
}
|
||||
return Array.from(keys);
|
||||
}, [items]);
|
||||
|
||||
const rows = items.slice(0, MAX_TABLE_ROWS);
|
||||
|
||||
return (
|
||||
<div className="overflow-x-auto rounded-md border border-border/60">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
{columns.map((col) => (
|
||||
<TableHead key={col} className="whitespace-nowrap">
|
||||
{col}
|
||||
</TableHead>
|
||||
))}
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{rows.map((item, i) => (
|
||||
// biome-ignore lint/suspicious/noArrayIndexKey: rows have no stable id
|
||||
<TableRow key={i}>
|
||||
{columns.map((col) => (
|
||||
<TableCell key={col} className="max-w-xs truncate align-top text-xs">
|
||||
{cellText(item[col])}
|
||||
</TableCell>
|
||||
))}
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function OutputViewer({ data, filenameBase }: { data: unknown; filenameBase?: string }) {
|
||||
const items = useMemo(() => extractItems(data), [data]);
|
||||
const [view, setView] = useState<"table" | "json">(items ? "table" : "json");
|
||||
const [copied, setCopied] = useState(false);
|
||||
|
||||
const json = useMemo(() => JSON.stringify(data, null, 2), [data]);
|
||||
|
||||
const copy = () => {
|
||||
navigator.clipboard.writeText(json).then(() => {
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 1500);
|
||||
});
|
||||
};
|
||||
|
||||
// Export the full item set (not just the display-capped rows) as CSV, with a
|
||||
// column per union key so nothing is dropped.
|
||||
const exportCsv = () => {
|
||||
if (!items || items.length === 0) return;
|
||||
const columns = Array.from(
|
||||
items.reduce((set, item) => {
|
||||
for (const key of Object.keys(item)) set.add(key);
|
||||
return set;
|
||||
}, new Set<string>())
|
||||
);
|
||||
downloadCsv(filenameBase ?? "output", rowsToCsv(items, columns));
|
||||
};
|
||||
|
||||
const truncated = items && items.length > MAX_TABLE_ROWS;
|
||||
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="inline-flex rounded-md border border-border/60 p-0.5">
|
||||
{items && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setView("table")}
|
||||
className={cn(
|
||||
"rounded px-2.5 py-1 text-xs font-medium transition-colors",
|
||||
view === "table"
|
||||
? "bg-accent text-accent-foreground"
|
||||
: "text-muted-foreground hover:text-foreground"
|
||||
)}
|
||||
>
|
||||
Table
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setView("json")}
|
||||
className={cn(
|
||||
"rounded px-2.5 py-1 text-xs font-medium transition-colors",
|
||||
view === "json"
|
||||
? "bg-accent text-accent-foreground"
|
||||
: "text-muted-foreground hover:text-foreground"
|
||||
)}
|
||||
>
|
||||
JSON
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex items-center gap-1">
|
||||
{items && items.length > 0 && (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={exportCsv}
|
||||
className="gap-1.5"
|
||||
>
|
||||
<Download className="h-3.5 w-3.5" />
|
||||
Export CSV
|
||||
</Button>
|
||||
)}
|
||||
<Button type="button" variant="ghost" size="sm" onClick={copy} className="gap-1.5">
|
||||
{copied ? <Check className="h-3.5 w-3.5" /> : <Copy className="h-3.5 w-3.5" />}
|
||||
{copied ? "Copied" : "Copy JSON"}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{items && items.length === 0 && (
|
||||
<p className="rounded-md border border-dashed border-border/60 px-4 py-6 text-center text-sm text-muted-foreground">
|
||||
No items returned.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{view === "table" && items && items.length > 0 ? (
|
||||
<>
|
||||
<ResultTable items={items} />
|
||||
{truncated && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Showing first {MAX_TABLE_ROWS} of {items.length} items. Use Copy JSON for the full
|
||||
output.
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<pre className="max-h-[480px] overflow-auto rounded-md border border-border/60 bg-muted/20 p-3 text-xs">
|
||||
<code>{json}</code>
|
||||
</pre>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
"use client";
|
||||
|
||||
import { ArrowRight, History, KeyRound } from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { PLAYGROUND_PLATFORMS } from "@/lib/playground/catalog";
|
||||
|
||||
export function PlaygroundIndex({ workspaceId }: { workspaceId: number }) {
|
||||
const base = `/dashboard/${workspaceId}/playground`;
|
||||
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
<div>
|
||||
<h1 className="text-xl font-semibold">API Playground</h1>
|
||||
<p className="mt-1 text-sm text-muted-foreground">
|
||||
Manually run SurfSense's platform-native APIs and inspect their output. Every run is
|
||||
captured under Runs.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-2">
|
||||
<Link
|
||||
href={`${base}/runs`}
|
||||
className="flex items-center justify-between rounded-lg border border-border/60 bg-accent/40 px-4 py-3 transition-colors hover:bg-accent"
|
||||
>
|
||||
<div className="flex items-center gap-3">
|
||||
<History className="h-5 w-5 text-muted-foreground" />
|
||||
<div>
|
||||
<p className="text-sm font-medium">Runs</p>
|
||||
<p className="text-xs text-muted-foreground">See every API run in this workspace</p>
|
||||
</div>
|
||||
</div>
|
||||
<ArrowRight className="h-4 w-4 text-muted-foreground" />
|
||||
</Link>
|
||||
<Link
|
||||
href={`${base}/api-keys`}
|
||||
className="flex items-center justify-between rounded-lg border border-border/60 bg-accent/40 px-4 py-3 transition-colors hover:bg-accent"
|
||||
>
|
||||
<div className="flex items-center gap-3">
|
||||
<KeyRound className="h-5 w-5 text-muted-foreground" />
|
||||
<div>
|
||||
<p className="text-sm font-medium">API Keys</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Enable workspace access and manage keys
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<ArrowRight className="h-4 w-4 text-muted-foreground" />
|
||||
</Link>
|
||||
</div>
|
||||
|
||||
<div className="space-y-6">
|
||||
{PLAYGROUND_PLATFORMS.map((platform) => (
|
||||
<div key={platform.id} className="space-y-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<platform.icon className="h-4 w-4 text-muted-foreground" />
|
||||
<h2 className="text-sm font-semibold">{platform.label}</h2>
|
||||
</div>
|
||||
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-3">
|
||||
{platform.verbs.map((verb) => (
|
||||
<Link
|
||||
key={verb.name}
|
||||
href={`${base}/${platform.id}/${verb.verb}`}
|
||||
className="group flex flex-col rounded-lg border border-border/60 p-4 transition-colors hover:border-border hover:bg-muted/30"
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-sm font-medium">{verb.label}</span>
|
||||
<ArrowRight className="h-4 w-4 text-muted-foreground opacity-0 transition-opacity group-hover:opacity-100" />
|
||||
</div>
|
||||
<code className="mt-2 text-xs text-muted-foreground">{verb.name}</code>
|
||||
</Link>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,277 @@
|
|||
"use client";
|
||||
|
||||
import { AlertTriangle, Coins, Hash, Loader2, Play, Timer, X } from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Spinner } from "@/components/ui/spinner";
|
||||
import { useRunStream } from "@/hooks/use-run-stream";
|
||||
import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities";
|
||||
import { scrapersApiService } from "@/lib/apis/scrapers-api.service";
|
||||
import { AbortedError, AppError } from "@/lib/error";
|
||||
import { findVerb } from "@/lib/playground/catalog";
|
||||
import { formatCost, formatDuration } from "@/lib/playground/format";
|
||||
import { buildPayload, initialFormValues, parseSchemaFields } from "@/lib/playground/json-schema";
|
||||
import { ApiReference } from "./api-reference";
|
||||
import { OutputViewer } from "./output-viewer";
|
||||
import { RunProgressPanel } from "./run-progress-panel";
|
||||
import { SchemaForm } from "./schema-form";
|
||||
|
||||
interface PlaygroundRunnerProps {
|
||||
workspaceId: number;
|
||||
platform: string;
|
||||
verb: string;
|
||||
}
|
||||
|
||||
const MAX_OUTPUT_LINES = 200;
|
||||
|
||||
/** Parse the stored JSONL output into objects, capped for rendering. */
|
||||
function parseJsonlOutput(text: string | null | undefined): { items: unknown[] } {
|
||||
if (!text) return { items: [] };
|
||||
const lines = text.split("\n").filter((line) => line.trim().length > 0);
|
||||
const items: unknown[] = [];
|
||||
for (const line of lines.slice(0, MAX_OUTPUT_LINES)) {
|
||||
try {
|
||||
items.push(JSON.parse(line));
|
||||
} catch {
|
||||
items.push(line);
|
||||
}
|
||||
}
|
||||
return { items };
|
||||
}
|
||||
|
||||
function RunStat({
|
||||
icon: Icon,
|
||||
label,
|
||||
value,
|
||||
}: {
|
||||
icon: typeof Hash;
|
||||
label: string;
|
||||
value: string;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex items-center gap-1.5 rounded-md border border-border/60 px-2.5 py-1.5">
|
||||
<Icon className="h-3.5 w-3.5 text-muted-foreground" />
|
||||
<span className="text-xs text-muted-foreground">{label}</span>
|
||||
<span className="text-xs font-medium tabular-nums">{value}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ErrorPanel({ error, workspaceId }: { error: unknown; workspaceId: number }) {
|
||||
if (error instanceof AbortedError) {
|
||||
return null;
|
||||
}
|
||||
const status = error instanceof AppError ? error.status : undefined;
|
||||
|
||||
if (status === 402) {
|
||||
return (
|
||||
<div className="rounded-md border border-destructive/40 bg-destructive/5 p-4 text-sm">
|
||||
<p className="font-medium text-destructive">Insufficient credits</p>
|
||||
<p className="mt-1 text-muted-foreground">You don't have enough credits to run this API.</p>
|
||||
<Button asChild size="sm" variant="outline" className="mt-3">
|
||||
<Link href={`/dashboard/${workspaceId}/buy-more`}>Buy credits</Link>
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const message =
|
||||
status === 422
|
||||
? "Invalid input. Check the fields above and try again."
|
||||
: error instanceof Error && error.message
|
||||
? error.message
|
||||
: "Something went wrong running this API.";
|
||||
|
||||
return (
|
||||
<div className="flex items-start gap-2 rounded-md border border-destructive/40 bg-destructive/5 p-4 text-sm text-destructive">
|
||||
<AlertTriangle className="mt-0.5 h-4 w-4 shrink-0" />
|
||||
<span>{message}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunnerProps) {
|
||||
const catalogVerb = findVerb(platform, verb);
|
||||
const {
|
||||
data: capabilities,
|
||||
isLoading,
|
||||
error: capabilitiesError,
|
||||
} = useScraperCapabilities(workspaceId);
|
||||
|
||||
const capability = useMemo(() => {
|
||||
if (!capabilities) return undefined;
|
||||
const name = catalogVerb?.name ?? `${platform}.${verb}`;
|
||||
return capabilities.find((c) => c.name === name);
|
||||
}, [capabilities, catalogVerb, platform, verb]);
|
||||
|
||||
const fields = useMemo(() => parseSchemaFields(capability?.input_schema), [capability]);
|
||||
|
||||
const [values, setValues] = useState<Record<string, unknown>>({});
|
||||
const run = useRunStream(workspaceId);
|
||||
const isRunning = run.status === "running";
|
||||
|
||||
// Seed form defaults once the schema is available.
|
||||
useEffect(() => {
|
||||
if (fields.length > 0) {
|
||||
setValues(initialFormValues(fields));
|
||||
}
|
||||
}, [fields]);
|
||||
|
||||
// Survive a refresh: if this verb's newest run is still ``running``, re-attach
|
||||
// to its live stream instead of showing an idle form.
|
||||
const reattachedRef = useRef(false);
|
||||
const { reattach } = run;
|
||||
useEffect(() => {
|
||||
if (reattachedRef.current || !capability) return;
|
||||
reattachedRef.current = true;
|
||||
let cancelled = false;
|
||||
(async () => {
|
||||
try {
|
||||
const recent = await scrapersApiService.listRuns(workspaceId, {
|
||||
capability: capability.name,
|
||||
limit: 1,
|
||||
});
|
||||
const top = recent[0];
|
||||
if (!cancelled && top && top.status === "running") {
|
||||
reattach(top.id);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort reattach; a fresh run still works.
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [capability, workspaceId, reattach]);
|
||||
|
||||
const handleChange = (name: string, value: unknown) => {
|
||||
setValues((prev) => ({ ...prev, [name]: value }));
|
||||
};
|
||||
|
||||
const handleRun = useCallback(() => {
|
||||
const payload = buildPayload(fields, values);
|
||||
void run.start(platform, verb, payload);
|
||||
}, [fields, values, platform, verb, run]);
|
||||
|
||||
const output = useMemo(
|
||||
() => (run.detail ? parseJsonlOutput(run.detail.output_text) : null),
|
||||
[run.detail]
|
||||
);
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex h-64 items-center justify-center">
|
||||
<Spinner size="lg" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (capabilitiesError) {
|
||||
return (
|
||||
<div className="rounded-md border border-destructive/40 bg-destructive/5 p-4 text-sm text-destructive">
|
||||
Couldn't load API definitions
|
||||
{capabilitiesError.message ? `: ${capabilitiesError.message}` : "."}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!capability || !catalogVerb) {
|
||||
return (
|
||||
<div className="rounded-md border border-dashed border-border/60 px-4 py-8 text-center text-sm text-muted-foreground">
|
||||
Unknown API: {platform}.{verb}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-10">
|
||||
<div className="grid gap-6 lg:grid-cols-2">
|
||||
<div className="space-y-5">
|
||||
<div>
|
||||
<h1 className="text-lg font-semibold">
|
||||
{catalogVerb.label} <span className="text-muted-foreground">· {platform}</span>
|
||||
</h1>
|
||||
{capability.description && (
|
||||
<p className="mt-1 text-sm text-muted-foreground">{capability.description}</p>
|
||||
)}
|
||||
<code className="mt-2 inline-block rounded bg-muted/40 px-1.5 py-0.5 text-xs text-muted-foreground">
|
||||
POST /workspaces/{workspaceId}/scrapers/{platform}/{verb}
|
||||
</code>
|
||||
</div>
|
||||
|
||||
<SchemaForm
|
||||
fields={fields}
|
||||
values={values}
|
||||
onChange={handleChange}
|
||||
disabled={isRunning}
|
||||
/>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<Button type="button" onClick={handleRun} disabled={isRunning} className="gap-1.5">
|
||||
{isRunning ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
) : (
|
||||
<Play className="h-4 w-4" />
|
||||
)}
|
||||
Run
|
||||
</Button>
|
||||
{isRunning && (
|
||||
<Button type="button" variant="outline" onClick={run.cancel} className="gap-1.5">
|
||||
<X className="h-4 w-4" />
|
||||
Cancel
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{run.status === "error" && <ErrorPanel error={run.error} workspaceId={workspaceId} />}
|
||||
</div>
|
||||
|
||||
<div className="space-y-3">
|
||||
<h2 className="text-sm font-medium text-muted-foreground">Output</h2>
|
||||
{isRunning ? (
|
||||
<RunProgressPanel
|
||||
latest={run.latest}
|
||||
events={run.events}
|
||||
elapsedMs={run.elapsedMs}
|
||||
/>
|
||||
) : run.status === "cancelled" ? (
|
||||
<div className="flex h-64 items-center justify-center rounded-md border border-border/60 px-4 text-center text-sm text-muted-foreground">
|
||||
Run cancelled.
|
||||
</div>
|
||||
) : run.status === "success" && output ? (
|
||||
<>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<RunStat
|
||||
icon={Hash}
|
||||
label="Items"
|
||||
value={String(run.detail?.item_count ?? output.items.length)}
|
||||
/>
|
||||
<RunStat
|
||||
icon={Timer}
|
||||
label="Time"
|
||||
value={formatDuration(run.detail?.duration_ms ?? run.elapsedMs)}
|
||||
/>
|
||||
<RunStat icon={Coins} label="Cost" value={formatCost(run.detail?.cost_micros)} />
|
||||
</div>
|
||||
<OutputViewer data={output} filenameBase={`${platform}-${verb}`} />
|
||||
</>
|
||||
) : (
|
||||
<div className="flex h-64 items-center justify-center rounded-md border border-dashed border-border/60 px-4 text-center text-sm text-muted-foreground">
|
||||
Run the API to see output here.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ApiReference
|
||||
workspaceId={workspaceId}
|
||||
platform={platform}
|
||||
verb={verb}
|
||||
fields={fields}
|
||||
inputSchema={capability.input_schema}
|
||||
outputSchema={capability.output_schema}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
"use client";
|
||||
|
||||
import { useMemo } from "react";
|
||||
import { Spinner } from "@/components/ui/spinner";
|
||||
import { useScraperRun } from "@/hooks/use-scraper-runs";
|
||||
import { OutputViewer } from "./output-viewer";
|
||||
|
||||
const MAX_OUTPUT_LINES = 200;
|
||||
|
||||
/** One-line label for a persisted ``run.progress`` event object. */
|
||||
function progressLine(event: Record<string, unknown>): string {
|
||||
const message = typeof event.message === "string" ? event.message : undefined;
|
||||
const phase = typeof event.phase === "string" ? event.phase : undefined;
|
||||
const current = typeof event.current === "number" ? event.current : undefined;
|
||||
const total = typeof event.total === "number" ? event.total : undefined;
|
||||
const base = message || (phase ? phase.replace(/_/g, " ") : "step");
|
||||
if (current !== undefined) {
|
||||
return `${base} (${total !== undefined ? `${current}/${total}` : current})`;
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
/** Parse the stored JSONL output into objects, capped for rendering. */
|
||||
function parseJsonl(text: string | null): { items: unknown[]; total: number } {
|
||||
if (!text) return { items: [], total: 0 };
|
||||
const lines = text.split("\n").filter((line) => line.trim().length > 0);
|
||||
const items: unknown[] = [];
|
||||
for (const line of lines.slice(0, MAX_OUTPUT_LINES)) {
|
||||
try {
|
||||
items.push(JSON.parse(line));
|
||||
} catch {
|
||||
items.push(line);
|
||||
}
|
||||
}
|
||||
return { items, total: lines.length };
|
||||
}
|
||||
|
||||
export function RunDetail({
|
||||
workspaceId,
|
||||
runId,
|
||||
}: {
|
||||
workspaceId: number;
|
||||
runId: string;
|
||||
}) {
|
||||
const { data: run, isLoading, error } = useScraperRun(workspaceId, runId);
|
||||
|
||||
const parsed = useMemo(() => parseJsonl(run?.output_text ?? null), [run?.output_text]);
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex h-32 items-center justify-center">
|
||||
<Spinner size="md" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<p className="p-4 text-sm text-destructive">
|
||||
Couldn't load run{error.message ? `: ${error.message}` : "."}
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
if (!run) return null;
|
||||
|
||||
return (
|
||||
<div className="space-y-4 border-t border-border/60 bg-muted/10 p-4">
|
||||
{run.error && (
|
||||
<div className="rounded-md border border-destructive/40 bg-destructive/5 p-3 text-sm text-destructive">
|
||||
{run.error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{run.progress && run.progress.length > 0 && (
|
||||
<div>
|
||||
<h4 className="mb-1.5 text-xs font-medium text-muted-foreground">Progress</h4>
|
||||
<div className="max-h-48 space-y-1 overflow-y-auto rounded-md border border-border/60 bg-background p-3 font-mono text-xs text-muted-foreground">
|
||||
{run.progress.map((event, i) => (
|
||||
<div key={i} className="truncate">
|
||||
{progressLine(event)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<h4 className="mb-1.5 text-xs font-medium text-muted-foreground">Input</h4>
|
||||
<pre className="max-h-64 overflow-auto rounded-md border border-border/60 bg-background p-3 text-xs">
|
||||
<code>{JSON.stringify(run.input ?? {}, null, 2)}</code>
|
||||
</pre>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h4 className="mb-1.5 text-xs font-medium text-muted-foreground">
|
||||
Output {parsed.total > 0 && `(${parsed.total} items)`}
|
||||
</h4>
|
||||
{run.output_text ? (
|
||||
<>
|
||||
<OutputViewer
|
||||
data={{ items: parsed.items }}
|
||||
filenameBase={`${run.capability}-${run.id}`}
|
||||
/>
|
||||
{parsed.total > MAX_OUTPUT_LINES && (
|
||||
<p className="mt-2 text-xs text-muted-foreground">
|
||||
Showing first {MAX_OUTPUT_LINES} of {parsed.total} stored items.
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">No output stored for this run.</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
"use client";
|
||||
|
||||
import { Loader2 } from "lucide-react";
|
||||
import { useEffect, useRef } from "react";
|
||||
import type { ScraperRunEvent } from "@/contracts/types/scraper.types";
|
||||
import { formatDuration } from "@/lib/playground/format";
|
||||
|
||||
/** One-line human label for a progress/lifecycle event. */
|
||||
function eventLabel(event: ScraperRunEvent): string {
|
||||
const base =
|
||||
event.message ||
|
||||
(event.phase ? event.phase.replace(/_/g, " ").replace(/^\w/, (c) => c.toUpperCase()) : "Working");
|
||||
if (event.current !== undefined && event.current !== null) {
|
||||
const counter =
|
||||
event.total !== undefined && event.total !== null
|
||||
? `${event.current}/${event.total}`
|
||||
: String(event.current);
|
||||
return `${base} (${counter})`;
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
/**
|
||||
* Live progress for an in-flight async run: a headline status, a determinate
|
||||
* bar when counts are known, a running elapsed timer, and a scrolling event log.
|
||||
*/
|
||||
export function RunProgressPanel({
|
||||
latest,
|
||||
events,
|
||||
elapsedMs,
|
||||
}: {
|
||||
latest: ScraperRunEvent | null;
|
||||
events: ScraperRunEvent[];
|
||||
elapsedMs: number;
|
||||
}) {
|
||||
const logRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Keep the newest line in view as the log grows.
|
||||
useEffect(() => {
|
||||
const el = logRef.current;
|
||||
if (el) el.scrollTop = el.scrollHeight;
|
||||
}, [events.length]);
|
||||
|
||||
const total = latest?.total ?? undefined;
|
||||
const current = latest?.current ?? undefined;
|
||||
const pct =
|
||||
typeof current === "number" && typeof total === "number" && total > 0
|
||||
? Math.min(100, Math.round((current / total) * 100))
|
||||
: null;
|
||||
|
||||
return (
|
||||
<div className="space-y-3 rounded-md border border-border/60 p-4">
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<Loader2 className="h-4 w-4 shrink-0 animate-spin text-muted-foreground" />
|
||||
<span className="truncate text-sm font-medium">
|
||||
{latest ? eventLabel(latest) : "Starting…"}
|
||||
</span>
|
||||
</div>
|
||||
<span className="shrink-0 text-xs tabular-nums text-muted-foreground">
|
||||
{formatDuration(elapsedMs)}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{pct !== null && (
|
||||
<div className="h-1.5 w-full overflow-hidden rounded-full bg-muted">
|
||||
<div
|
||||
className="h-full rounded-full bg-primary transition-all duration-300"
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{events.length > 0 && (
|
||||
<div
|
||||
ref={logRef}
|
||||
className="max-h-56 space-y-1 overflow-y-auto rounded border border-border/40 bg-muted/20 p-2 font-mono text-xs text-muted-foreground"
|
||||
>
|
||||
{events.map((event, i) => (
|
||||
<div key={`${event.ts ?? i}-${i}`} className="truncate">
|
||||
{eventLabel(event)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
import { Loader2 } from "lucide-react";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
|
||||
/** Scraper runs: ``running`` (async, in-flight), ``success``, ``error``, ``cancelled``. */
|
||||
export function RunStatusBadge({ status }: { status: string }) {
|
||||
const normalized = status.toLowerCase();
|
||||
if (normalized === "running") {
|
||||
return (
|
||||
<Badge variant="secondary" className="gap-1 bg-blue-500/15 text-blue-600 dark:text-blue-400">
|
||||
<Loader2 className="h-3 w-3 animate-spin" />
|
||||
Running
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
if (normalized === "success") {
|
||||
return (
|
||||
<Badge variant="secondary" className="bg-emerald-500/15 text-emerald-600 dark:text-emerald-400">
|
||||
Success
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
if (normalized === "error") {
|
||||
return <Badge variant="destructive">Error</Badge>;
|
||||
}
|
||||
if (normalized === "cancelled") {
|
||||
return (
|
||||
<Badge variant="secondary" className="bg-amber-500/15 text-amber-600 dark:text-amber-400">
|
||||
Cancelled
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
return <Badge variant="outline">{status}</Badge>;
|
||||
}
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
"use client";
|
||||
|
||||
import { useInfiniteQuery } from "@tanstack/react-query";
|
||||
import { ChevronDown, ChevronRight, History } from "lucide-react";
|
||||
import { Fragment, useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Spinner } from "@/components/ui/spinner";
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
import { scrapersApiService } from "@/lib/apis/scrapers-api.service";
|
||||
import { formatRelativeDate } from "@/lib/format-date";
|
||||
import { PLAYGROUND_PLATFORMS } from "@/lib/playground/catalog";
|
||||
import { formatCost, formatDuration } from "@/lib/playground/format";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { RunDetail } from "./run-detail";
|
||||
import { RunStatusBadge } from "./run-status-badge";
|
||||
|
||||
const PAGE_SIZE = 25;
|
||||
const ALL = "__all__";
|
||||
|
||||
const CAPABILITY_OPTIONS = PLAYGROUND_PLATFORMS.flatMap((platform) =>
|
||||
platform.verbs.map((verb) => verb.name)
|
||||
);
|
||||
|
||||
export function RunsTable({ workspaceId }: { workspaceId: number }) {
|
||||
const [capability, setCapability] = useState<string>(ALL);
|
||||
const [status, setStatus] = useState<string>(ALL);
|
||||
const [expanded, setExpanded] = useState<string | null>(null);
|
||||
|
||||
const filters = {
|
||||
capability: capability === ALL ? undefined : capability,
|
||||
status: status === ALL ? undefined : status,
|
||||
};
|
||||
|
||||
const query = useInfiniteQuery({
|
||||
queryKey: [...cacheKeys.scrapers.runs(workspaceId), filters],
|
||||
queryFn: ({ pageParam }) =>
|
||||
scrapersApiService.listRuns(workspaceId, {
|
||||
limit: PAGE_SIZE,
|
||||
offset: pageParam,
|
||||
...filters,
|
||||
}),
|
||||
initialPageParam: 0,
|
||||
getNextPageParam: (lastPage, allPages) =>
|
||||
lastPage.length === PAGE_SIZE ? allPages.length * PAGE_SIZE : undefined,
|
||||
// Poll while any visible run is still in flight so it flips to a terminal
|
||||
// state without a manual refresh; idle otherwise.
|
||||
refetchInterval: (q) =>
|
||||
q.state.data?.pages.flat().some((r) => r.status === "running") ? 5000 : false,
|
||||
});
|
||||
|
||||
const runs = query.data?.pages.flat() ?? [];
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h1 className="text-xl font-semibold">Runs</h1>
|
||||
<p className="mt-1 text-sm text-muted-foreground">
|
||||
Every platform-native API run in this workspace — from the playground, API keys, and
|
||||
agents. Newest first.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Select value={capability} onValueChange={setCapability}>
|
||||
<SelectTrigger className="w-48">
|
||||
<SelectValue placeholder="All APIs" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value={ALL}>All APIs</SelectItem>
|
||||
{CAPABILITY_OPTIONS.map((name) => (
|
||||
<SelectItem key={name} value={name}>
|
||||
{name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Select value={status} onValueChange={setStatus}>
|
||||
<SelectTrigger className="w-40">
|
||||
<SelectValue placeholder="All statuses" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value={ALL}>All statuses</SelectItem>
|
||||
<SelectItem value="running">Running</SelectItem>
|
||||
<SelectItem value="success">Success</SelectItem>
|
||||
<SelectItem value="error">Error</SelectItem>
|
||||
<SelectItem value="cancelled">Cancelled</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
|
||||
{query.isLoading ? (
|
||||
<div className="flex h-48 items-center justify-center">
|
||||
<Spinner size="lg" />
|
||||
</div>
|
||||
) : query.isError ? (
|
||||
<p className="text-sm text-destructive">
|
||||
Couldn't load runs{query.error.message ? `: ${query.error.message}` : "."}
|
||||
</p>
|
||||
) : runs.length === 0 ? (
|
||||
<div className="rounded-md border border-dashed border-border/60 bg-muted/20 px-4 py-12 text-center">
|
||||
<History className="mx-auto h-8 w-8 text-muted-foreground" aria-hidden />
|
||||
<p className="mt-2 text-sm font-medium">No runs yet</p>
|
||||
<p className="mt-1 text-xs text-muted-foreground">
|
||||
Run an API from the playground and it will show up here.
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
<div className="overflow-hidden rounded-md border border-border/60">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead className="w-8" />
|
||||
<TableHead>API</TableHead>
|
||||
<TableHead>Origin</TableHead>
|
||||
<TableHead>Status</TableHead>
|
||||
<TableHead className="text-right">Items</TableHead>
|
||||
<TableHead className="text-right">Duration</TableHead>
|
||||
<TableHead className="text-right">Cost</TableHead>
|
||||
<TableHead className="text-right">When</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{runs.map((run) => {
|
||||
const isOpen = expanded === run.id;
|
||||
return (
|
||||
<Fragment key={run.id}>
|
||||
<TableRow
|
||||
className="cursor-pointer"
|
||||
onClick={() => setExpanded(isOpen ? null : run.id)}
|
||||
>
|
||||
<TableCell>
|
||||
{isOpen ? (
|
||||
<ChevronDown className="h-4 w-4 text-muted-foreground" />
|
||||
) : (
|
||||
<ChevronRight className="h-4 w-4 text-muted-foreground" />
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell className="font-mono text-xs">{run.capability}</TableCell>
|
||||
<TableCell className="text-xs text-muted-foreground">{run.origin}</TableCell>
|
||||
<TableCell>
|
||||
<RunStatusBadge status={run.status} />
|
||||
</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{run.item_count}</TableCell>
|
||||
<TableCell className="text-right tabular-nums text-muted-foreground">
|
||||
{formatDuration(run.duration_ms)}
|
||||
</TableCell>
|
||||
<TableCell className="text-right tabular-nums text-muted-foreground">
|
||||
{formatCost(run.cost_micros)}
|
||||
</TableCell>
|
||||
<TableCell className="text-right text-xs text-muted-foreground">
|
||||
{formatRelativeDate(run.created_at)}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
{isOpen && (
|
||||
<TableRow className="hover:bg-transparent">
|
||||
<TableCell colSpan={8} className="p-0">
|
||||
<RunDetail workspaceId={workspaceId} runId={run.id} />
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
)}
|
||||
</Fragment>
|
||||
);
|
||||
})}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{query.hasNextPage && (
|
||||
<div className="flex justify-center">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
onClick={() => query.fetchNextPage()}
|
||||
disabled={query.isFetchingNextPage}
|
||||
className={cn(query.isFetchingNextPage && "opacity-70")}
|
||||
>
|
||||
{query.isFetchingNextPage ? "Loading…" : "Load more"}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,215 @@
|
|||
"use client";
|
||||
|
||||
import { ChevronDown } from "lucide-react";
|
||||
import { useMemo, useState } from "react";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import type { FormField } from "@/lib/playground/json-schema";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
interface SchemaFormProps {
|
||||
fields: FormField[];
|
||||
values: Record<string, unknown>;
|
||||
onChange: (name: string, value: unknown) => void;
|
||||
disabled?: boolean;
|
||||
/** Field names flagged by a 422 response, shown with error styling. */
|
||||
fieldErrors?: Record<string, string>;
|
||||
}
|
||||
|
||||
function FieldControl({
|
||||
field,
|
||||
value,
|
||||
onChange,
|
||||
disabled,
|
||||
invalid,
|
||||
}: {
|
||||
field: FormField;
|
||||
value: unknown;
|
||||
onChange: (value: unknown) => void;
|
||||
disabled?: boolean;
|
||||
invalid?: boolean;
|
||||
}) {
|
||||
const id = `field-${field.name}`;
|
||||
|
||||
if (field.kind === "boolean") {
|
||||
return (
|
||||
<Switch
|
||||
id={id}
|
||||
checked={Boolean(value)}
|
||||
onCheckedChange={onChange}
|
||||
disabled={disabled}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (field.kind === "enum" && field.enumValues) {
|
||||
return (
|
||||
<Select
|
||||
value={value ? String(value) : undefined}
|
||||
onValueChange={onChange}
|
||||
disabled={disabled}
|
||||
>
|
||||
<SelectTrigger id={id} className={cn("w-full", invalid && "border-destructive")}>
|
||||
<SelectValue placeholder="Select…" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{field.enumValues.map((option) => (
|
||||
<SelectItem key={option} value={option}>
|
||||
{option}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
);
|
||||
}
|
||||
|
||||
if (field.kind === "string_array") {
|
||||
return (
|
||||
<Textarea
|
||||
id={id}
|
||||
value={String(value ?? "")}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
placeholder="One value per line"
|
||||
disabled={disabled}
|
||||
rows={4}
|
||||
className={cn("font-mono text-xs", invalid && "border-destructive")}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (field.kind === "integer" || field.kind === "number") {
|
||||
return (
|
||||
<Input
|
||||
id={id}
|
||||
type="number"
|
||||
value={value === undefined || value === null ? "" : String(value)}
|
||||
min={field.minimum}
|
||||
max={field.maximum}
|
||||
step={field.kind === "integer" ? 1 : "any"}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
disabled={disabled}
|
||||
className={cn(invalid && "border-destructive")}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Input
|
||||
id={id}
|
||||
type="text"
|
||||
value={String(value ?? "")}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
disabled={disabled}
|
||||
className={cn(invalid && "border-destructive")}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function FieldRow({
|
||||
field,
|
||||
value,
|
||||
onChange,
|
||||
disabled,
|
||||
error,
|
||||
}: {
|
||||
field: FormField;
|
||||
value: unknown;
|
||||
onChange: (value: unknown) => void;
|
||||
disabled?: boolean;
|
||||
error?: string;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-1.5">
|
||||
<div className="flex items-center gap-2">
|
||||
<Label htmlFor={`field-${field.name}`} className="text-sm font-medium">
|
||||
{field.title}
|
||||
</Label>
|
||||
{field.required && <span className="text-xs text-destructive">required</span>}
|
||||
</div>
|
||||
{field.description && (
|
||||
<p className="text-xs text-muted-foreground">{field.description}</p>
|
||||
)}
|
||||
<FieldControl
|
||||
field={field}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
disabled={disabled}
|
||||
invalid={!!error}
|
||||
/>
|
||||
{error && <p className="text-xs text-destructive">{error}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function SchemaForm({
|
||||
fields,
|
||||
values,
|
||||
onChange,
|
||||
disabled,
|
||||
fieldErrors,
|
||||
}: SchemaFormProps) {
|
||||
const [showAdvanced, setShowAdvanced] = useState(false);
|
||||
|
||||
const { primary, advanced } = useMemo(() => {
|
||||
const primaryFields = fields.filter((f) => f.required);
|
||||
const advancedFields = fields.filter((f) => !f.required);
|
||||
return { primary: primaryFields, advanced: advancedFields };
|
||||
}, [fields]);
|
||||
|
||||
return (
|
||||
<div className="space-y-5">
|
||||
{primary.map((field) => (
|
||||
<FieldRow
|
||||
key={field.name}
|
||||
field={field}
|
||||
value={values[field.name]}
|
||||
onChange={(value) => onChange(field.name, value)}
|
||||
disabled={disabled}
|
||||
error={fieldErrors?.[field.name]}
|
||||
/>
|
||||
))}
|
||||
|
||||
{advanced.length > 0 && (
|
||||
<div className="rounded-md border border-border/60">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowAdvanced((prev) => !prev)}
|
||||
className="flex w-full items-center justify-between px-4 py-2.5 text-sm font-medium hover:bg-muted/30 transition-colors"
|
||||
aria-expanded={showAdvanced}
|
||||
>
|
||||
<span>Advanced ({advanced.length})</span>
|
||||
<ChevronDown
|
||||
className={cn(
|
||||
"h-4 w-4 text-muted-foreground transition-transform",
|
||||
showAdvanced && "rotate-180"
|
||||
)}
|
||||
/>
|
||||
</button>
|
||||
{showAdvanced && (
|
||||
<div className="space-y-5 border-t border-border/60 px-4 py-4">
|
||||
{advanced.map((field) => (
|
||||
<FieldRow
|
||||
key={field.name}
|
||||
field={field}
|
||||
value={values[field.name]}
|
||||
onChange={(value) => onChange(field.name, value)}
|
||||
disabled={disabled}
|
||||
error={fieldErrors?.[field.name]}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import { PlaygroundIndex } from "./components/playground-index";
|
||||
|
||||
export default async function PlaygroundPage({
|
||||
params,
|
||||
}: {
|
||||
params: Promise<{ workspace_id: string }>;
|
||||
}) {
|
||||
const { workspace_id } = await params;
|
||||
|
||||
return (
|
||||
<div className="mx-auto w-full max-w-5xl">
|
||||
<PlaygroundIndex workspaceId={Number(workspace_id)} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import { RunsTable } from "../components/runs-table";
|
||||
|
||||
export default async function PlaygroundRunsPage({
|
||||
params,
|
||||
}: {
|
||||
params: Promise<{ workspace_id: string }>;
|
||||
}) {
|
||||
const { workspace_id } = await params;
|
||||
|
||||
return (
|
||||
<div className="mx-auto w-full max-w-6xl">
|
||||
<RunsTable workspaceId={Number(workspace_id)} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -41,7 +41,13 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
|
|||
{ url: `${BASE_URL}/`, lastModified, changeFrequency: "daily", priority: 1 },
|
||||
{ url: `${BASE_URL}/free`, lastModified, changeFrequency: "daily", priority: 0.95 },
|
||||
{ url: `${BASE_URL}/connectors`, lastModified, changeFrequency: "weekly", priority: 0.9 },
|
||||
{ url: `${BASE_URL}/mcp-connector`, lastModified, changeFrequency: "weekly", priority: 0.85 },
|
||||
{ url: `${BASE_URL}/mcp-server`, lastModified, changeFrequency: "weekly", priority: 0.85 },
|
||||
{
|
||||
url: `${BASE_URL}/external-mcp-connectors`,
|
||||
lastModified,
|
||||
changeFrequency: "weekly",
|
||||
priority: 0.85,
|
||||
},
|
||||
{ url: `${BASE_URL}/pricing`, lastModified, changeFrequency: "weekly", priority: 0.9 },
|
||||
{ url: `${BASE_URL}/contact`, lastModified, changeFrequency: "monthly", priority: 0.7 },
|
||||
{ url: `${BASE_URL}/blog`, lastModified, changeFrequency: "daily", priority: 0.9 },
|
||||
|
|
|
|||
10
surfsense_web/atoms/layout/playground.atom.ts
Normal file
10
surfsense_web/atoms/layout/playground.atom.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { atom } from "jotai";
|
||||
|
||||
/**
|
||||
* Whether the second-level API Playground sidebar is open. Toggled by the
|
||||
* Playground nav button and kept in memory for the session, so it survives
|
||||
* in-app navigation (opening a new chat won't close it) and only closes when
|
||||
* the user clicks the toggle. It defaults to open, so a fresh app load — a new
|
||||
* signup or a relogin — always starts with the playground visible.
|
||||
*/
|
||||
export const playgroundSidebarOpenAtom = atom<boolean>(true);
|
||||
|
|
@ -19,7 +19,7 @@ So we asked what these agents still lack. Not reasoning. Not tools. **Live marke
|
|||
### What SurfSense is now
|
||||
|
||||
- **Platform-native connectors for live market data.** [Reddit](/reddit), [YouTube](/youtube), [Google Maps](/google-maps), [Google Search](/google-search), and the [open web](/web-crawl). Each one is a typed REST endpoint you can call from any language with your SurfSense API key.
|
||||
- **An MCP server that makes every connector an agent tool.** Add SurfSense to Claude, Cursor, or your own agent framework. And with the [MCP connector](/mcp-connector), your SurfSense agents can use any MCP server too, with one-click OAuth for apps like Notion, Slack, and Jira.
|
||||
- **An MCP server that makes every connector an agent tool.** Add the [SurfSense MCP server](/mcp-server) to Claude, Cursor, or your own agent framework. And with [external MCP connectors](/external-mcp-connectors), your SurfSense agents can use any MCP server too, with one-click OAuth for apps like Notion, Slack, and Jira.
|
||||
- **An agent harness, not just raw data.** Retries, structured output, and credit metering are built in, so agents go from a question to a brief without you building the plumbing.
|
||||
- **Automations for the workflows that matter.** Scheduled and event-triggered agents watch competitor pricing, track brand mentions, monitor rankings and reviews, then deliver briefs and alerts.
|
||||
- **Simpler pricing.** Self-hosting stays free with no metering. Cloud is pay as you go: start with $5 of free credit, connectors bill per item returned, crawls per page fetched, models at provider cost. Failed calls are never billed. Details on the [pricing page](/pricing).
|
||||
|
|
|
|||
|
|
@ -116,7 +116,8 @@ import {
|
|||
promoteRecentMention,
|
||||
} from "../new-chat/document-mention-picker";
|
||||
|
||||
const COMPOSER_PLACEHOLDER = "Ask anything, type / for prompts, type @ to mention docs";
|
||||
const COMPOSER_PLACEHOLDER =
|
||||
"Track competitors, scrape platforms, automate briefs — / for prompts, @ for docs";
|
||||
|
||||
type ComposerSuggestionAnchorPoint = {
|
||||
left: number;
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ export type HeroChatDemoScript = {
|
|||
|
||||
type Stage = "typing" | "steps" | "answer" | "done";
|
||||
|
||||
const PLACEHOLDER = "Ask anything, type / for prompts, type @ to mention docs";
|
||||
const PLACEHOLDER = "Track competitors, scrape platforms, automate briefs — / for prompts, @ for docs";
|
||||
|
||||
/** Blinking caret for the typewriter (overlay only, never inside the real input). */
|
||||
function Caret() {
|
||||
|
|
|
|||
|
|
@ -66,6 +66,144 @@ const HERO_REALTIME = "/homepage/hero_realtime";
|
|||
* `src`; everything else plays the chat demo.
|
||||
*/
|
||||
const CATEGORIES: HeroCategory[] = [
|
||||
{
|
||||
id: "connector-workflows",
|
||||
label: "Multi-Connector Workflows",
|
||||
useCases: [
|
||||
{
|
||||
id: "launch-impact",
|
||||
title: "Launch Impact, Across Every Platform",
|
||||
description:
|
||||
"One prompt chains Google Search, Reddit, and YouTube into a single cited brief on how a competitor launch actually landed.",
|
||||
src: null,
|
||||
demo: {
|
||||
prompt:
|
||||
"Our competitor launched v2 yesterday. Measure the reaction across search, Reddit, and YouTube.",
|
||||
steps: [
|
||||
{
|
||||
title: "Google Search",
|
||||
items: ["Scraping 8 SERPs · launch coverage + AI Overviews"],
|
||||
},
|
||||
{
|
||||
title: "Reddit",
|
||||
items: ['"competitor v2" · 23 threads in the past 48h'],
|
||||
},
|
||||
{
|
||||
title: "Youtube",
|
||||
items: ["6 launch videos · 1,904 comments pulled"],
|
||||
},
|
||||
{
|
||||
title: "Plan tasks",
|
||||
items: ["Merge all three signals into one launch-impact brief"],
|
||||
},
|
||||
],
|
||||
rows: [
|
||||
{
|
||||
primary: "5 of 8 SERPs show launch coverage",
|
||||
secondary: "2 already trigger AI Overviews citing their blog",
|
||||
},
|
||||
{
|
||||
primary: "Reddit: pricing backlash in 9 of 23 threads",
|
||||
secondary: '"v2 doubled the price" · top thread 412 upvotes',
|
||||
},
|
||||
{
|
||||
primary: "YouTube: creators praise UI, question pricing",
|
||||
secondary: "61% positive on features · pricing the top complaint",
|
||||
},
|
||||
],
|
||||
summary: "3 connectors, one cited brief · saved to your workspace",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "local-teardown",
|
||||
title: "Local Competitor Teardown",
|
||||
description:
|
||||
"Google Maps finds the players, the Web Crawler reads their sites, and Google Search shows who wins the query, in one run.",
|
||||
src: null,
|
||||
demo: {
|
||||
prompt:
|
||||
'Tear down the top-rated gyms in Austin: reviews, pricing pages, and who ranks for "gym austin".',
|
||||
steps: [
|
||||
{
|
||||
title: "Google Maps",
|
||||
items: ['"gym austin" · top 10 places + 2,400 reviews'],
|
||||
},
|
||||
{
|
||||
title: "Web Crawler",
|
||||
items: ["Visiting 10 gym sites", "Extracting pricing and membership pages"],
|
||||
},
|
||||
{
|
||||
title: "Google Search",
|
||||
items: ['SERP for "gym austin" · organic, ads, map pack'],
|
||||
},
|
||||
],
|
||||
rows: [
|
||||
{
|
||||
primary: "Review themes: crowding + billing complaints",
|
||||
secondary: "appear in 31% of 1-3★ reviews across 10 gyms",
|
||||
},
|
||||
{
|
||||
primary: "Pricing: $89–149/mo · 3 hide it behind forms",
|
||||
secondary: "extracted from all 10 sites with source pages",
|
||||
},
|
||||
{
|
||||
primary: "2 gyms buy ads on their own brand name",
|
||||
secondary: "map pack and organic top 3 don't overlap",
|
||||
},
|
||||
],
|
||||
summary: "Maps + Crawler + Search in one run · teardown saved",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "competitor-360",
|
||||
title: "Competitor 360, on a Schedule",
|
||||
description:
|
||||
"An automation chains four connectors every week: site changes, rank movements, Reddit sentiment, and YouTube reaction.",
|
||||
src: null,
|
||||
demo: {
|
||||
prompt:
|
||||
"Every Monday, build me a 360 on our top competitor: site changes, rankings, Reddit, and YouTube.",
|
||||
steps: [
|
||||
{
|
||||
title: "Web Crawler",
|
||||
items: ["pricing + changelog pages · 2 changes detected"],
|
||||
},
|
||||
{
|
||||
title: "Google Search",
|
||||
items: ["12 shared keywords · rank movements captured"],
|
||||
},
|
||||
{
|
||||
title: "Reddit",
|
||||
items: ["18 mentions this week · sentiment tagged"],
|
||||
},
|
||||
{
|
||||
title: "Youtube",
|
||||
items: ["2 new videos · comments and transcripts pulled"],
|
||||
},
|
||||
{
|
||||
title: "Create automation",
|
||||
items: ["Weekly 360 brief · Mondays 8:00"],
|
||||
},
|
||||
],
|
||||
rows: [
|
||||
{
|
||||
primary: "Shipped: usage-based pricing page",
|
||||
secondary: "pricing + changelog diff · detected Jul 3",
|
||||
},
|
||||
{
|
||||
primary: 'Took #2 on "competitive intelligence api"',
|
||||
secondary: "you hold #4 · gap widened two weeks in a row",
|
||||
},
|
||||
{
|
||||
primary: "Reddit sentiment down 12 pts since the change",
|
||||
secondary: "churn signals in 5 threads · quotes linked",
|
||||
},
|
||||
],
|
||||
summary: "4 connectors, 1 automation · first brief lands Monday 8:00",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "competitor-monitoring",
|
||||
label: "Competitor Monitoring",
|
||||
|
|
@ -580,8 +718,8 @@ export function HeroSection() {
|
|||
)}
|
||||
>
|
||||
SurfSense is an open-source competitive intelligence platform. Your AI agents monitor
|
||||
competitors, track rankings, and listen to your market with live data from the
|
||||
platforms that matter, through one API or MCP server.
|
||||
competitors, track rankings, and listen to your market with live data from platforms
|
||||
like Reddit, YouTube, Google Maps, Google Search, and the open web.
|
||||
</p>
|
||||
|
||||
<div className="relative mb-4 flex w-full flex-col justify-center gap-y-2 sm:flex-row sm:justify-start sm:space-y-0 sm:space-x-4">
|
||||
|
|
|
|||
86
surfsense_web/components/homepage/persona-paths.tsx
Normal file
86
surfsense_web/components/homepage/persona-paths.tsx
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import { ArrowRight, Code2, Megaphone } from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { Reveal } from "@/components/connectors-marketing/reveal";
|
||||
import { UseCaseArt, type UseCaseArtVariant } from "@/components/homepage/use-case-art";
|
||||
import { MarketingSection } from "@/components/marketing/section";
|
||||
|
||||
/**
|
||||
* Answers "is this for me?" right below the hero: one card per audience.
|
||||
* Revenue persona (founders / marketing teams) first, growth persona
|
||||
* (developers / agent builders) second.
|
||||
*/
|
||||
const PATHS: {
|
||||
icon: typeof Megaphone;
|
||||
art: UseCaseArtVariant;
|
||||
eyebrow: string;
|
||||
title: string;
|
||||
description: string;
|
||||
links: { label: string; href: string }[];
|
||||
}[] = [
|
||||
{
|
||||
icon: Megaphone,
|
||||
art: "chat",
|
||||
eyebrow: "For founders & marketing teams",
|
||||
title: "Competitor and market intelligence without the enterprise price tag",
|
||||
description:
|
||||
"Ask for a competitor teardown, a lead list, or a weekly market brief in plain English. The agent gathers live data, cites its sources, and automations keep watch so you hear about changes first. Start free, pay only for what you use.",
|
||||
links: [
|
||||
{ label: "See what teams build", href: "/connectors" },
|
||||
{ label: "Pricing", href: "/pricing" },
|
||||
],
|
||||
},
|
||||
{
|
||||
icon: Code2,
|
||||
art: "api",
|
||||
eyebrow: "For developers & agents",
|
||||
title: "The whole platform is programmable",
|
||||
description:
|
||||
"Everything SurfSense agents can do is a typed REST API: scrape Reddit, YouTube, Google Maps, Google Search, and the open web, search the knowledge base, run automations. One key, JSON in and out, $5 free credit, pay as you go. Already running agents in Claude, Cursor, or your own harness? The SurfSense MCP server hands them the same tools natively.",
|
||||
links: [
|
||||
{ label: "Read the docs", href: "/docs" },
|
||||
{ label: "SurfSense MCP server", href: "/mcp-server" },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
export function PersonaPaths() {
|
||||
return (
|
||||
<MarketingSection>
|
||||
<Reveal>
|
||||
<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">Who SurfSense is for</h2>
|
||||
</Reveal>
|
||||
<div className="mt-8 grid gap-6 md:grid-cols-2">
|
||||
{PATHS.map((path) => {
|
||||
const Icon = path.icon;
|
||||
return (
|
||||
<Reveal key={path.eyebrow}>
|
||||
<div className="flex h-full flex-col rounded-xl border bg-card p-6">
|
||||
<UseCaseArt variant={path.art} />
|
||||
<div className="flex items-center gap-2 text-sm font-medium text-brand">
|
||||
<Icon className="size-4" aria-hidden />
|
||||
{path.eyebrow}
|
||||
</div>
|
||||
<h3 className="mt-3 text-lg font-semibold">{path.title}</h3>
|
||||
<p className="mt-2 flex-1 text-sm leading-relaxed text-muted-foreground">
|
||||
{path.description}
|
||||
</p>
|
||||
<div className="mt-4 flex flex-wrap gap-4">
|
||||
{path.links.map((link) => (
|
||||
<Link
|
||||
key={link.href}
|
||||
href={link.href}
|
||||
className="group inline-flex items-center gap-1 text-sm font-medium text-foreground"
|
||||
>
|
||||
{link.label}
|
||||
<ArrowRight className="size-4 transition-transform group-hover:translate-x-0.5" />
|
||||
</Link>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</Reveal>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</MarketingSection>
|
||||
);
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ import { motion, useReducedMotion } from "motion/react";
|
|||
const EASE_OUT: [number, number, number, number] = [0.16, 1, 0.3, 1];
|
||||
const VIEWPORT = { once: true, amount: 0.4 } as const;
|
||||
|
||||
export type UseCaseArtVariant = "price" | "brand" | "leads" | "serp";
|
||||
export type UseCaseArtVariant = "price" | "brand" | "leads" | "serp" | "chat" | "api";
|
||||
|
||||
/** Soft infinite pulse ring marking the "live" signal in each artifact. */
|
||||
function Pulse({
|
||||
|
|
@ -282,11 +282,154 @@ function SerpArt({ reduce }: { reduce: boolean }) {
|
|||
);
|
||||
}
|
||||
|
||||
/** Founders & marketers: a plain-English ask streams back an agent brief with cited sources. */
|
||||
function ChatArt({ reduce }: { reduce: boolean }) {
|
||||
const briefLines = [
|
||||
{ y: 38, width: 150, delay: 0.55 },
|
||||
{ y: 52, width: 180, delay: 0.7 },
|
||||
{ y: 66, width: 120, delay: 0.85 },
|
||||
];
|
||||
return (
|
||||
<motion.svg
|
||||
viewBox="0 0 240 96"
|
||||
className="h-auto w-full"
|
||||
initial={reduce ? undefined : "hidden"}
|
||||
whileInView="visible"
|
||||
viewport={VIEWPORT}
|
||||
>
|
||||
{/* User ask: right-aligned bubble */}
|
||||
<motion.g
|
||||
variants={{ hidden: { opacity: 0, y: -6 }, visible: { opacity: 1, y: 0 } }}
|
||||
transition={{ duration: 0.35, ease: EASE_OUT, delay: 0.1 }}
|
||||
>
|
||||
<rect x="112" y="8" width="116" height="18" rx="9" className="fill-muted-foreground/15" />
|
||||
<rect x="124" y="14" width="92" height="6" rx="3" className="fill-muted-foreground/35" />
|
||||
</motion.g>
|
||||
{/* Agent avatar */}
|
||||
<motion.circle
|
||||
cx="22"
|
||||
cy="44"
|
||||
r="6"
|
||||
className="fill-brand"
|
||||
variants={{ hidden: { opacity: 0, scale: 0 }, visible: { opacity: 1, scale: 1 } }}
|
||||
style={{ transformBox: "fill-box", transformOrigin: "center" }}
|
||||
transition={{ duration: 0.3, ease: EASE_OUT, delay: 0.4 }}
|
||||
/>
|
||||
<Pulse cx={22} cy={44} reduce={reduce} delay={0.7} />
|
||||
{/* Brief streams in line by line */}
|
||||
{briefLines.map((line) => (
|
||||
<motion.rect
|
||||
key={line.y}
|
||||
x="36"
|
||||
y={line.y}
|
||||
width={line.width}
|
||||
height="6"
|
||||
rx="3"
|
||||
className="fill-muted-foreground/30"
|
||||
variants={{ hidden: { opacity: 0, x: -8 }, visible: { opacity: 1, x: 0 } }}
|
||||
transition={{ duration: 0.3, ease: EASE_OUT, delay: line.delay }}
|
||||
/>
|
||||
))}
|
||||
{/* Citation chip */}
|
||||
<motion.g
|
||||
variants={{ hidden: { opacity: 0, y: 6 }, visible: { opacity: 1, y: 0 } }}
|
||||
transition={{ duration: 0.3, ease: EASE_OUT, delay: 1.05 }}
|
||||
>
|
||||
<rect x="36" y="76" width="64" height="15" rx="7.5" className="fill-brand/10" />
|
||||
<text x="68" y="87" textAnchor="middle" className="fill-brand text-[9px] font-medium">
|
||||
3 sources
|
||||
</text>
|
||||
</motion.g>
|
||||
</motion.svg>
|
||||
);
|
||||
}
|
||||
|
||||
/** Developers & agents: a typed request gets a 200 and JSON streams back. */
|
||||
function ApiArt({ reduce }: { reduce: boolean }) {
|
||||
const jsonLines = [
|
||||
{ y: 50, text: '{ "items": [', delay: 0.6 },
|
||||
{ y: 66, text: ' { "title": "...", "score": 812 },', delay: 0.75 },
|
||||
{ y: 82, text: "] }", delay: 0.9 },
|
||||
];
|
||||
return (
|
||||
<motion.svg
|
||||
viewBox="0 0 240 96"
|
||||
className="h-auto w-full"
|
||||
initial={reduce ? undefined : "hidden"}
|
||||
whileInView="visible"
|
||||
viewport={VIEWPORT}
|
||||
>
|
||||
{/* Request line */}
|
||||
<motion.g
|
||||
variants={{ hidden: { opacity: 0, x: -8 }, visible: { opacity: 1, x: 0 } }}
|
||||
transition={{ duration: 0.35, ease: EASE_OUT, delay: 0.1 }}
|
||||
>
|
||||
<text x="12" y="22" className="fill-brand font-mono text-[10px] font-semibold">
|
||||
POST
|
||||
</text>
|
||||
<text x="44" y="22" className="fill-muted-foreground font-mono text-[10px]">
|
||||
/scrapers/reddit/scrape
|
||||
</text>
|
||||
</motion.g>
|
||||
{/* 200 OK badge */}
|
||||
<motion.g
|
||||
variants={{ hidden: { opacity: 0, scale: 0.8 }, visible: { opacity: 1, scale: 1 } }}
|
||||
style={{ transformBox: "fill-box", transformOrigin: "center" }}
|
||||
transition={{ duration: 0.3, ease: EASE_OUT, delay: 0.45 }}
|
||||
>
|
||||
<rect x="182" y="10" width="46" height="16" rx="4" className="fill-brand/10" />
|
||||
<text x="205" y="21" textAnchor="middle" className="fill-brand text-[9px] font-medium">
|
||||
200 OK
|
||||
</text>
|
||||
</motion.g>
|
||||
<line
|
||||
x1="12"
|
||||
y1="32"
|
||||
x2="228"
|
||||
y2="32"
|
||||
className="stroke-muted-foreground/15"
|
||||
strokeWidth="1"
|
||||
/>
|
||||
{/* JSON response streams in */}
|
||||
{jsonLines.map((line) => (
|
||||
<motion.text
|
||||
key={line.y}
|
||||
x="12"
|
||||
y={line.y}
|
||||
className="fill-muted-foreground font-mono text-[10px]"
|
||||
style={{ whiteSpace: "pre" }}
|
||||
variants={{ hidden: { opacity: 0, x: -8 }, visible: { opacity: 1, x: 0 } }}
|
||||
transition={{ duration: 0.3, ease: EASE_OUT, delay: line.delay }}
|
||||
>
|
||||
{line.text}
|
||||
</motion.text>
|
||||
))}
|
||||
{/* Blinking cursor after the closing brace */}
|
||||
<motion.rect
|
||||
x="34"
|
||||
y="74"
|
||||
width="5"
|
||||
height="10"
|
||||
className="fill-brand"
|
||||
variants={{ hidden: { opacity: 0 }, visible: { opacity: 1 } }}
|
||||
animate={reduce ? undefined : { opacity: [1, 1, 0, 0] }}
|
||||
transition={
|
||||
reduce
|
||||
? { duration: 0.2, delay: 1 }
|
||||
: { duration: 1, times: [0, 0.5, 0.5, 1], repeat: Infinity, delay: 1 }
|
||||
}
|
||||
/>
|
||||
</motion.svg>
|
||||
);
|
||||
}
|
||||
|
||||
const ART: Record<UseCaseArtVariant, (props: { reduce: boolean }) => React.ReactNode> = {
|
||||
price: PriceArt,
|
||||
brand: BrandArt,
|
||||
leads: LeadsArt,
|
||||
serp: SerpArt,
|
||||
chat: ChatArt,
|
||||
api: ApiArt,
|
||||
};
|
||||
|
||||
/** Small animated artifact rendered at the top of each use-case card. */
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
"use client";
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { useAtomValue, useSetAtom } from "jotai";
|
||||
import { AlarmClock, AlertTriangle, Boxes, Inbox } from "lucide-react";
|
||||
import { useAtom, useAtomValue, useSetAtom } from "jotai";
|
||||
import { AlarmClock, AlertTriangle, Boxes, Inbox, SquareTerminal } from "lucide-react";
|
||||
import { useParams, usePathname, useRouter } from "next/navigation";
|
||||
import { useTranslations } from "next-intl";
|
||||
import { useTheme } from "next-themes";
|
||||
|
|
@ -11,6 +11,7 @@ import { toast } from "sonner";
|
|||
import { currentThreadAtom, resetCurrentThreadAtom } from "@/atoms/chat/current-thread.atom";
|
||||
import { statusInboxItemsAtom } from "@/atoms/inbox/status-inbox.atom";
|
||||
import { announcementsDialogAtom } from "@/atoms/layout/dialogs.atom";
|
||||
import { playgroundSidebarOpenAtom } from "@/atoms/layout/playground.atom";
|
||||
import { removeChatTabAtom, syncChatTabAtom, type Tab } from "@/atoms/tabs/tabs.atom";
|
||||
import { currentUserAtom } from "@/atoms/user/user-query.atoms";
|
||||
import { deleteWorkspaceMutationAtom } from "@/atoms/workspaces/workspace-mutation.atoms";
|
||||
|
|
@ -42,6 +43,7 @@ import { Spinner } from "@/components/ui/spinner";
|
|||
import { useActivateChatThread } from "@/hooks/use-activate-chat-thread";
|
||||
import { useAnnouncements } from "@/hooks/use-announcements";
|
||||
import { useInbox } from "@/hooks/use-inbox";
|
||||
import { useIsMobile } from "@/hooks/use-mobile";
|
||||
import { useArchiveThread, useDeleteThread, useRenameThread } from "@/hooks/use-thread-mutations";
|
||||
import { notificationsApiService } from "@/lib/apis/notifications-api.service";
|
||||
import { workspacesApiService } from "@/lib/apis/workspaces-api.service";
|
||||
|
|
@ -77,6 +79,8 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
|
|||
const params = useParams();
|
||||
const pathname = usePathname();
|
||||
const { theme, setTheme } = useTheme();
|
||||
const isMobile = useIsMobile();
|
||||
const [playgroundSidebarOpen, setPlaygroundSidebarOpen] = useAtom(playgroundSidebarOpenAtom);
|
||||
|
||||
// Announcements
|
||||
const { unreadCount: announcementUnreadCount } = useAnnouncements();
|
||||
|
|
@ -319,6 +323,7 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
|
|||
// list). Documents is embedded below Recents; announcements live in the avatar dropdown.
|
||||
const isAutomationsActive = pathname?.includes("/automations") === true;
|
||||
const isArtifactsActive = pathname?.endsWith("/artifacts") === true;
|
||||
const isPlaygroundRoute = pathname?.includes("/playground") === true;
|
||||
const navItems: NavItem[] = useMemo(
|
||||
() =>
|
||||
(
|
||||
|
|
@ -342,6 +347,14 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
|
|||
icon: Boxes,
|
||||
isActive: isArtifactsActive,
|
||||
},
|
||||
{
|
||||
title: "Playground",
|
||||
url: `/dashboard/${workspaceId}/playground`,
|
||||
icon: SquareTerminal,
|
||||
// Mobile has no second-level sidebar: Playground is a plain link
|
||||
// there, so highlight by route. Desktop highlights the toggle state.
|
||||
isActive: isMobile ? isPlaygroundRoute : playgroundSidebarOpen,
|
||||
},
|
||||
] as (NavItem | null)[]
|
||||
).filter((item): item is NavItem => item !== null),
|
||||
[
|
||||
|
|
@ -350,6 +363,9 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
|
|||
workspaceId,
|
||||
isAutomationsActive,
|
||||
isArtifactsActive,
|
||||
isPlaygroundRoute,
|
||||
playgroundSidebarOpen,
|
||||
isMobile,
|
||||
]
|
||||
);
|
||||
|
||||
|
|
@ -491,9 +507,18 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
|
|||
setActiveSlideoutPanel((prev) => (prev === "inbox" ? null : "inbox"));
|
||||
return;
|
||||
}
|
||||
// Desktop: Playground is a persistent toggle, not a plain link — it just
|
||||
// opens the second-level sidebar (which holds the whole API playground)
|
||||
// and only closes on a second click, never navigating away from the
|
||||
// current page (e.g. a new chat). Mobile has no second-level sidebar,
|
||||
// so there it navigates to the playground index page instead.
|
||||
if (item.url.endsWith("/playground") && !isMobile) {
|
||||
setPlaygroundSidebarOpen((prev) => !prev);
|
||||
return;
|
||||
}
|
||||
router.push(item.url);
|
||||
},
|
||||
[router]
|
||||
[router, setPlaygroundSidebarOpen, setActiveSlideoutPanel, isMobile]
|
||||
);
|
||||
|
||||
const handleNewChat = useCallback(() => {
|
||||
|
|
@ -660,6 +685,7 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
|
|||
const isTeamPage = pathname?.endsWith("/team") === true;
|
||||
const isAutomationsPage = pathname?.includes("/automations") === true;
|
||||
const isArtifactsPage = pathname?.endsWith("/artifacts") === true;
|
||||
const isPlaygroundPage = pathname?.includes("/playground") === true;
|
||||
const isAllChatsPage = pathname?.endsWith("/chats") === true;
|
||||
const handleViewAllChats = useCallback(() => {
|
||||
setActiveSlideoutPanel(null);
|
||||
|
|
@ -676,6 +702,7 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
|
|||
isTeamPage ||
|
||||
isAutomationsPage ||
|
||||
isArtifactsPage ||
|
||||
isPlaygroundPage ||
|
||||
isAllChatsPage;
|
||||
|
||||
return (
|
||||
|
|
@ -714,6 +741,7 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
|
|||
setTheme={setTheme}
|
||||
isChatPage={isChatPage}
|
||||
isAllChatsPage={isAllChatsPage}
|
||||
showPlaygroundSidebar={playgroundSidebarOpen}
|
||||
useWorkspacePanel={useWorkspacePanel}
|
||||
workspacePanelViewportClassName={
|
||||
isUserSettingsPage ||
|
||||
|
|
@ -721,12 +749,13 @@ export function LayoutDataProvider({ workspaceId, children }: LayoutDataProvider
|
|||
isTeamPage ||
|
||||
isAutomationsPage ||
|
||||
isArtifactsPage ||
|
||||
isPlaygroundPage ||
|
||||
isAllChatsPage
|
||||
? "items-start justify-center px-6 py-8 md:px-10 md:pb-10 md:pt-16"
|
||||
: undefined
|
||||
}
|
||||
workspacePanelContentClassName={
|
||||
isAutomationsPage
|
||||
isAutomationsPage || isPlaygroundPage
|
||||
? "max-w-none select-none"
|
||||
: isAllChatsPage
|
||||
? "max-w-5xl"
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import {
|
|||
InboxSidebarContent,
|
||||
MobileSidebar,
|
||||
MobileSidebarTrigger,
|
||||
PlaygroundSidebar,
|
||||
Sidebar,
|
||||
SidebarCollapseButton,
|
||||
} from "../sidebar";
|
||||
|
|
@ -133,6 +134,7 @@ interface LayoutShellProps {
|
|||
defaultCollapsed?: boolean;
|
||||
isChatPage?: boolean;
|
||||
isAllChatsPage?: boolean;
|
||||
showPlaygroundSidebar?: boolean;
|
||||
useWorkspacePanel?: boolean;
|
||||
workspacePanelViewportClassName?: string;
|
||||
workspacePanelContentClassName?: string;
|
||||
|
|
@ -237,6 +239,7 @@ export function LayoutShell({
|
|||
defaultCollapsed = false,
|
||||
isChatPage = false,
|
||||
isAllChatsPage = false,
|
||||
showPlaygroundSidebar = false,
|
||||
useWorkspacePanel = false,
|
||||
workspacePanelViewportClassName,
|
||||
workspacePanelContentClassName,
|
||||
|
|
@ -403,11 +406,30 @@ export function LayoutShell({
|
|||
/>
|
||||
</div>
|
||||
|
||||
{/* Playground second-level sidebar — contextual, desktop only. Sits
|
||||
between the icon rail and the main sidebar. On Mac it becomes the
|
||||
leftmost panel, so it takes the rounded-corner/left-border treatment. */}
|
||||
{showPlaygroundSidebar && activeWorkspaceId != null && (
|
||||
<div
|
||||
className={cn(
|
||||
"relative hidden md:flex shrink-0 z-20 -mr-2 bg-panel",
|
||||
isMacDesktop ? "rounded-tl-xl border-t border-r border-l" : "border-r"
|
||||
)}
|
||||
>
|
||||
<PlaygroundSidebar workspaceId={activeWorkspaceId} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Sidebar + slide-out panels share one container; overflow visible so panels can overlay main content. Negative right margin closes the flex gap so the sidebar sits flush against the main panel, separated only by a border. */}
|
||||
<div
|
||||
className={cn(
|
||||
"relative hidden md:flex shrink-0 z-20 -mr-2 bg-panel",
|
||||
isMacDesktop ? "rounded-tl-xl border-t border-r border-l" : "border-r"
|
||||
isMacDesktop
|
||||
? cn(
|
||||
"border-t border-r",
|
||||
!showPlaygroundSidebar && "rounded-tl-xl border-l"
|
||||
)
|
||||
: "border-r"
|
||||
)}
|
||||
>
|
||||
<Sidebar
|
||||
|
|
@ -443,7 +465,10 @@ export function LayoutShell({
|
|||
<Logo disableLink priority className="h-7 w-7 rounded-md" />
|
||||
) : undefined
|
||||
}
|
||||
className={cn("flex shrink-0", isMacDesktop && "rounded-tl-xl")}
|
||||
className={cn(
|
||||
"flex shrink-0",
|
||||
isMacDesktop && !showPlaygroundSidebar && "rounded-tl-xl"
|
||||
)}
|
||||
isLoadingChats={isLoadingChats}
|
||||
sidebarWidth={sidebarWidth}
|
||||
isResizing={isResizing}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
"use client";
|
||||
|
||||
import { History, KeyRound } from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { usePathname } from "next/navigation";
|
||||
import { PLAYGROUND_PLATFORMS, type PlatformIcon } from "@/lib/playground/catalog";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
interface PlaygroundSidebarProps {
|
||||
workspaceId: number | string;
|
||||
}
|
||||
|
||||
function PlaygroundNavLink({
|
||||
href,
|
||||
label,
|
||||
icon: Icon,
|
||||
isActive,
|
||||
indented = false,
|
||||
}: {
|
||||
href: string;
|
||||
label: string;
|
||||
icon?: PlatformIcon;
|
||||
isActive: boolean;
|
||||
indented?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<Link
|
||||
href={href}
|
||||
aria-current={isActive ? "page" : undefined}
|
||||
className={cn(
|
||||
"group/link relative flex h-9 items-center gap-2 rounded-md mx-2 px-2 text-sm text-left",
|
||||
"transition-colors hover:bg-accent hover:text-accent-foreground",
|
||||
"focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring",
|
||||
indented && "pl-8",
|
||||
isActive && "bg-accent text-accent-foreground"
|
||||
)}
|
||||
>
|
||||
{Icon ? <Icon className="h-3.5 w-3.5 shrink-0" /> : null}
|
||||
<span className="min-w-0 flex-1 truncate">{label}</span>
|
||||
</Link>
|
||||
);
|
||||
}
|
||||
|
||||
export function PlaygroundSidebar({ workspaceId }: PlaygroundSidebarProps) {
|
||||
const pathname = usePathname();
|
||||
const base = `/dashboard/${workspaceId}/playground`;
|
||||
|
||||
return (
|
||||
<div className="relative flex h-full w-[240px] flex-col bg-panel text-sidebar-foreground overflow-hidden select-none">
|
||||
<div className="flex h-12 shrink-0 items-center px-4">
|
||||
<span className="text-sm font-semibold">API Playground</span>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-0.5 pt-1.5 pb-1.5 after:mx-3 after:mt-1.5 after:block after:h-px after:bg-border">
|
||||
<PlaygroundNavLink
|
||||
href={`${base}/runs`}
|
||||
label="Runs"
|
||||
icon={History}
|
||||
isActive={pathname === `${base}/runs`}
|
||||
/>
|
||||
<PlaygroundNavLink
|
||||
href={`${base}/api-keys`}
|
||||
label="API Keys"
|
||||
icon={KeyRound}
|
||||
isActive={pathname === `${base}/api-keys`}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex-1 w-full min-h-0 overflow-y-auto overflow-x-hidden scrollbar-thin scrollbar-thumb-muted-foreground/20 scrollbar-track-transparent pb-2">
|
||||
{PLAYGROUND_PLATFORMS.map((platform) => (
|
||||
<div key={platform.id} className="flex flex-col gap-0.5 pt-2">
|
||||
<div className="flex items-center gap-2 pl-4 pr-2.5 py-1 text-xs font-medium text-muted-foreground">
|
||||
<platform.icon className="h-3.5 w-3.5 shrink-0" />
|
||||
<span className="truncate">{platform.label}</span>
|
||||
</div>
|
||||
{platform.verbs.map((verb) => {
|
||||
const href = `${base}/${platform.id}/${verb.verb}`;
|
||||
return (
|
||||
<PlaygroundNavLink
|
||||
key={verb.name}
|
||||
href={href}
|
||||
label={verb.label}
|
||||
isActive={pathname === href}
|
||||
indented
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -150,13 +150,18 @@ export function Sidebar({
|
|||
() => navItems.find((item) => item.url.endsWith("/artifacts")),
|
||||
[navItems]
|
||||
);
|
||||
const playgroundItem = useMemo(
|
||||
() => navItems.find((item) => item.url.endsWith("/playground")),
|
||||
[navItems]
|
||||
);
|
||||
const footerNavItems = useMemo(
|
||||
() =>
|
||||
navItems.filter(
|
||||
(item) =>
|
||||
item.url !== "#inbox" &&
|
||||
!item.url.endsWith("/automations") &&
|
||||
!item.url.endsWith("/artifacts")
|
||||
!item.url.endsWith("/artifacts") &&
|
||||
!item.url.endsWith("/playground")
|
||||
),
|
||||
[navItems]
|
||||
);
|
||||
|
|
@ -267,6 +272,16 @@ export function Sidebar({
|
|||
tooltipContent={isCollapsed ? artifactsItem.title : undefined}
|
||||
/>
|
||||
)}
|
||||
{playgroundItem && (
|
||||
<SidebarButton
|
||||
icon={playgroundItem.icon}
|
||||
label={playgroundItem.title}
|
||||
onClick={() => onNavItemClick?.(playgroundItem)}
|
||||
isCollapsed={isCollapsed}
|
||||
isActive={playgroundItem.isActive}
|
||||
tooltipContent={isCollapsed ? playgroundItem.title : undefined}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Chat sections - fills available space */}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ export { DocumentsSidebar } from "./DocumentsSidebar";
|
|||
export { InboxSidebar, InboxSidebarContent } from "./InboxSidebar";
|
||||
export { MobileSidebar, MobileSidebarTrigger } from "./MobileSidebar";
|
||||
export { NavSection } from "./NavSection";
|
||||
export { PlaygroundSidebar } from "./PlaygroundSidebar";
|
||||
export { Sidebar } from "./Sidebar";
|
||||
export { SidebarCollapseButton } from "./SidebarCollapseButton";
|
||||
export { SidebarHeader } from "./SidebarHeader";
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@
|
|||
|
||||
import {
|
||||
AlarmClock,
|
||||
FilePlus2,
|
||||
type LucideIcon,
|
||||
Search,
|
||||
MessagesSquare,
|
||||
Radar,
|
||||
Settings2,
|
||||
WandSparkles,
|
||||
Workflow,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { memo, useCallback, useState } from "react";
|
||||
|
|
@ -20,8 +21,9 @@ interface ChatExamplePromptsProps {
|
|||
}
|
||||
|
||||
const CATEGORY_ICONS: Record<string, LucideIcon> = {
|
||||
search: Search,
|
||||
create: FilePlus2,
|
||||
monitor: Radar,
|
||||
listen: MessagesSquare,
|
||||
workflows: Workflow,
|
||||
automate: AlarmClock,
|
||||
tools: Settings2,
|
||||
};
|
||||
|
|
|
|||
84
surfsense_web/contracts/types/scraper.types.ts
Normal file
84
surfsense_web/contracts/types/scraper.types.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import { z } from "zod";
|
||||
|
||||
/**
|
||||
* A platform-native scraper verb and its input/output JSON schemas. Both are
|
||||
* arbitrary JSON Schema (pydantic ``model_json_schema()``): the input schema is
|
||||
* consumed by the playground's generic form renderer, the output schema by the
|
||||
* API reference docs — so they are intentionally untyped here.
|
||||
*/
|
||||
export const scraperCapability = z.object({
|
||||
name: z.string(),
|
||||
description: z.string(),
|
||||
input_schema: z.record(z.string(), z.unknown()),
|
||||
// Optional so a backend that predates output schemas degrades to just not
|
||||
// showing the output-schema block instead of failing the whole fetch.
|
||||
output_schema: z.record(z.string(), z.unknown()).optional(),
|
||||
});
|
||||
|
||||
export const listCapabilitiesResponse = z.array(scraperCapability);
|
||||
|
||||
/**
|
||||
* Run origin: ``ui`` (in-app), ``api`` (PAT/public API), ``agent`` (chat tools).
|
||||
*/
|
||||
export const scraperRunSummary = z.object({
|
||||
id: z.string(),
|
||||
capability: z.string(),
|
||||
origin: z.string(),
|
||||
status: z.string(),
|
||||
item_count: z.number(),
|
||||
char_count: z.number(),
|
||||
duration_ms: z.number().nullable(),
|
||||
cost_micros: z.number().nullable(),
|
||||
error: z.string().nullable(),
|
||||
created_at: z.string(),
|
||||
});
|
||||
|
||||
export const scraperRunDetail = scraperRunSummary.extend({
|
||||
thread_id: z.string().nullable(),
|
||||
input: z.record(z.string(), z.unknown()).nullable(),
|
||||
output_text: z.string().nullable(),
|
||||
// Coarse progress log captured during the run (nullable/absent for older or
|
||||
// zero-progress runs). Each entry is a ``run.progress`` event object.
|
||||
progress: z.array(z.record(z.string(), z.unknown())).nullable().optional(),
|
||||
});
|
||||
|
||||
export const listRunsResponse = z.array(scraperRunSummary);
|
||||
|
||||
/** Response of an async run start (``POST ...?mode=async`` -> 202). */
|
||||
export const startAsyncRunResponse = z.object({
|
||||
run_id: z.string(),
|
||||
status: z.string(),
|
||||
});
|
||||
|
||||
export type ScraperCapability = z.infer<typeof scraperCapability>;
|
||||
export type ScraperRunSummary = z.infer<typeof scraperRunSummary>;
|
||||
export type ScraperRunDetail = z.infer<typeof scraperRunDetail>;
|
||||
export type StartAsyncRunResponse = z.infer<typeof startAsyncRunResponse>;
|
||||
|
||||
export interface ListScraperRunsParams {
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
capability?: string;
|
||||
status?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A live progress event streamed from ``GET .../runs/{run_id}/events`` (SSE).
|
||||
* One flexible shape mirrors the backend: lifecycle events (``run.started`` /
|
||||
* ``run.finished`` / ``run.heartbeat``) and fine-grained ``run.progress``.
|
||||
*/
|
||||
export interface ScraperRunEvent {
|
||||
type: string;
|
||||
ts?: number;
|
||||
run_id?: string;
|
||||
status?: string;
|
||||
capability?: string;
|
||||
phase?: string;
|
||||
message?: string;
|
||||
current?: number;
|
||||
total?: number;
|
||||
unit?: string;
|
||||
item_count?: number;
|
||||
error?: string | null;
|
||||
detail?: Record<string, unknown>;
|
||||
}
|
||||
|
|
@ -5,10 +5,11 @@ import type { TimelineToolProps } from "./types";
|
|||
* Lossless mapping ``ToolCallItem → TimelineToolProps``. Pure;
|
||||
* extracts only the fields tool components actually consume.
|
||||
*
|
||||
* ``id``, ``kind``, ``items``, ``spanId``, ``thinkingStepId`` are
|
||||
* intentionally dropped — they're timeline-internal concerns (React
|
||||
* key, dispatch, indentation, back-correlation) that tool components
|
||||
* have no reason to see.
|
||||
* ``id``, ``kind``, ``spanId``, ``thinkingStepId`` are intentionally
|
||||
* dropped — they're timeline-internal concerns (React key, dispatch,
|
||||
* indentation, back-correlation) that tool components have no reason to
|
||||
* see. ``items`` is forwarded as ``progress`` so a tool body can show its
|
||||
* live activity (e.g. streamed scraper progress) inside its own card.
|
||||
*/
|
||||
export function adaptItemToProps(item: ToolCallItem): TimelineToolProps {
|
||||
return {
|
||||
|
|
@ -19,5 +20,6 @@ export function adaptItemToProps(item: ToolCallItem): TimelineToolProps {
|
|||
result: item.result,
|
||||
langchainToolCallId: item.langchainToolCallId,
|
||||
status: item.status,
|
||||
progress: item.items,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,10 +56,12 @@ export const DefaultFallbackCard: TimelineToolComponent = ({
|
|||
result,
|
||||
status,
|
||||
langchainToolCallId,
|
||||
progress,
|
||||
}) => {
|
||||
const isCancelled = status === "cancelled";
|
||||
const isError = status === "error";
|
||||
const isRunning = status === "running";
|
||||
const liveProgress = isRunning ? (progress ?? []) : [];
|
||||
|
||||
const [isExpanded, setIsExpanded] = useState(isRunning);
|
||||
useEffect(() => {
|
||||
|
|
@ -72,10 +74,13 @@ export const DefaultFallbackCard: TimelineToolComponent = ({
|
|||
[result]
|
||||
);
|
||||
|
||||
const subtitle = useMemo(
|
||||
() => (isError || isCancelled ? deriveResultMessage(result) : null),
|
||||
[isError, isCancelled, result]
|
||||
);
|
||||
const subtitle = useMemo(() => {
|
||||
if (isError || isCancelled) return deriveResultMessage(result);
|
||||
// While running, surface the latest streamed activity line so progress
|
||||
// is visible even when the card is collapsed.
|
||||
if (isRunning && liveProgress.length > 0) return liveProgress[liveProgress.length - 1];
|
||||
return null;
|
||||
}, [isError, isCancelled, isRunning, liveProgress, result]);
|
||||
|
||||
const displayName = getToolDisplayName(toolName);
|
||||
|
||||
|
|
@ -196,6 +201,25 @@ export const DefaultFallbackCard: TimelineToolComponent = ({
|
|||
</NestedScroll>
|
||||
</div>
|
||||
)}
|
||||
{isRunning && liveProgress.length > 0 && (
|
||||
<>
|
||||
<Separator />
|
||||
<div className="flex flex-col gap-1 min-w-0">
|
||||
<p className="text-xs font-medium text-muted-foreground">Progress</p>
|
||||
<div className="flex flex-col gap-1.5 rounded-md bg-muted/40 px-3 py-2">
|
||||
{liveProgress.map((line) => (
|
||||
<div
|
||||
key={line}
|
||||
className="flex items-center gap-2 text-xs text-foreground/80"
|
||||
>
|
||||
<Spinner size="sm" className="shrink-0 text-primary" />
|
||||
<span className="min-w-0 wrap-break-word">{line}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{!isCancelled && result !== undefined && (
|
||||
<>
|
||||
<Separator />
|
||||
|
|
|
|||
|
|
@ -22,6 +22,13 @@ export interface TimelineToolProps {
|
|||
result?: unknown;
|
||||
langchainToolCallId?: string;
|
||||
status: ItemStatus;
|
||||
/**
|
||||
* Live activity lines for this tool (the joined thinking-step items).
|
||||
* While ``status === "running"`` these are the streamed progress updates
|
||||
* (e.g. scraper ``scraper_progress`` events) so the fallback card can show
|
||||
* output inside its dropdown instead of a static "input only" view.
|
||||
*/
|
||||
progress?: readonly string[];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
164
surfsense_web/hooks/use-run-stream.ts
Normal file
164
surfsense_web/hooks/use-run-stream.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
"use client";
|
||||
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import type { ScraperRunDetail, ScraperRunEvent } from "@/contracts/types/scraper.types";
|
||||
import { scrapersApiService } from "@/lib/apis/scrapers-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
|
||||
export type RunStatus = "idle" | "running" | "success" | "error" | "cancelled";
|
||||
|
||||
/** Keep the live progress log bounded — it's a scrolling view, not a record. */
|
||||
const MAX_LOG = 200;
|
||||
|
||||
interface RunStreamState {
|
||||
runId: string | null;
|
||||
status: RunStatus;
|
||||
/** Recent ``run.progress`` events, newest last. */
|
||||
events: ScraperRunEvent[];
|
||||
/** Latest lifecycle/progress event, for the headline status line. */
|
||||
latest: ScraperRunEvent | null;
|
||||
/** Fetched once on successful finish (carries output + authoritative metrics). */
|
||||
detail: ScraperRunDetail | null;
|
||||
/** Raw error (AppError for start failures, Error for stream/terminal failures). */
|
||||
error: unknown;
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
const INITIAL: RunStreamState = {
|
||||
runId: null,
|
||||
status: "idle",
|
||||
events: [],
|
||||
latest: null,
|
||||
detail: null,
|
||||
error: null,
|
||||
elapsedMs: 0,
|
||||
};
|
||||
|
||||
/**
|
||||
* Drive a single async scraper run: start it, tail its SSE progress, and expose
|
||||
* a small reactive state for the playground. Also supports re-attaching to a run
|
||||
* that is still ``running`` (e.g. after a page refresh).
|
||||
*/
|
||||
export function useRunStream(workspaceId: number) {
|
||||
const queryClient = useQueryClient();
|
||||
const [state, setState] = useState<RunStreamState>(INITIAL);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
const runIdRef = useRef<string | null>(null);
|
||||
const startedAtRef = useRef<number>(0);
|
||||
|
||||
// Live elapsed timer, ticking only while a run is in flight.
|
||||
useEffect(() => {
|
||||
if (state.status !== "running") return;
|
||||
const id = setInterval(() => {
|
||||
setState((s) => ({ ...s, elapsedMs: Date.now() - startedAtRef.current }));
|
||||
}, 500);
|
||||
return () => clearInterval(id);
|
||||
}, [state.status]);
|
||||
|
||||
// Abort any in-flight stream on unmount.
|
||||
useEffect(() => () => abortRef.current?.abort(), []);
|
||||
|
||||
const consume = useCallback(
|
||||
async (runId: string, signal: AbortSignal) => {
|
||||
try {
|
||||
for await (const ev of scrapersApiService.streamRunEvents(workspaceId, runId, signal)) {
|
||||
if (ev.type === "run.finished") {
|
||||
const finalStatus = (ev.status as RunStatus) || "success";
|
||||
let detail: ScraperRunDetail | null = null;
|
||||
if (finalStatus === "success") {
|
||||
try {
|
||||
detail = await scrapersApiService.getRun(workspaceId, runId);
|
||||
} catch {
|
||||
detail = null;
|
||||
}
|
||||
}
|
||||
setState((s) => ({
|
||||
...s,
|
||||
status: finalStatus,
|
||||
detail,
|
||||
error: ev.error ? new Error(ev.error) : s.error,
|
||||
}));
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.scrapers.runs(workspaceId),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (ev.type === "run.heartbeat") continue;
|
||||
setState((s) => ({
|
||||
...s,
|
||||
latest: ev,
|
||||
events:
|
||||
ev.type === "run.progress"
|
||||
? [...s.events.slice(-(MAX_LOG - 1)), ev]
|
||||
: s.events,
|
||||
}));
|
||||
}
|
||||
} catch (e) {
|
||||
if (signal.aborted) return;
|
||||
setState((s) =>
|
||||
s.status === "running" ? { ...s, status: "error", error: e } : s
|
||||
);
|
||||
}
|
||||
},
|
||||
[workspaceId, queryClient]
|
||||
);
|
||||
|
||||
const start = useCallback(
|
||||
async (platform: string, verb: string, payload: Record<string, unknown>) => {
|
||||
abortRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
startedAtRef.current = Date.now();
|
||||
setState({ ...INITIAL, status: "running" });
|
||||
try {
|
||||
const started = await scrapersApiService.runAsync(
|
||||
workspaceId,
|
||||
platform,
|
||||
verb,
|
||||
payload
|
||||
);
|
||||
runIdRef.current = started.run_id;
|
||||
setState((s) => ({ ...s, runId: started.run_id }));
|
||||
void consume(started.run_id, controller.signal);
|
||||
} catch (e) {
|
||||
runIdRef.current = null;
|
||||
setState((s) => ({ ...s, status: "error", error: e }));
|
||||
}
|
||||
},
|
||||
[workspaceId, consume]
|
||||
);
|
||||
|
||||
const reattach = useCallback(
|
||||
(runId: string) => {
|
||||
abortRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
startedAtRef.current = Date.now();
|
||||
runIdRef.current = runId;
|
||||
setState({ ...INITIAL, runId, status: "running" });
|
||||
void consume(runId, controller.signal);
|
||||
},
|
||||
[consume]
|
||||
);
|
||||
|
||||
const cancel = useCallback(async () => {
|
||||
const runId = runIdRef.current;
|
||||
if (!runId) return;
|
||||
// The stream delivers the terminal ``run.finished`` (cancelled) which flips
|
||||
// our state; the request just asks the server to stop.
|
||||
try {
|
||||
await scrapersApiService.cancelRun(workspaceId, runId);
|
||||
} catch {
|
||||
// Best-effort: if the run already finished the stream reflects the truth.
|
||||
}
|
||||
}, [workspaceId]);
|
||||
|
||||
const reset = useCallback(() => {
|
||||
abortRef.current?.abort();
|
||||
runIdRef.current = null;
|
||||
setState(INITIAL);
|
||||
}, []);
|
||||
|
||||
return { ...state, start, cancel, reattach, reset };
|
||||
}
|
||||
18
surfsense_web/hooks/use-scraper-capabilities.ts
Normal file
18
surfsense_web/hooks/use-scraper-capabilities.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
"use client";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import type { ScraperCapability } from "@/contracts/types/scraper.types";
|
||||
import { scrapersApiService } from "@/lib/apis/scrapers-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
|
||||
/**
|
||||
* The platform-native verb registry only changes on deploy, so it is cached
|
||||
* aggressively. Returns every verb with its input JSON schema for the form.
|
||||
*/
|
||||
export function useScraperCapabilities(workspaceId: number | string) {
|
||||
return useQuery<ScraperCapability[], Error>({
|
||||
queryKey: cacheKeys.scrapers.capabilities(workspaceId),
|
||||
queryFn: () => scrapersApiService.listCapabilities(workspaceId),
|
||||
staleTime: 60 * 60 * 1000,
|
||||
gcTime: 60 * 60 * 1000,
|
||||
});
|
||||
}
|
||||
33
surfsense_web/hooks/use-scraper-runs.ts
Normal file
33
surfsense_web/hooks/use-scraper-runs.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
"use client";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import type {
|
||||
ListScraperRunsParams,
|
||||
ScraperRunDetail,
|
||||
ScraperRunSummary,
|
||||
} from "@/contracts/types/scraper.types";
|
||||
import { scrapersApiService } from "@/lib/apis/scrapers-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
|
||||
/** Paginated scraper-run history for a workspace, newest-first. */
|
||||
export function useScraperRuns(workspaceId: number | string, params: ListScraperRunsParams = {}) {
|
||||
return useQuery<ScraperRunSummary[], Error>({
|
||||
queryKey: [...cacheKeys.scrapers.runs(workspaceId), params],
|
||||
queryFn: () => scrapersApiService.listRuns(workspaceId, params),
|
||||
staleTime: 10_000,
|
||||
});
|
||||
}
|
||||
|
||||
/** Full run record (input + stored output) — fetched lazily on row expand. */
|
||||
export function useScraperRun(
|
||||
workspaceId: number | string,
|
||||
runId: string | undefined,
|
||||
options: { enabled?: boolean } = {}
|
||||
) {
|
||||
const { enabled = true } = options;
|
||||
return useQuery<ScraperRunDetail, Error>({
|
||||
queryKey: cacheKeys.scrapers.run(workspaceId, runId ?? ""),
|
||||
queryFn: () => scrapersApiService.getRun(workspaceId, runId as string),
|
||||
enabled: enabled && !!runId,
|
||||
staleTime: 30_000,
|
||||
});
|
||||
}
|
||||
114
surfsense_web/lib/apis/scrapers-api.service.ts
Normal file
114
surfsense_web/lib/apis/scrapers-api.service.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
import {
|
||||
type ListScraperRunsParams,
|
||||
type ScraperRunEvent,
|
||||
type StartAsyncRunResponse,
|
||||
listCapabilitiesResponse,
|
||||
listRunsResponse,
|
||||
scraperRunDetail,
|
||||
startAsyncRunResponse,
|
||||
} from "@/contracts/types/scraper.types";
|
||||
import { readSSEStream } from "@/lib/chat/streaming-state";
|
||||
import { buildBackendUrl } from "@/lib/env-config";
|
||||
import { authenticatedFetch } from "@/lib/auth-fetch";
|
||||
import { baseApiService } from "./base-api.service";
|
||||
|
||||
const base = (workspaceId: number | string) => `/api/v1/workspaces/${workspaceId}/scrapers`;
|
||||
|
||||
class ScrapersApiService {
|
||||
/** List the platform-native verbs and their input JSON schemas. */
|
||||
listCapabilities = async (workspaceId: number | string) => {
|
||||
return baseApiService.get(`${base(workspaceId)}/capabilities`, listCapabilitiesResponse);
|
||||
};
|
||||
|
||||
/**
|
||||
* Manually invoke one verb. Returns the raw typed output (``{ items: [...] }``);
|
||||
* the response is not schema-validated here because each verb has its own shape.
|
||||
*/
|
||||
run = async (
|
||||
workspaceId: number | string,
|
||||
platform: string,
|
||||
verb: string,
|
||||
payload: Record<string, unknown>,
|
||||
signal?: AbortSignal
|
||||
) => {
|
||||
return baseApiService.post<unknown>(`${base(workspaceId)}/${platform}/${verb}`, undefined, {
|
||||
body: payload,
|
||||
signal,
|
||||
});
|
||||
};
|
||||
|
||||
listRuns = async (workspaceId: number | string, params: ListScraperRunsParams = {}) => {
|
||||
const qs = new URLSearchParams();
|
||||
if (params.limit !== undefined) qs.set("limit", String(params.limit));
|
||||
if (params.offset !== undefined) qs.set("offset", String(params.offset));
|
||||
if (params.capability) qs.set("capability", params.capability);
|
||||
if (params.status) qs.set("status", params.status);
|
||||
const query = qs.toString();
|
||||
return baseApiService.get(
|
||||
`${base(workspaceId)}/runs${query ? `?${query}` : ""}`,
|
||||
listRunsResponse
|
||||
);
|
||||
};
|
||||
|
||||
getRun = async (workspaceId: number | string, runId: string) => {
|
||||
return baseApiService.get(`${base(workspaceId)}/runs/${runId}`, scraperRunDetail);
|
||||
};
|
||||
|
||||
/**
|
||||
* Start a verb as a background job (``?mode=async``). Returns the run id
|
||||
* immediately (202); tail progress via {@link streamRunEvents}.
|
||||
*/
|
||||
runAsync = async (
|
||||
workspaceId: number | string,
|
||||
platform: string,
|
||||
verb: string,
|
||||
payload: Record<string, unknown>
|
||||
): Promise<StartAsyncRunResponse> => {
|
||||
return baseApiService.post<StartAsyncRunResponse>(
|
||||
`${base(workspaceId)}/${platform}/${verb}?mode=async`,
|
||||
startAsyncRunResponse,
|
||||
{ body: payload }
|
||||
);
|
||||
};
|
||||
|
||||
/** Request cancellation of an in-progress run. */
|
||||
cancelRun = async (workspaceId: number | string, runId: string) => {
|
||||
return baseApiService.post<StartAsyncRunResponse>(
|
||||
`${base(workspaceId)}/runs/${runId}/cancel`,
|
||||
startAsyncRunResponse
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Tail a run's live progress over SSE. Yields each event until the terminal
|
||||
* ``run.finished`` (or the stream closes / the ``signal`` aborts). Uses
|
||||
* ``authenticatedFetch`` (desktop token + cookie auth + 401 refresh), which
|
||||
* is why ``EventSource`` — no custom headers — can't be used here.
|
||||
*/
|
||||
streamRunEvents = async function* (
|
||||
workspaceId: number | string,
|
||||
runId: string,
|
||||
signal?: AbortSignal
|
||||
): AsyncGenerator<ScraperRunEvent> {
|
||||
const url = buildBackendUrl(`${base(workspaceId)}/runs/${runId}/events`);
|
||||
const response = await authenticatedFetch(url, {
|
||||
method: "GET",
|
||||
headers: { Accept: "text/event-stream" },
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to open run stream (${response.status})`);
|
||||
}
|
||||
for await (const event of readSSEStream(response)) {
|
||||
// ``readSSEStream`` yields the parsed JSON object directly; our run
|
||||
// events aren't part of the chat ``SSEEvent`` union, hence the cast.
|
||||
const data = event as unknown as ScraperRunEvent;
|
||||
if (data && typeof data === "object" && typeof data.type === "string") {
|
||||
yield data;
|
||||
if (data.type === "run.finished") return;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export const scrapersApiService = new ScrapersApiService();
|
||||
|
|
@ -32,6 +32,16 @@ const PUBLIC_ROUTE_PREFIXES = [
|
|||
"/privacy",
|
||||
"/terms",
|
||||
"/changelog",
|
||||
"/announcements",
|
||||
// Connector marketing pages (see lib/connectors-marketing)
|
||||
"/connectors",
|
||||
"/mcp-server",
|
||||
"/external-mcp-connectors",
|
||||
"/reddit",
|
||||
"/youtube",
|
||||
"/google-maps",
|
||||
"/google-search",
|
||||
"/web-crawl",
|
||||
];
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -20,33 +20,43 @@ export interface ChatExampleCategory {
|
|||
|
||||
export const CHAT_EXAMPLE_CATEGORIES: ChatExampleCategory[] = [
|
||||
{
|
||||
id: "search",
|
||||
label: "Search & Summarize",
|
||||
id: "monitor",
|
||||
label: "Monitor Competitors",
|
||||
prompts: [
|
||||
"Summarize the key points across all the documents in this space",
|
||||
"What do my files say about [topic]? Answer with citations",
|
||||
"Find every mention of [keyword] and list the sources",
|
||||
"Compare these two documents and highlight the differences",
|
||||
"Extract every plan, price, and limit from [competitor]'s pricing page",
|
||||
"Crawl [competitor]'s changelog and brief me on what they shipped this month",
|
||||
"Track who ranks and runs ads for [keyword] in the US",
|
||||
"Which of our target keywords trigger an AI Overview, and who gets cited?",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "create",
|
||||
label: "Create",
|
||||
id: "listen",
|
||||
label: "Market Listening",
|
||||
prompts: [
|
||||
"Write a cited research report on [topic] from my documents",
|
||||
"Turn this folder into a two-host podcast I can listen to",
|
||||
"Create a slide deck and a narrated video overview from these sources",
|
||||
"Tailor my resume to this job description so it gets past ATS and lands an interview",
|
||||
"Find 20 Reddit posts where people ask for an alternative to [product]",
|
||||
"Analyze the comments on [channel]'s last 10 videos and cluster the complaints",
|
||||
"Pull the Google Maps reviews for [business] and summarize the top complaints",
|
||||
"Find people actively looking to switch away from [competitor] this month",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "workflows",
|
||||
label: "Multi-Connector",
|
||||
prompts: [
|
||||
"Measure the reaction to [competitor]'s launch across search, Reddit, and YouTube",
|
||||
"Find the top-rated [category] businesses in [city], crawl their sites, and build a lead list with contacts",
|
||||
"Map who ranks for [keyword], crawl each result, and compare their pricing in one table",
|
||||
"Build a 360 on [competitor]: site changes, rank movements, Reddit sentiment, and YouTube reaction",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "automate",
|
||||
label: "Automate",
|
||||
prompts: [
|
||||
"Email me a daily brief of new documents in my knowledge base every morning",
|
||||
"When a PDF lands in my Research folder, generate a cited AI summary",
|
||||
"Generate a weekly status report from my Slack and Gmail every Friday",
|
||||
"Build an automation that turns new meeting notes into minutes with action items",
|
||||
"Re-check [competitor]'s pricing page daily and alert me on any change",
|
||||
"Every Monday, crawl our competitors' changelogs and send me a brief",
|
||||
"Track our brand mentions on Reddit daily and tag buying intent",
|
||||
"Send me a weekly report on [keyword] rankings and AI Overview citations",
|
||||
],
|
||||
},
|
||||
{
|
||||
|
|
@ -55,7 +65,7 @@ export const CHAT_EXAMPLE_CATEGORIES: ChatExampleCategory[] = [
|
|||
prompts: [
|
||||
"Search across my Notion, Slack, Google Drive and Gmail for [topic]",
|
||||
"Post this research summary to my Notion workspace",
|
||||
"Send these meeting action items to our team Slack channel",
|
||||
"Send these findings to our team Slack channel",
|
||||
"Create a Jira ticket from this bug report",
|
||||
],
|
||||
},
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ export const googleMaps: ConnectorPageContent = {
|
|||
"google maps scraper python",
|
||||
],
|
||||
|
||||
h1: "Google Maps API for Lead Generation and Local Market Intelligence",
|
||||
h1: "Google Maps Scraper API for Lead Generation and Local Market Intelligence",
|
||||
heroLede:
|
||||
"The SurfSense Google Maps API turns Maps into a B2B lead engine. Extract business names, categories, phones, websites, ratings, and reviews at scale, then point your AI agents at any territory or category to build lead lists and map a local market in minutes.",
|
||||
|
||||
|
|
@ -305,7 +305,7 @@ export const googleMaps: ConnectorPageContent = {
|
|||
{ label: "YouTube API", href: "/youtube" },
|
||||
{ label: "SERP API", href: "/google-search" },
|
||||
{ label: "Web Crawl API", href: "/web-crawl" },
|
||||
{ label: "MCP Connector", href: "/mcp-connector" },
|
||||
{ label: "SurfSense MCP Server", href: "/mcp-server" },
|
||||
{ label: "Read the docs", href: "/docs" },
|
||||
],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ export const googleSearch: ConnectorPageContent = {
|
|||
{ label: "Web Crawl API", href: "/web-crawl" },
|
||||
{ label: "Google Maps API", href: "/google-maps" },
|
||||
{ label: "Reddit API", href: "/reddit" },
|
||||
{ label: "MCP Connector", href: "/mcp-connector" },
|
||||
{ label: "SurfSense MCP Server", href: "/mcp-server" },
|
||||
{ label: "Read the docs", href: "/docs" },
|
||||
],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ export const reddit: ConnectorPageContent = {
|
|||
"brand monitoring tool",
|
||||
],
|
||||
|
||||
h1: "Reddit API for Brand Monitoring and Market Research",
|
||||
h1: "Reddit Scraper API for Brand Monitoring and Market Research",
|
||||
heroLede:
|
||||
"The SurfSense Reddit API extracts posts, comments, subreddits, and user data without the official Reddit API's rate limits or per-call pricing. Give your AI agents a live feed of what your market says about your brand, your competitors, and your category, so you hear it first.",
|
||||
|
||||
|
|
@ -313,7 +313,7 @@ export const reddit: ConnectorPageContent = {
|
|||
{ label: "Google Maps API", href: "/google-maps" },
|
||||
{ label: "SERP API", href: "/google-search" },
|
||||
{ label: "Web Crawl API", href: "/web-crawl" },
|
||||
{ label: "MCP Connector", href: "/mcp-connector" },
|
||||
{ label: "SurfSense MCP Server", href: "/mcp-server" },
|
||||
{ label: "Read the docs", href: "/docs" },
|
||||
],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -7,11 +7,12 @@ export const webCrawl: ConnectorPageContent = {
|
|||
cardTitle: "Web Crawl API",
|
||||
icon: IconWorldWww,
|
||||
|
||||
metaTitle: "Web Crawl API for AI Agents and Deep Research | SurfSense",
|
||||
metaTitle: "Web Scraping and Crawl API for AI Agents | SurfSense",
|
||||
metaDescription:
|
||||
"Point your agents at any URL and get clean, LLM-ready markdown, links, and contacts back. The SurfSense Web Crawl API for deep research and competitor monitoring. Start free.",
|
||||
keywords: [
|
||||
"web crawl api",
|
||||
"web scraping api",
|
||||
"web crawler api",
|
||||
"web crawling api",
|
||||
"firecrawl alternative",
|
||||
|
|
@ -281,7 +282,7 @@ export const webCrawl: ConnectorPageContent = {
|
|||
{ label: "SERP API", href: "/google-search" },
|
||||
{ label: "Google Maps API", href: "/google-maps" },
|
||||
{ label: "Reddit API", href: "/reddit" },
|
||||
{ label: "MCP Connector", href: "/mcp-connector" },
|
||||
{ label: "SurfSense MCP Server", href: "/mcp-server" },
|
||||
{ label: "Read the docs", href: "/docs" },
|
||||
],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ export const youtube: ConnectorPageContent = {
|
|||
"audience research tools",
|
||||
],
|
||||
|
||||
h1: "YouTube API for Audience Sentiment and Content Intelligence",
|
||||
h1: "YouTube Scraper API for Audience Sentiment and Content Intelligence",
|
||||
heroLede:
|
||||
"The SurfSense YouTube API pulls videos, channel stats, comments, and transcripts at scale, without YouTube Data API quotas. Feed your AI agents every comment and transcript that matters to your market, so you know what people actually think about your product and your competitors' content.",
|
||||
|
||||
|
|
@ -273,7 +273,7 @@ export const youtube: ConnectorPageContent = {
|
|||
{ label: "Google Maps API", href: "/google-maps" },
|
||||
{ label: "SERP API", href: "/google-search" },
|
||||
{ label: "Web Crawl API", href: "/web-crawl" },
|
||||
{ label: "MCP Connector", href: "/mcp-connector" },
|
||||
{ label: "SurfSense MCP Server", href: "/mcp-server" },
|
||||
{ label: "Read the docs", href: "/docs" },
|
||||
],
|
||||
};
|
||||
|
|
|
|||
80
surfsense_web/lib/playground/catalog.ts
Normal file
80
surfsense_web/lib/playground/catalog.ts
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
import {
|
||||
IconBrandGoogle,
|
||||
IconBrandGoogleMaps,
|
||||
IconBrandReddit,
|
||||
IconBrandYoutube,
|
||||
IconWorldWww,
|
||||
} from "@tabler/icons-react";
|
||||
import type { ComponentType } from "react";
|
||||
|
||||
/** Icon component that accepts a ``className`` (Tabler + Lucide both satisfy this). */
|
||||
export type PlatformIcon = ComponentType<{ className?: string }>;
|
||||
|
||||
/**
|
||||
* Static catalog of the platform-native scraper verbs, used to render the
|
||||
* playground navigation (sidebar + index grid) without waiting on a fetch.
|
||||
* Verb ``name`` mirrors the backend capability registry ``platform.verb`` and
|
||||
* maps directly to the REST path ``/scrapers/{platform}/{verb}``.
|
||||
*/
|
||||
export interface PlaygroundVerb {
|
||||
/** Capability name, e.g. ``reddit.scrape``. */
|
||||
name: string;
|
||||
/** URL segment, e.g. ``scrape``. */
|
||||
verb: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface PlaygroundPlatform {
|
||||
/** URL segment + REST platform, e.g. ``google_maps``. */
|
||||
id: string;
|
||||
label: string;
|
||||
icon: PlatformIcon;
|
||||
verbs: PlaygroundVerb[];
|
||||
}
|
||||
|
||||
export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [
|
||||
{
|
||||
id: "reddit",
|
||||
label: "Reddit",
|
||||
icon: IconBrandReddit,
|
||||
verbs: [{ name: "reddit.scrape", verb: "scrape", label: "Scrape" }],
|
||||
},
|
||||
{
|
||||
id: "youtube",
|
||||
label: "YouTube",
|
||||
icon: IconBrandYoutube,
|
||||
verbs: [
|
||||
{ name: "youtube.scrape", verb: "scrape", label: "Scrape" },
|
||||
{ name: "youtube.comments", verb: "comments", label: "Comments" },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "google_maps",
|
||||
label: "Google Maps",
|
||||
icon: IconBrandGoogleMaps,
|
||||
verbs: [
|
||||
{ name: "google_maps.scrape", verb: "scrape", label: "Scrape" },
|
||||
{ name: "google_maps.reviews", verb: "reviews", label: "Reviews" },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "google_search",
|
||||
label: "Google Search",
|
||||
icon: IconBrandGoogle,
|
||||
verbs: [{ name: "google_search.scrape", verb: "scrape", label: "Scrape" }],
|
||||
},
|
||||
{
|
||||
id: "web",
|
||||
label: "Web",
|
||||
icon: IconWorldWww,
|
||||
verbs: [{ name: "web.crawl", verb: "crawl", label: "Crawl" }],
|
||||
},
|
||||
];
|
||||
|
||||
export function findPlatform(platformId: string): PlaygroundPlatform | undefined {
|
||||
return PLAYGROUND_PLATFORMS.find((p) => p.id === platformId);
|
||||
}
|
||||
|
||||
export function findVerb(platformId: string, verb: string): PlaygroundVerb | undefined {
|
||||
return findPlatform(platformId)?.verbs.find((v) => v.verb === verb);
|
||||
}
|
||||
50
surfsense_web/lib/playground/code-snippets.selfcheck.ts
Normal file
50
surfsense_web/lib/playground/code-snippets.selfcheck.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/**
|
||||
* Runnable self-check for the API-reference snippet generator. No test
|
||||
* framework — run with: ``npx tsx lib/playground/code-snippets.selfcheck.ts``
|
||||
* Exits non-zero on the first failed assertion.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { buildExamplePayload, buildSnippets } from "./code-snippets";
|
||||
import type { FormField } from "./json-schema";
|
||||
|
||||
const fields: FormField[] = [
|
||||
{ name: "urls", title: "Urls", kind: "string_array", required: true },
|
||||
{ name: "sort", title: "Sort", kind: "enum", required: false, default: "new" },
|
||||
{ name: "limit", title: "Limit", kind: "integer", required: false },
|
||||
{ name: "query", title: "Query", kind: "string", required: true },
|
||||
];
|
||||
|
||||
// Required fields get placeholders, optional-with-default keeps the default,
|
||||
// optional-without-default is omitted.
|
||||
const payload = buildExamplePayload(fields);
|
||||
assert.deepEqual(payload, {
|
||||
urls: ["<urls>"],
|
||||
sort: "new",
|
||||
query: "<query>",
|
||||
});
|
||||
|
||||
const snippets = buildSnippets("https://api.example.com", "/api/v1/workspaces/1/scrapers/x/y", payload);
|
||||
|
||||
// Every popular language is present.
|
||||
assert.deepEqual(
|
||||
snippets.map((s) => s.id),
|
||||
["curl", "python", "javascript", "typescript", "go", "java", "csharp", "php", "ruby"]
|
||||
);
|
||||
|
||||
for (const snippet of snippets) {
|
||||
// Each snippet targets the right endpoint and authenticates with the env key.
|
||||
assert.ok(
|
||||
snippet.code.includes("https://api.example.com/api/v1/workspaces/1/scrapers/x/y"),
|
||||
`${snippet.id}: missing url`
|
||||
);
|
||||
assert.ok(snippet.code.includes("SURFSENSE_API_KEY"), `${snippet.id}: missing auth env var`);
|
||||
// The payload made it into the body (PHP re-renders it as an array literal).
|
||||
const marker = snippet.id === "php" ? '"urls" =>' : '"urls"';
|
||||
assert.ok(snippet.code.includes(marker), `${snippet.id}: missing payload`);
|
||||
}
|
||||
|
||||
// PHP array rendering handles nesting.
|
||||
const php = snippets.find((s) => s.id === "php");
|
||||
assert.ok(php?.code.includes('"sort" => "new"'), "php: enum default not rendered");
|
||||
|
||||
console.log("code-snippets.selfcheck: all assertions passed");
|
||||
251
surfsense_web/lib/playground/code-snippets.ts
Normal file
251
surfsense_web/lib/playground/code-snippets.ts
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
/**
|
||||
* Code-example generators for the playground API reference. One snippet per
|
||||
* popular language, all derived from the same endpoint URL + example payload
|
||||
* so they can't drift from each other.
|
||||
*/
|
||||
|
||||
import type { FormField } from "./json-schema";
|
||||
|
||||
export interface CodeSnippet {
|
||||
id: string;
|
||||
label: string;
|
||||
code: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Example request payload derived from the verb's parsed input fields:
|
||||
* required fields always appear (default or a named placeholder), optional
|
||||
* fields appear only when they carry a server default worth showing.
|
||||
*/
|
||||
export function buildExamplePayload(fields: FormField[]): Record<string, unknown> {
|
||||
const example: Record<string, unknown> = {};
|
||||
for (const field of fields) {
|
||||
if (field.default !== undefined && field.default !== null) {
|
||||
example[field.name] = field.default;
|
||||
continue;
|
||||
}
|
||||
if (!field.required) continue;
|
||||
switch (field.kind) {
|
||||
case "string_array":
|
||||
example[field.name] = [`<${field.name}>`];
|
||||
break;
|
||||
case "integer":
|
||||
case "number":
|
||||
example[field.name] = field.minimum ?? 1;
|
||||
break;
|
||||
case "boolean":
|
||||
example[field.name] = false;
|
||||
break;
|
||||
case "enum":
|
||||
example[field.name] = field.enumValues?.[0] ?? `<${field.name}>`;
|
||||
break;
|
||||
default:
|
||||
example[field.name] = `<${field.name}>`;
|
||||
}
|
||||
}
|
||||
return example;
|
||||
}
|
||||
|
||||
function indent(text: string, spaces: number): string {
|
||||
const pad = " ".repeat(spaces);
|
||||
return text
|
||||
.split("\n")
|
||||
.map((line, i) => (i === 0 ? line : pad + line))
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
export function buildSnippets(
|
||||
baseUrl: string,
|
||||
path: string,
|
||||
payload: Record<string, unknown>
|
||||
): CodeSnippet[] {
|
||||
const url = `${baseUrl}${path}`;
|
||||
const body = JSON.stringify(payload, null, 2);
|
||||
|
||||
const curl = `curl -X POST "${url}" \\
|
||||
-H "Authorization: Bearer $SURFSENSE_API_KEY" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d '${body.replace(/'/g, "'\\''")}'`;
|
||||
|
||||
const python = `import os
|
||||
import requests
|
||||
|
||||
response = requests.post(
|
||||
"${url}",
|
||||
headers={"Authorization": f"Bearer {os.environ['SURFSENSE_API_KEY']}"},
|
||||
json=${indent(body, 4)},
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
print(data["items"])`;
|
||||
|
||||
const javascript = `const response = await fetch("${url}", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: \`Bearer \${process.env.SURFSENSE_API_KEY}\`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(${indent(body, 2)}),
|
||||
});
|
||||
if (!response.ok) throw new Error(\`HTTP \${response.status}\`);
|
||||
const data = await response.json();
|
||||
console.log(data.items);`;
|
||||
|
||||
const typescript = `interface RunResult {
|
||||
items: Record<string, unknown>[];
|
||||
}
|
||||
|
||||
const response = await fetch("${url}", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: \`Bearer \${process.env.SURFSENSE_API_KEY}\`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(${indent(body, 2)}),
|
||||
});
|
||||
if (!response.ok) throw new Error(\`HTTP \${response.status}\`);
|
||||
const data = (await response.json()) as RunResult;
|
||||
console.log(data.items);`;
|
||||
|
||||
const go = `package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
payload := []byte(\`${body}\`)
|
||||
|
||||
req, err := http.NewRequest("POST", "${url}", bytes.NewBuffer(payload))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+os.Getenv("SURFSENSE_API_KEY"))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
data, _ := io.ReadAll(resp.Body)
|
||||
fmt.Println(resp.Status, string(data))
|
||||
}`;
|
||||
|
||||
const java = `import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
|
||||
public class SurfSenseExample {
|
||||
public static void main(String[] args) throws Exception {
|
||||
String payload = """
|
||||
${indent(body, 8)}
|
||||
""";
|
||||
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create("${url}"))
|
||||
.header("Authorization", "Bearer " + System.getenv("SURFSENSE_API_KEY"))
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(payload))
|
||||
.build();
|
||||
|
||||
HttpResponse<String> response = HttpClient.newHttpClient()
|
||||
.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
System.out.println(response.statusCode() + " " + response.body());
|
||||
}
|
||||
}`;
|
||||
|
||||
const csharp = `using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
|
||||
var payload = """
|
||||
${indent(body, 4)}
|
||||
""";
|
||||
|
||||
using var client = new HttpClient();
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(
|
||||
"Bearer", Environment.GetEnvironmentVariable("SURFSENSE_API_KEY"));
|
||||
|
||||
var response = await client.PostAsync(
|
||||
"${url}",
|
||||
new StringContent(payload, Encoding.UTF8, "application/json"));
|
||||
response.EnsureSuccessStatusCode();
|
||||
Console.WriteLine(await response.Content.ReadAsStringAsync());`;
|
||||
|
||||
const php = `<?php
|
||||
$payload = json_encode(${indent(phpArray(payload), 0)});
|
||||
|
||||
$ch = curl_init("${url}");
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_POSTFIELDS => $payload,
|
||||
CURLOPT_HTTPHEADER => [
|
||||
"Authorization: Bearer " . getenv("SURFSENSE_API_KEY"),
|
||||
"Content-Type: application/json",
|
||||
],
|
||||
]);
|
||||
|
||||
$response = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
|
||||
$data = json_decode($response, true);
|
||||
print_r($data["items"]);`;
|
||||
|
||||
const ruby = `require "net/http"
|
||||
require "json"
|
||||
|
||||
uri = URI("${url}")
|
||||
request = Net::HTTP::Post.new(uri)
|
||||
request["Authorization"] = "Bearer #{ENV["SURFSENSE_API_KEY"]}"
|
||||
request["Content-Type"] = "application/json"
|
||||
request.body = <<~JSON
|
||||
${indent(body, 2).replace(/^/, " ")}
|
||||
JSON
|
||||
|
||||
response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
|
||||
http.request(request)
|
||||
end
|
||||
data = JSON.parse(response.body)
|
||||
puts data["items"]`;
|
||||
|
||||
return [
|
||||
{ id: "curl", label: "cURL", code: curl },
|
||||
{ id: "python", label: "Python", code: python },
|
||||
{ id: "javascript", label: "JavaScript", code: javascript },
|
||||
{ id: "typescript", label: "TypeScript", code: typescript },
|
||||
{ id: "go", label: "Go", code: go },
|
||||
{ id: "java", label: "Java", code: java },
|
||||
{ id: "csharp", label: "C#", code: csharp },
|
||||
{ id: "php", label: "PHP", code: php },
|
||||
{ id: "ruby", label: "Ruby", code: ruby },
|
||||
];
|
||||
}
|
||||
|
||||
/** Render a JSON-ish value as PHP array syntax for the PHP snippet. */
|
||||
function phpArray(value: unknown, depth = 0): string {
|
||||
const pad = " ".repeat(depth + 1);
|
||||
const closePad = " ".repeat(depth);
|
||||
if (Array.isArray(value)) {
|
||||
if (value.length === 0) return "[]";
|
||||
const items = value.map((v) => `${pad}${phpArray(v, depth + 1)}`).join(",\n");
|
||||
return `[\n${items},\n${closePad}]`;
|
||||
}
|
||||
if (typeof value === "object" && value !== null) {
|
||||
const entries = Object.entries(value as Record<string, unknown>);
|
||||
if (entries.length === 0) return "[]";
|
||||
const items = entries
|
||||
.map(([k, v]) => `${pad}${JSON.stringify(k)} => ${phpArray(v, depth + 1)}`)
|
||||
.join(",\n");
|
||||
return `[\n${items},\n${closePad}]`;
|
||||
}
|
||||
if (typeof value === "string") return JSON.stringify(value);
|
||||
if (value === null) return "null";
|
||||
return String(value);
|
||||
}
|
||||
42
surfsense_web/lib/playground/csv.selfcheck.ts
Normal file
42
surfsense_web/lib/playground/csv.selfcheck.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
/**
|
||||
* Runnable self-check for the playground CSV serializer. No test framework —
|
||||
* run with: ``npx tsx lib/playground/csv.selfcheck.ts``
|
||||
* Exits non-zero on the first failed assertion.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { rowsToCsv } from "./csv";
|
||||
|
||||
// Union of keys, stable order, header + rows joined with CRLF.
|
||||
assert.equal(
|
||||
rowsToCsv([
|
||||
{ a: 1, b: "x" },
|
||||
{ a: 2, c: true },
|
||||
]),
|
||||
"a,b,c\r\n1,x,\r\n2,,true"
|
||||
);
|
||||
|
||||
// Explicit columns fix order and subset.
|
||||
assert.equal(rowsToCsv([{ a: 1, b: 2, c: 3 }], ["c", "a"]), "c,a\r\n3,1");
|
||||
|
||||
// RFC 4180 quoting: commas, quotes, and newlines.
|
||||
assert.equal(rowsToCsv([{ v: 'he said "hi", bye' }]), 'v\r\n"he said ""hi"", bye"');
|
||||
assert.equal(rowsToCsv([{ v: "line1\nline2" }]), 'v\r\n"line1\nline2"');
|
||||
|
||||
// Objects/arrays are JSON-stringified (then quoted for the comma).
|
||||
assert.equal(rowsToCsv([{ v: { k: 1 } }]), 'v\r\n"{""k"":1}"');
|
||||
|
||||
// Null/undefined become empty cells.
|
||||
assert.equal(rowsToCsv([{ v: null }]), "v\r\n");
|
||||
|
||||
// Formula-injection guard: dangerous leads get a ' prefix, real numbers don't.
|
||||
assert.equal(rowsToCsv([{ v: "=1+2" }]), "v\r\n'=1+2");
|
||||
assert.equal(rowsToCsv([{ v: "+1-800" }]), "v\r\n'+1-800");
|
||||
assert.equal(rowsToCsv([{ v: "@handle" }]), "v\r\n'@handle");
|
||||
assert.equal(rowsToCsv([{ v: "-cmd" }]), "v\r\n'-cmd");
|
||||
assert.equal(rowsToCsv([{ v: "-5" }]), "v\r\n-5");
|
||||
assert.equal(rowsToCsv([{ v: -5 }]), "v\r\n-5");
|
||||
|
||||
// Header only when there are no rows.
|
||||
assert.equal(rowsToCsv([], ["a", "b"]), "a,b");
|
||||
|
||||
console.log("csv.selfcheck: all assertions passed");
|
||||
58
surfsense_web/lib/playground/csv.ts
Normal file
58
surfsense_web/lib/playground/csv.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
/** CSV export helpers for the playground runs table and API output tables. */
|
||||
|
||||
function cellToString(value: unknown): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
if (typeof value === "object") return JSON.stringify(value);
|
||||
return String(value);
|
||||
}
|
||||
|
||||
// Leading chars a spreadsheet treats as the start of a formula. Scraped API
|
||||
// data is untrusted, so we neutralize these to prevent CSV injection.
|
||||
const FORMULA_START = /^[=+@\t\r]/;
|
||||
const LOOKS_NUMERIC = /^-?\d/;
|
||||
|
||||
function escapeCsvCell(value: unknown): string {
|
||||
let s = cellToString(value);
|
||||
if (FORMULA_START.test(s) || (s.startsWith("-") && !LOOKS_NUMERIC.test(s))) {
|
||||
s = `'${s}`;
|
||||
}
|
||||
if (/[",\n\r]/.test(s)) {
|
||||
s = `"${s.replace(/"/g, '""')}"`;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize rows to an RFC 4180 CSV string. Columns default to the union of
|
||||
* keys across all rows (stable insertion order); pass ``columns`` to fix the
|
||||
* order and subset.
|
||||
*/
|
||||
export function rowsToCsv(rows: Record<string, unknown>[], columns?: string[]): string {
|
||||
const cols =
|
||||
columns ??
|
||||
Array.from(
|
||||
rows.reduce((set, row) => {
|
||||
for (const key of Object.keys(row)) set.add(key);
|
||||
return set;
|
||||
}, new Set<string>())
|
||||
);
|
||||
const header = cols.map(escapeCsvCell).join(",");
|
||||
if (rows.length === 0) return header;
|
||||
const body = rows.map((row) => cols.map((col) => escapeCsvCell(row[col])).join(",")).join("\r\n");
|
||||
return `${header}\r\n${body}`;
|
||||
}
|
||||
|
||||
/** Trigger a browser download of ``csv`` as ``{filenameBase}.csv`` (UTF-8, Excel-friendly). */
|
||||
export function downloadCsv(filenameBase: string, csv: string): void {
|
||||
const safe = filenameBase.replace(/[^\w.-]+/g, "_") || "export";
|
||||
// Prepend a BOM so Excel detects UTF-8.
|
||||
const blob = new Blob([`\uFEFF${csv}`], { type: "text/csv;charset=utf-8" });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = `${safe}.csv`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
13
surfsense_web/lib/playground/format.ts
Normal file
13
surfsense_web/lib/playground/format.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/** Formatting helpers shared by the playground runs table and runner output. */
|
||||
|
||||
export function formatCost(costMicros: number | null | undefined): string {
|
||||
if (costMicros == null) return "—";
|
||||
if (costMicros === 0) return "Free";
|
||||
return `$${(costMicros / 1_000_000).toFixed(4)}`;
|
||||
}
|
||||
|
||||
export function formatDuration(ms: number | null | undefined): string {
|
||||
if (ms == null) return "—";
|
||||
if (ms < 1000) return `${ms}ms`;
|
||||
return `${(ms / 1000).toFixed(1)}s`;
|
||||
}
|
||||
79
surfsense_web/lib/playground/json-schema.selfcheck.ts
Normal file
79
surfsense_web/lib/playground/json-schema.selfcheck.ts
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* Runnable self-check for the playground schema parser. No test framework —
|
||||
* run with: ``npx tsx lib/playground/json-schema.selfcheck.ts``
|
||||
* Exits non-zero on the first failed assertion.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { buildPayload, initialFormValues, parseSchemaFields } from "./json-schema";
|
||||
|
||||
// Mirrors the pydantic v2 ``model_json_schema()`` output for a reddit-like verb:
|
||||
// a required string[], an enum with a default (allOf + default), an optional
|
||||
// string (anyOf: [str, null]), and an integer with bounds + default.
|
||||
const schema = {
|
||||
$defs: {
|
||||
RedditSort: { enum: ["new", "hot", "top"], title: "RedditSort", type: "string" },
|
||||
},
|
||||
properties: {
|
||||
urls: { items: { type: "string" }, type: "array", title: "Urls", description: "Target URLs" },
|
||||
sort: { allOf: [{ $ref: "#/$defs/RedditSort" }], default: "new" },
|
||||
community: { anyOf: [{ type: "string" }, { type: "null" }], default: null, title: "Community" },
|
||||
max_items: { type: "integer", minimum: 1, maximum: 100, default: 25, title: "Max Items" },
|
||||
skip_comments: { type: "boolean", default: false, title: "Skip Comments" },
|
||||
},
|
||||
required: ["urls"],
|
||||
title: "ScrapeInput",
|
||||
type: "object",
|
||||
};
|
||||
|
||||
const fields = parseSchemaFields(schema);
|
||||
const byName = Object.fromEntries(fields.map((f) => [f.name, f]));
|
||||
|
||||
// string[] detection
|
||||
assert.equal(byName.urls.kind, "string_array");
|
||||
assert.equal(byName.urls.required, true);
|
||||
assert.equal(byName.urls.description, "Target URLs");
|
||||
|
||||
// enum resolved through allOf + $ref, default preserved
|
||||
assert.equal(byName.sort.kind, "enum");
|
||||
assert.deepEqual(byName.sort.enumValues, ["new", "hot", "top"]);
|
||||
assert.equal(byName.sort.default, "new");
|
||||
assert.equal(byName.sort.required, false);
|
||||
|
||||
// optional via anyOf: [str, null] -> string
|
||||
assert.equal(byName.community.kind, "string");
|
||||
assert.equal(byName.community.required, false);
|
||||
|
||||
// integer bounds
|
||||
assert.equal(byName.max_items.kind, "integer");
|
||||
assert.equal(byName.max_items.minimum, 1);
|
||||
assert.equal(byName.max_items.maximum, 100);
|
||||
|
||||
// boolean
|
||||
assert.equal(byName.skip_comments.kind, "boolean");
|
||||
|
||||
// defaults seed the form (string_array default would be joined, none here)
|
||||
const initial = initialFormValues(fields);
|
||||
assert.equal(initial.urls, ""); // required, no default
|
||||
assert.equal(initial.sort, "new");
|
||||
assert.equal(initial.max_items, 25);
|
||||
assert.equal(initial.skip_comments, false);
|
||||
|
||||
// payload building: split lines, coerce number/bool, omit empty optionals
|
||||
const payload = buildPayload(fields, {
|
||||
urls: "https://a.com\n https://b.com \n\n",
|
||||
sort: "top",
|
||||
community: "",
|
||||
max_items: "50",
|
||||
skip_comments: true,
|
||||
});
|
||||
assert.deepEqual(payload.urls, ["https://a.com", "https://b.com"]);
|
||||
assert.equal(payload.sort, "top");
|
||||
assert.equal("community" in payload, false); // empty optional omitted
|
||||
assert.equal(payload.max_items, 50);
|
||||
assert.equal(payload.skip_comments, true);
|
||||
|
||||
// unknown / empty schema is safe
|
||||
assert.deepEqual(parseSchemaFields(undefined), []);
|
||||
assert.deepEqual(parseSchemaFields({ type: "object" }), []);
|
||||
|
||||
console.log("json-schema self-check passed");
|
||||
184
surfsense_web/lib/playground/json-schema.ts
Normal file
184
surfsense_web/lib/playground/json-schema.ts
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
/**
|
||||
* Minimal JSON-Schema reader for the playground form. It only understands the
|
||||
* shapes pydantic v2 ``model_json_schema()`` actually emits for the scraper
|
||||
* verbs: primitives, ``string[]``, enums (via ``$defs`` + ``$ref``/``allOf``),
|
||||
* and optionals (``anyOf: [T, null]``). Anything unrecognized falls back to a
|
||||
* free-text string field so the form never silently drops an input.
|
||||
*/
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
|
||||
export type FieldKind =
|
||||
| "string"
|
||||
| "string_array"
|
||||
| "integer"
|
||||
| "number"
|
||||
| "boolean"
|
||||
| "enum";
|
||||
|
||||
export interface FormField {
|
||||
name: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
kind: FieldKind;
|
||||
required: boolean;
|
||||
default?: unknown;
|
||||
enumValues?: string[];
|
||||
minimum?: number;
|
||||
maximum?: number;
|
||||
}
|
||||
|
||||
function isObject(value: unknown): value is JsonObject {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function resolveRef(ref: string, defs: JsonObject): JsonObject | undefined {
|
||||
// Only local ``#/$defs/Name`` refs are emitted by pydantic.
|
||||
const name = ref.split("/").pop();
|
||||
if (!name) return undefined;
|
||||
const target = defs[name];
|
||||
return isObject(target) ? target : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse ``$ref`` / ``allOf`` / ``anyOf`` wrappers down to the concrete
|
||||
* subschema that carries ``type``/``enum``. Returns the original node if it is
|
||||
* already concrete.
|
||||
*/
|
||||
function resolveNode(node: JsonObject, defs: JsonObject, depth = 0): JsonObject {
|
||||
if (depth > 5) return node;
|
||||
|
||||
if (typeof node.$ref === "string") {
|
||||
const target = resolveRef(node.$ref, defs);
|
||||
return target ? resolveNode(target, defs, depth + 1) : node;
|
||||
}
|
||||
|
||||
if (Array.isArray(node.allOf) && node.allOf.length > 0 && isObject(node.allOf[0])) {
|
||||
return resolveNode(node.allOf[0], defs, depth + 1);
|
||||
}
|
||||
|
||||
if (Array.isArray(node.anyOf)) {
|
||||
const branch = node.anyOf.find(
|
||||
(entry): entry is JsonObject => isObject(entry) && entry.type !== "null"
|
||||
);
|
||||
if (branch) return resolveNode(branch, defs, depth + 1);
|
||||
}
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
function toEnumValues(node: JsonObject): string[] | undefined {
|
||||
if (!Array.isArray(node.enum)) return undefined;
|
||||
return node.enum.map((value) => String(value));
|
||||
}
|
||||
|
||||
function detectKind(node: JsonObject): FieldKind {
|
||||
if (Array.isArray(node.enum)) return "enum";
|
||||
if (node.type === "array") {
|
||||
const items = isObject(node.items) ? node.items : undefined;
|
||||
// string[] is the only array shape used; other item types still render as
|
||||
// a line-per-value textarea, which is a reasonable fallback.
|
||||
void items;
|
||||
return "string_array";
|
||||
}
|
||||
if (node.type === "integer") return "integer";
|
||||
if (node.type === "number") return "number";
|
||||
if (node.type === "boolean") return "boolean";
|
||||
return "string";
|
||||
}
|
||||
|
||||
function readNumber(value: unknown): number | undefined {
|
||||
return typeof value === "number" ? value : undefined;
|
||||
}
|
||||
|
||||
/** Parse a full input schema into an ordered list of form fields. */
|
||||
export function parseSchemaFields(schema: JsonObject | undefined): FormField[] {
|
||||
if (!isObject(schema) || !isObject(schema.properties)) return [];
|
||||
const defs = isObject(schema.$defs) ? schema.$defs : {};
|
||||
const required = new Set(
|
||||
Array.isArray(schema.required) ? schema.required.map((r) => String(r)) : []
|
||||
);
|
||||
|
||||
return Object.entries(schema.properties).map(([name, rawProp]) => {
|
||||
const prop = isObject(rawProp) ? rawProp : {};
|
||||
const resolved = resolveNode(prop, defs);
|
||||
const kind = detectKind(resolved);
|
||||
|
||||
return {
|
||||
name,
|
||||
title: typeof prop.title === "string" ? prop.title : name,
|
||||
description:
|
||||
typeof prop.description === "string"
|
||||
? prop.description
|
||||
: typeof resolved.description === "string"
|
||||
? resolved.description
|
||||
: undefined,
|
||||
kind,
|
||||
required: required.has(name),
|
||||
default: "default" in prop ? prop.default : undefined,
|
||||
enumValues: toEnumValues(resolved),
|
||||
minimum: readNumber(resolved.minimum) ?? readNumber(prop.minimum),
|
||||
maximum: readNumber(resolved.maximum) ?? readNumber(prop.maximum),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Build the initial form value map from field defaults. */
|
||||
export function initialFormValues(fields: FormField[]): Record<string, unknown> {
|
||||
const values: Record<string, unknown> = {};
|
||||
for (const field of fields) {
|
||||
if (field.default !== undefined && field.default !== null) {
|
||||
values[field.name] =
|
||||
field.kind === "string_array" && Array.isArray(field.default)
|
||||
? field.default.join("\n")
|
||||
: field.default;
|
||||
} else if (field.kind === "boolean") {
|
||||
values[field.name] = false;
|
||||
} else {
|
||||
values[field.name] = "";
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert the form's string-ish state into a typed request payload. Empty
|
||||
* optional fields are omitted so backend defaults apply.
|
||||
*/
|
||||
export function buildPayload(
|
||||
fields: FormField[],
|
||||
values: Record<string, unknown>
|
||||
): Record<string, unknown> {
|
||||
const payload: Record<string, unknown> = {};
|
||||
|
||||
for (const field of fields) {
|
||||
const raw = values[field.name];
|
||||
|
||||
if (field.kind === "boolean") {
|
||||
payload[field.name] = Boolean(raw);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (field.kind === "string_array") {
|
||||
const lines = String(raw ?? "")
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
if (lines.length > 0) payload[field.name] = lines;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (field.kind === "integer" || field.kind === "number") {
|
||||
if (raw === "" || raw === undefined || raw === null) continue;
|
||||
const num = Number(raw);
|
||||
if (!Number.isNaN(num)) payload[field.name] = num;
|
||||
continue;
|
||||
}
|
||||
|
||||
// string + enum
|
||||
if (raw === "" || raw === undefined || raw === null) continue;
|
||||
payload[field.name] = raw;
|
||||
}
|
||||
|
||||
return payload;
|
||||
}
|
||||
|
|
@ -111,6 +111,13 @@ export const cacheKeys = {
|
|||
batchUnreadCounts: (workspaceId: number | null) =>
|
||||
["notifications", "unread-counts-batch", workspaceId] as const,
|
||||
},
|
||||
scrapers: {
|
||||
capabilities: (workspaceId: number | string) =>
|
||||
["scrapers", "capabilities", workspaceId] as const,
|
||||
runs: (workspaceId: number | string) => ["scrapers", "runs", workspaceId] as const,
|
||||
run: (workspaceId: number | string, runId: string) =>
|
||||
["scrapers", "run", workspaceId, runId] as const,
|
||||
},
|
||||
automations: {
|
||||
// list endpoint is keyed by pagination too so distinct pages don't collide
|
||||
list: (workspaceId: number, limit: number, offset: number) =>
|
||||
|
|
|
|||
|
|
@ -11,6 +11,13 @@ const withNextIntl = createNextIntlPlugin("./i18n/request.ts");
|
|||
// ships what desktop users actually need.
|
||||
const nextConfig: NextConfig = {
|
||||
output: "standalone",
|
||||
async redirects() {
|
||||
return [
|
||||
// /mcp-connector was split into the two MCP directions; the external
|
||||
// (client) side kept the original content.
|
||||
{ source: "/mcp-connector", destination: "/external-mcp-connectors", permanent: true },
|
||||
];
|
||||
},
|
||||
outputFileTracingRoot: path.join(__dirname, ".."),
|
||||
reactStrictMode: false,
|
||||
typescript: {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue