feat(schema): add API schema definitions for connectors

- Introduced a new schema structure for API connectors, including request and response fields.
- Updated connector implementations for Google Maps, Google Search, Reddit, Web Crawl, and YouTube to include detailed schema information.
- Enhanced the ConnectorPage component to render request and response schemas dynamically.
- Added SchemaField interface to standardize schema field definitions across connectors.

surfsense_web/components/connectors-marketing/connector-page.tsx

@@ -6,7 +6,7 @@ import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Separator } from "@/components/ui/separator";
-import type { ConnectorPageContent } from "@/lib/connectors-marketing/types";
+import type { ConnectorPageContent, SchemaField } from "@/lib/connectors-marketing/types";
 import { AgentTranscript } from "./agent-transcript";
 import { ApiMcpTabs } from "./api-mcp-tabs";
 import { ConnectorFaq } from "./connector-faq";
@@ -14,6 +14,46 @@ import { Reveal } from "./reveal";
 
 const GITHUB_URL = "https://github.com/MODSetter/SurfSense";
 
+function SchemaTable({ caption, fields }: { caption: string; fields: SchemaField[] }) {
+	return (
+		<div className="overflow-x-auto rounded-xl border bg-card">
+			<table className="w-full min-w-xl text-sm">
+				<caption className="sr-only">{caption}</caption>
+				<thead>
+					<tr className="border-b bg-muted/40 text-left">
+						<th className="p-4 font-medium">Field</th>
+						<th className="p-4 font-medium">Type</th>
+						<th className="p-4 font-medium">Description</th>
+					</tr>
+				</thead>
+				<tbody>
+					{fields.map((field) => (
+						<tr key={field.name} className="border-b align-top last:border-b-0">
+							<th scope="row" className="p-4 text-left">
+								<code className="font-mono text-[13px] font-semibold">{field.name}</code>
+							</th>
+							<td className="whitespace-nowrap p-4">
+								<code className="font-mono text-[13px] text-muted-foreground">{field.type}</code>
+								{field.required ? (
+									<span className="ml-2 rounded-full bg-brand/10 px-2 py-0.5 text-xs font-medium text-brand">
+										required
+									</span>
+								) : null}
+								{field.defaultValue !== undefined ? (
+									<div className="mt-1 text-xs text-muted-foreground">
+										default <code className="font-mono">{field.defaultValue}</code>
+									</div>
+								) : null}
+							</td>
+							<td className="p-4 text-muted-foreground leading-relaxed">{field.description}</td>
+						</tr>
+					))}
+				</tbody>
+			</table>
+		</div>
+	);
+}
+
 export function ConnectorPage({ content }: { content: ConnectorPageContent }) {
 	const Icon = content.icon;
 	const label = content.cardTitle ?? `${content.name} API`;
@@ -136,6 +176,44 @@ export function ConnectorPage({ content }: { content: ConnectorPageContent }) {
 				</Reveal>
 			</MarketingSection>
 
+			{/* Request / response schema */}
+			<MarketingSection>
+				<Reveal>
+					<h2 className="text-2xl font-bold tracking-tight sm:text-3xl">
+						{content.name} API request and response schema
+					</h2>
+					<p className="mt-3 max-w-2xl text-muted-foreground leading-relaxed">
+						The exact contract behind{" "}
+						<code className="rounded bg-muted px-1.5 py-0.5 font-mono text-sm">
+							POST /workspaces/{"{workspace_id}"}/scrapers/{content.api.platform}/{content.api.verb}
+						</code>
+						. The same fields power the{" "}
+						<code className="rounded bg-muted px-1.5 py-0.5 font-mono text-sm">
+							{content.api.mcpTool}
+						</code>{" "}
+						MCP tool.
+					</p>
+				</Reveal>
+				<Reveal>
+					<h3 className="mt-8 text-lg font-semibold">Request parameters</h3>
+					<p className="mt-2 max-w-2xl text-sm text-muted-foreground leading-relaxed">
+						{content.schema.requestNote}
+					</p>
+					<div className="mt-4">
+						<SchemaTable caption="Request parameters" fields={content.schema.request} />
+					</div>
+				</Reveal>
+				<Reveal>
+					<h3 className="mt-10 text-lg font-semibold">Response fields</h3>
+					<p className="mt-2 max-w-2xl text-sm text-muted-foreground leading-relaxed">
+						{content.schema.responseNote}
+					</p>
+					<div className="mt-4">
+						<SchemaTable caption="Response fields" fields={content.schema.response} />
+					</div>
+				</Reveal>
+			</MarketingSection>
+
 			{/* Comparison */}
 			<MarketingSection>
 				<Reveal>

surfsense_web/lib/connectors-marketing/google-maps.tsx

@@ -148,6 +148,135 @@ export const googleMaps: ConnectorPageContent = {
 		},
 	},
 
+	schema: {
+		requestNote:
+			"Provide at least one source: search_queries, urls, or place_ids. Up to 20 sources per call.",
+		request: [
+			{
+				name: "search_queries",
+				type: "string[]",
+				defaultValue: "[]",
+				description:
+					"Google Maps search terms, e.g. 'coffee shops', 'dentist'. Each returns up to max_places. Pair with location to scope the search. Max 20.",
+			},
+			{
+				name: "urls",
+				type: "string[]",
+				defaultValue: "[]",
+				description:
+					"Google Maps URLs: a place page (/maps/place/...) or a search results URL. Max 20.",
+			},
+			{
+				name: "place_ids",
+				type: "string[]",
+				defaultValue: "[]",
+				description: "Known Google place IDs (ChIJ...) to fetch directly. Max 20.",
+			},
+			{
+				name: "location",
+				type: "string",
+				description: "Location to scope search_queries to, e.g. 'New York, USA'.",
+			},
+			{
+				name: "max_places",
+				type: "integer",
+				defaultValue: "10",
+				description: "Max places to return per search query. 1 to 1,000.",
+			},
+			{
+				name: "language",
+				type: "string",
+				defaultValue: '"en"',
+				description: "Result language code, e.g. 'en', 'fr'.",
+			},
+			{
+				name: "include_details",
+				type: "boolean",
+				defaultValue: "false",
+				description:
+					"Also fetch each place's detail page: opening hours, popular times, and extra contact info. Slower.",
+			},
+			{
+				name: "max_reviews",
+				type: "integer",
+				defaultValue: "0",
+				description: "Reviews to attach per place, up to 100,000. 0 disables reviews.",
+			},
+			{
+				name: "max_images",
+				type: "integer",
+				defaultValue: "0",
+				description: "Images to attach per place. 0 disables images.",
+			},
+		],
+		responseNote:
+			"The response is { items: [...] } with one item per place. One returned place is one billable unit; attached reviews are metered separately.",
+		response: [
+			{
+				name: "title / categoryName / categories",
+				type: "string / string[]",
+				description: "Business name and its Google Maps categories.",
+			},
+			{
+				name: "placeId / cid / url",
+				type: "string",
+				description: "Stable Google identifiers and the place's Maps URL.",
+			},
+			{
+				name: "address / street / city / state / postalCode / countryCode",
+				type: "string",
+				description: "Full and structured address components.",
+			},
+			{
+				name: "location",
+				type: "object",
+				description: "Coordinates: { lat, lng }.",
+			},
+			{
+				name: "website / phone",
+				type: "string",
+				description:
+					"Contact details as listed on the profile. Null website is a classic lead-gen signal.",
+			},
+			{
+				name: "totalScore / reviewsCount / reviewsDistribution",
+				type: "number / integer / object",
+				description: "Average rating, review count, and the one-to-five-star breakdown.",
+			},
+			{
+				name: "permanentlyClosed / temporarilyClosed",
+				type: "boolean",
+				description: "Business status flags.",
+			},
+			{
+				name: "openingHours",
+				type: "object[]",
+				description: "Day-by-day hours. Populated when include_details is true.",
+			},
+			{
+				name: "reviews",
+				type: "object[]",
+				description:
+					"Attached reviews when max_reviews > 0: text, stars, publishedAtDate, likesCount, reviewer info, and the owner's response.",
+			},
+			{
+				name: "images / imageUrl / imagesCount",
+				type: "object[] / string / integer",
+				description: "Photos attached when max_images > 0, plus the cover image and count.",
+			},
+			{
+				name: "searchString / rank",
+				type: "string / integer",
+				description: "Provenance: which query found this place and its position in the results.",
+			},
+			{
+				name: "scrapedAt",
+				type: "string",
+				description: "ISO timestamp for when the place was scraped.",
+			},
+		],
+	},
+
 	faq: [
 		{
 			question: "Is scraping Google Maps legal?",

surfsense_web/lib/connectors-marketing/google-search.tsx

@@ -156,6 +156,83 @@ export const googleSearch: ConnectorPageContent = {
 		},
 	},
 
+	schema: {
+		requestNote: "Only queries is required. Up to 20 queries per call.",
+		request: [
+			{
+				name: "queries",
+				type: "string[]",
+				required: true,
+				description:
+					"Search terms (e.g. 'wedding photographers denver') or full Google Search URLs. Each term is searched; each URL is scraped as-is. 1 to 20.",
+			},
+			{
+				name: "max_pages_per_query",
+				type: "integer",
+				defaultValue: "1",
+				description: "Result pages to fetch per query, 1 to 10. 1 fetches the first page only.",
+			},
+			{
+				name: "country_code",
+				type: "string",
+				description: "Two-letter country to search from, e.g. 'us', 'fr'.",
+			},
+			{
+				name: "language_code",
+				type: "string",
+				defaultValue: '""',
+				description: "Result language code, e.g. 'en', 'fr'. Blank uses Google's default.",
+			},
+			{
+				name: "site",
+				type: "string",
+				description: "Restrict results to a single domain, e.g. 'example.com'.",
+			},
+		],
+		responseNote:
+			"The response is { items: [...] } with one item per fetched SERP page. One fetched page is one billable unit.",
+		response: [
+			{
+				name: "searchQuery",
+				type: "object",
+				description:
+					"Provenance for this page: the term or URL searched, page number, device, country, and language.",
+			},
+			{
+				name: "resultsTotal",
+				type: "integer",
+				description: "Google's estimated total result count for the query.",
+			},
+			{
+				name: "organicResults",
+				type: "object[]",
+				description:
+					"The organic listings: title, url, displayedUrl, description, date, emphasizedKeywords, siteLinks, and position.",
+			},
+			{
+				name: "paidResults / paidProducts",
+				type: "object[]",
+				description: "Ads and shopping placements on the page, with titles, URLs, and prices.",
+			},
+			{
+				name: "relatedQueries",
+				type: "object[]",
+				description: "The 'related searches' block: title and search URL for each suggestion.",
+			},
+			{
+				name: "peopleAlsoAsk",
+				type: "object[]",
+				description: "People Also Ask entries with the question, answer text, and source page URL.",
+			},
+			{
+				name: "aiOverview",
+				type: "object",
+				description:
+					"The AI Overview block when Google shows one: full answer content plus the sources it cites.",
+			},
+		],
+	},
+
 	faq: [
 		{
 			question: "What is a SERP API?",

surfsense_web/lib/connectors-marketing/reddit.tsx

@@ -153,6 +153,138 @@ export const reddit: ConnectorPageContent = {
 		},
 	},
 
+	schema: {
+		requestNote:
+			"Provide at least one source: urls, search_queries, or community. Up to 20 sources per call.",
+		request: [
+			{
+				name: "urls",
+				type: "string[]",
+				defaultValue: "[]",
+				description:
+					"Reddit URLs to scrape: a post, a subreddit (/r/name), a user (/user/name), or a search URL. Max 20.",
+			},
+			{
+				name: "search_queries",
+				type: "string[]",
+				defaultValue: "[]",
+				description:
+					"Search terms to run on Reddit. Each returns up to max_items results. Scope to one subreddit with community. Max 20.",
+			},
+			{
+				name: "community",
+				type: "string",
+				description:
+					"Subreddit name without the r/ prefix, e.g. 'python'. Scopes search_queries to that subreddit; with no search_queries, its listing is scraped.",
+			},
+			{
+				name: "sort",
+				type: "string",
+				defaultValue: '"new"',
+				description: "Result ordering: relevance, hot, top, new, rising, or comments.",
+			},
+			{
+				name: "time_filter",
+				type: "string",
+				description: "Time window for top sorts: hour, day, week, month, year, or all.",
+			},
+			{
+				name: "include_nsfw",
+				type: "boolean",
+				defaultValue: "true",
+				description: "Include posts flagged over-18 in the results.",
+			},
+			{
+				name: "skip_comments",
+				type: "boolean",
+				defaultValue: "false",
+				description: "Skip fetching comment trees. Faster when you only need posts or listings.",
+			},
+			{
+				name: "max_items",
+				type: "integer",
+				defaultValue: "10",
+				description: "Max total items to return across all sources. 1 to 100.",
+			},
+			{
+				name: "max_posts",
+				type: "integer",
+				defaultValue: "10",
+				description: "Max posts to pull per subreddit, user, or search target.",
+			},
+			{
+				name: "max_comments",
+				type: "integer",
+				defaultValue: "10",
+				description: "Max comments to pull per post. 0 disables comments.",
+			},
+			{
+				name: "post_date_limit",
+				type: "string",
+				description: "ISO date. Only return posts newer than this, for incremental scrapes.",
+			},
+			{
+				name: "comment_date_limit",
+				type: "string",
+				description: "ISO date. Only return comments newer than this, for incremental scrapes.",
+			},
+		],
+		responseNote:
+			"The response is { items: [...] } with one flat item per result, keyed by dataType. Fields that do not apply to a given dataType are null. One returned item is one billable unit.",
+		response: [
+			{
+				name: "dataType",
+				type: "string",
+				description: "What this item is: post, comment, community, or user.",
+			},
+			{
+				name: "id / url / username",
+				type: "string",
+				description: "Identity and provenance: the Reddit ID, permalink, and author username.",
+			},
+			{
+				name: "title / body",
+				type: "string",
+				description: "Post title and full text body (or comment text for comments).",
+			},
+			{
+				name: "communityName",
+				type: "string",
+				description: "The subreddit the item belongs to, plus numberOfMembers for communities.",
+			},
+			{
+				name: "upVotes / upVoteRatio",
+				type: "integer / number",
+				description: "Score and upvote ratio, the engagement signal for ranking what matters.",
+			},
+			{
+				name: "numberOfComments",
+				type: "integer",
+				description: "Comment count on a post; numberOfReplies for comments.",
+			},
+			{
+				name: "flair / over18 / isVideo",
+				type: "string / boolean",
+				description: "Post flair and content flags.",
+			},
+			{
+				name: "thumbnailUrl / imageUrls / videoUrls",
+				type: "string / string[]",
+				description: "Media attached to the post.",
+			},
+			{
+				name: "postId / parentId",
+				type: "string",
+				description: "Threading for comments: the parent post and parent comment IDs.",
+			},
+			{
+				name: "createdAt / scrapedAt",
+				type: "string",
+				description: "ISO timestamps for when the item was posted and when it was scraped.",
+			},
+		],
+	},
+
 	faq: [
 		{
 			question: "Is scraping Reddit legal?",

surfsense_web/lib/connectors-marketing/types.ts

@@ -67,6 +67,32 @@ export interface ApiSample {
 	requestBody: Record<string, unknown>;
 }
 
+/** One request parameter or response field in the schema reference tables. */
+export interface SchemaField {
+	name: string;
+	/** Short type label, e.g. "string[]", "integer", "boolean", "object[]". */
+	type: string;
+	/** Rendered as a "required" pill next to the type. */
+	required?: boolean;
+	/** Default value shown for optional request parameters, e.g. '"new"', "10". */
+	defaultValue?: string;
+	description: string;
+}
+
+/**
+ * The request/response contract rendered as the "API schema" section. Fields
+ * map 1:1 to the backend capability `ScrapeInput`/`ScrapeOutput` models so the
+ * reference is truthful.
+ */
+export interface ApiSchema {
+	/** One-liner above the request table (e.g. which sources are required). */
+	requestNote: string;
+	request: SchemaField[];
+	/** One-liner above the response table (envelope shape + billable unit). */
+	responseNote: string;
+	response: SchemaField[];
+}
+
 /** Everything needed to render one connector marketing page. */
 export interface ConnectorPageContent {
 	slug: string;
@@ -99,6 +125,7 @@ export interface ConnectorPageContent {
 		rows: ComparisonRow[];
 	};
 	api: ApiSample;
+	schema: ApiSchema;
 	faq: FaqItem[];
 	related: RelatedLink[];
 }

surfsense_web/lib/connectors-marketing/web-crawl.tsx

@@ -157,6 +157,103 @@ export const webCrawl: ConnectorPageContent = {
 		},
 	},
 
+	schema: {
+		requestNote:
+			"Only startUrls is required. With maxCrawlDepth 0 the call scrapes exactly those URLs; with a higher depth it spiders the site from them.",
+		request: [
+			{
+				name: "startUrls",
+				type: "string[]",
+				required: true,
+				description:
+					"Seed URLs to crawl, 1 to 20. With maxCrawlDepth 0 only these are fetched; otherwise they are the spider's entry points.",
+			},
+			{
+				name: "maxCrawlDepth",
+				type: "integer",
+				defaultValue: "0",
+				description:
+					"Link-hops to follow from each start URL, 0 to 5. 0 scrapes only the start URLs; 1 also fetches their linked pages. The spider stays on the start URL's site.",
+			},
+			{
+				name: "maxCrawlPages",
+				type: "integer",
+				defaultValue: "10",
+				description:
+					"Max pages to fetch in total, start URLs included, 1 to 200. The crawl stops at this ceiling.",
+			},
+			{
+				name: "maxLength",
+				type: "integer",
+				defaultValue: "50000",
+				description: "Max characters of cleaned markdown kept per page. Longer pages truncate.",
+			},
+			{
+				name: "includeUrlPatterns",
+				type: "string[]",
+				defaultValue: "[]",
+				description:
+					"Regex patterns a discovered link must match to be followed. Empty follows every same-site link. Start URLs are always fetched. Max 25.",
+			},
+			{
+				name: "excludeUrlPatterns",
+				type: "string[]",
+				defaultValue: "[]",
+				description:
+					"Regex patterns that exclude a discovered link from being followed. Wins over includeUrlPatterns. Max 25.",
+			},
+		],
+		responseNote:
+			"The response is { items: [...], contacts: {...} }: one item per fetched page in crawl order, plus site-wide deduplicated contacts. Only successful pages are billed.",
+		response: [
+			{
+				name: "items[].url / status",
+				type: "string",
+				description:
+					"The requested URL and its outcome: success, empty (fetched but no content), or failed.",
+			},
+			{
+				name: "items[].markdown",
+				type: "string",
+				description: "The page's content as cleaned markdown, ready for an LLM or a diff.",
+			},
+			{
+				name: "items[].metadata",
+				type: "object",
+				description: "Page metadata such as title and description.",
+			},
+			{
+				name: "items[].crawl",
+				type: "object",
+				description:
+					"Crawl provenance: the URL actually loaded, its link depth, and the referrer page it was discovered on.",
+			},
+			{
+				name: "items[].links",
+				type: "object[]",
+				description:
+					"Every link on the page with its anchor text and kind: internal, external, social, email, or tel.",
+			},
+			{
+				name: "items[].contacts",
+				type: "object",
+				description:
+					"Emails, phones, and social profile URLs harvested from the page's raw HTML, including footer boilerplate the markdown omits.",
+			},
+			{
+				name: "items[].error",
+				type: "string",
+				description: "Failure reason when status is not success.",
+			},
+			{
+				name: "contacts",
+				type: "object",
+				description:
+					"Site-wide contact rollup: every email, phone, and social URL deduplicated across pages, each with the pages it appeared on and a siteWide flag separating company boilerplate from page-local finds like team members.",
+			},
+		],
+	},
+
 	faq: [
 		{
 			question: "What formats does the Web Crawl API return?",

surfsense_web/lib/connectors-marketing/youtube.tsx

@@ -150,6 +150,101 @@ export const youtube: ConnectorPageContent = {
 		},
 	},
 
+	schema: {
+		requestNote: "Provide at least one source: urls or search_queries. Up to 20 of each per call.",
+		request: [
+			{
+				name: "urls",
+				type: "string[]",
+				defaultValue: "[]",
+				description:
+					"YouTube URLs to scrape: a video, channel (/@handle or /channel/UC...), playlist (?list=...), shorts, or hashtag page. Max 20.",
+			},
+			{
+				name: "search_queries",
+				type: "string[]",
+				defaultValue: "[]",
+				description:
+					"Search terms to run on YouTube. Each returns up to max_results videos. Max 20.",
+			},
+			{
+				name: "max_results",
+				type: "integer",
+				defaultValue: "10",
+				description:
+					"Max items per source and per content type, 1 to 1,000. For a channel, videos, shorts, and streams are capped independently.",
+			},
+			{
+				name: "download_subtitles",
+				type: "boolean",
+				defaultValue: "false",
+				description: "Also fetch each video's subtitle track. Slower.",
+			},
+			{
+				name: "subtitles_language",
+				type: "string",
+				defaultValue: '"en"',
+				description:
+					"Subtitle language code, e.g. 'en', 'fr'. Used when download_subtitles is true.",
+			},
+		],
+		responseNote:
+			"The response is { items: [...] } with one item per video, short, or stream. One returned item is one billable unit.",
+		response: [
+			{
+				name: "title / id / url",
+				type: "string",
+				description: "Video title, YouTube ID, and watch URL.",
+			},
+			{
+				name: "type",
+				type: "string",
+				description: "What this item is: video, short, or stream.",
+			},
+			{
+				name: "viewCount / likes / commentsCount",
+				type: "integer",
+				description: "Engagement metrics for ranking what resonates.",
+			},
+			{
+				name: "date / duration",
+				type: "string",
+				description: "Publish date and video length.",
+			},
+			{
+				name: "text / descriptionLinks / hashtags",
+				type: "string / object[] / string[]",
+				description: "Full description, the links it contains, and its hashtags.",
+			},
+			{
+				name: "subtitles",
+				type: "object[]",
+				description:
+					"Full transcript tracks with SRT content when download_subtitles is true. The raw material for content analysis.",
+			},
+			{
+				name: "thumbnailUrl",
+				type: "string",
+				description: "The video's thumbnail image.",
+			},
+			{
+				name: "channelName / channelUrl / channelId",
+				type: "string",
+				description: "The publishing channel's name, URL, and ID.",
+			},
+			{
+				name: "numberOfSubscribers / isChannelVerified",
+				type: "integer / boolean",
+				description: "Channel reach and verification status.",
+			},
+			{
+				name: "input / fromYTUrl / order",
+				type: "string / integer",
+				description: "Provenance: which source produced this item and its position.",
+			},
+		],
+	},
+
 	faq: [
 		{
 			question: "Is scraping YouTube legal?",