From a7c3b2ae63e67959655282968166c122ed3a3da4 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 5 Jul 2026 21:49:30 -0700 Subject: [PATCH] feat(schema): add API schema definitions for connectors - Introduced a new schema structure for API connectors, including request and response fields. - Updated connector implementations for Google Maps, Google Search, Reddit, Web Crawl, and YouTube to include detailed schema information. - Enhanced the ConnectorPage component to render request and response schemas dynamically. - Added SchemaField interface to standardize schema field definitions across connectors. --- .../connectors-marketing/connector-page.tsx | 80 ++++++++++- .../lib/connectors-marketing/google-maps.tsx | 129 +++++++++++++++++ .../connectors-marketing/google-search.tsx | 77 ++++++++++ .../lib/connectors-marketing/reddit.tsx | 132 ++++++++++++++++++ .../lib/connectors-marketing/types.ts | 27 ++++ .../lib/connectors-marketing/web-crawl.tsx | 97 +++++++++++++ .../lib/connectors-marketing/youtube.tsx | 95 +++++++++++++ 7 files changed, 636 insertions(+), 1 deletion(-) diff --git a/surfsense_web/components/connectors-marketing/connector-page.tsx b/surfsense_web/components/connectors-marketing/connector-page.tsx index 27fce9977..6e92585ab 100644 --- a/surfsense_web/components/connectors-marketing/connector-page.tsx +++ b/surfsense_web/components/connectors-marketing/connector-page.tsx @@ -6,7 +6,7 @@ import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Separator } from "@/components/ui/separator"; -import type { ConnectorPageContent } from "@/lib/connectors-marketing/types"; +import type { ConnectorPageContent, SchemaField } from "@/lib/connectors-marketing/types"; import { AgentTranscript } from "./agent-transcript"; import { ApiMcpTabs } from "./api-mcp-tabs"; import { ConnectorFaq } from "./connector-faq"; @@ -14,6 +14,46 @@ import { Reveal } from "./reveal"; const GITHUB_URL = "https://github.com/MODSetter/SurfSense"; +function SchemaTable({ caption, fields }: { caption: string; fields: SchemaField[] }) { + return ( +
+ + + + + + + + + + + {fields.map((field) => ( + + + + + + ))} + +
{caption}
FieldTypeDescription
+ {field.name} + + {field.type} + {field.required ? ( + + required + + ) : null} + {field.defaultValue !== undefined ? ( +
+ default {field.defaultValue} +
+ ) : null} + 		{field.description}
+
+ ); +} + export function ConnectorPage({ content }: { content: ConnectorPageContent }) { const Icon = content.icon; const label = content.cardTitle ?? `${content.name} API`; @@ -136,6 +176,44 @@ export function ConnectorPage({ content }: { content: ConnectorPageContent }) { + {/* Request / response schema */} + + +

+ {content.name} API request and response schema +

+

+ The exact contract behind{" "} + + POST /workspaces/{"{workspace_id}"}/scrapers/{content.api.platform}/{content.api.verb} + + . The same fields power the{" "} + + {content.api.mcpTool} + {" "} + MCP tool. +

+ + +

Request parameters

+

+ {content.schema.requestNote} +

+
+ +
+ + +

Response fields

+

+ {content.schema.responseNote} +

+
+ +
+ + + {/* Comparison */} diff --git a/surfsense_web/lib/connectors-marketing/google-maps.tsx b/surfsense_web/lib/connectors-marketing/google-maps.tsx index 026733bc3..c80dd3099 100644 --- a/surfsense_web/lib/connectors-marketing/google-maps.tsx +++ b/surfsense_web/lib/connectors-marketing/google-maps.tsx @@ -148,6 +148,135 @@ export const googleMaps: ConnectorPageContent = { }, }, + schema: { + requestNote: + "Provide at least one source: search_queries, urls, or place_ids. Up to 20 sources per call.", + request: [ + { + name: "search_queries", + type: "string[]", + defaultValue: "[]", + description: + "Google Maps search terms, e.g. 'coffee shops', 'dentist'. Each returns up to max_places. Pair with location to scope the search. Max 20.", + }, + { + name: "urls", + type: "string[]", + defaultValue: "[]", + description: + "Google Maps URLs: a place page (/maps/place/...) or a search results URL. Max 20.", + }, + { + name: "place_ids", + type: "string[]", + defaultValue: "[]", + description: "Known Google place IDs (ChIJ...) to fetch directly. Max 20.", + }, + { + name: "location", + type: "string", + description: "Location to scope search_queries to, e.g. 'New York, USA'.", + }, + { + name: "max_places", + type: "integer", + defaultValue: "10", + description: "Max places to return per search query. 1 to 1,000.", + }, + { + name: "language", + type: "string", + defaultValue: '"en"', + description: "Result language code, e.g. 'en', 'fr'.", + }, + { + name: "include_details", + type: "boolean", + defaultValue: "false", + description: + "Also fetch each place's detail page: opening hours, popular times, and extra contact info. Slower.", + }, + { + name: "max_reviews", + type: "integer", + defaultValue: "0", + description: "Reviews to attach per place, up to 100,000. 0 disables reviews.", + }, + { + name: "max_images", + type: "integer", + defaultValue: "0", + description: "Images to attach per place. 0 disables images.", + }, + ], + responseNote: + "The response is { items: [...] } with one item per place. One returned place is one billable unit; attached reviews are metered separately.", + response: [ + { + name: "title / categoryName / categories", + type: "string / string[]", + description: "Business name and its Google Maps categories.", + }, + { + name: "placeId / cid / url", + type: "string", + description: "Stable Google identifiers and the place's Maps URL.", + }, + { + name: "address / street / city / state / postalCode / countryCode", + type: "string", + description: "Full and structured address components.", + }, + { + name: "location", + type: "object", + description: "Coordinates: { lat, lng }.", + }, + { + name: "website / phone", + type: "string", + description: + "Contact details as listed on the profile. Null website is a classic lead-gen signal.", + }, + { + name: "totalScore / reviewsCount / reviewsDistribution", + type: "number / integer / object", + description: "Average rating, review count, and the one-to-five-star breakdown.", + }, + { + name: "permanentlyClosed / temporarilyClosed", + type: "boolean", + description: "Business status flags.", + }, + { + name: "openingHours", + type: "object[]", + description: "Day-by-day hours. Populated when include_details is true.", + }, + { + name: "reviews", + type: "object[]", + description: + "Attached reviews when max_reviews > 0: text, stars, publishedAtDate, likesCount, reviewer info, and the owner's response.", + }, + { + name: "images / imageUrl / imagesCount", + type: "object[] / string / integer", + description: "Photos attached when max_images > 0, plus the cover image and count.", + }, + { + name: "searchString / rank", + type: "string / integer", + description: "Provenance: which query found this place and its position in the results.", + }, + { + name: "scrapedAt", + type: "string", + description: "ISO timestamp for when the place was scraped.", + }, + ], + }, + faq: [ { question: "Is scraping Google Maps legal?", diff --git a/surfsense_web/lib/connectors-marketing/google-search.tsx b/surfsense_web/lib/connectors-marketing/google-search.tsx index 94253a032..c01149a50 100644 --- a/surfsense_web/lib/connectors-marketing/google-search.tsx +++ b/surfsense_web/lib/connectors-marketing/google-search.tsx @@ -156,6 +156,83 @@ export const googleSearch: ConnectorPageContent = { }, }, + schema: { + requestNote: "Only queries is required. Up to 20 queries per call.", + request: [ + { + name: "queries", + type: "string[]", + required: true, + description: + "Search terms (e.g. 'wedding photographers denver') or full Google Search URLs. Each term is searched; each URL is scraped as-is. 1 to 20.", + }, + { + name: "max_pages_per_query", + type: "integer", + defaultValue: "1", + description: "Result pages to fetch per query, 1 to 10. 1 fetches the first page only.", + }, + { + name: "country_code", + type: "string", + description: "Two-letter country to search from, e.g. 'us', 'fr'.", + }, + { + name: "language_code", + type: "string", + defaultValue: '""', + description: "Result language code, e.g. 'en', 'fr'. Blank uses Google's default.", + }, + { + name: "site", + type: "string", + description: "Restrict results to a single domain, e.g. 'example.com'.", + }, + ], + responseNote: + "The response is { items: [...] } with one item per fetched SERP page. One fetched page is one billable unit.", + response: [ + { + name: "searchQuery", + type: "object", + description: + "Provenance for this page: the term or URL searched, page number, device, country, and language.", + }, + { + name: "resultsTotal", + type: "integer", + description: "Google's estimated total result count for the query.", + }, + { + name: "organicResults", + type: "object[]", + description: + "The organic listings: title, url, displayedUrl, description, date, emphasizedKeywords, siteLinks, and position.", + }, + { + name: "paidResults / paidProducts", + type: "object[]", + description: "Ads and shopping placements on the page, with titles, URLs, and prices.", + }, + { + name: "relatedQueries", + type: "object[]", + description: "The 'related searches' block: title and search URL for each suggestion.", + }, + { + name: "peopleAlsoAsk", + type: "object[]", + description: "People Also Ask entries with the question, answer text, and source page URL.", + }, + { + name: "aiOverview", + type: "object", + description: + "The AI Overview block when Google shows one: full answer content plus the sources it cites.", + }, + ], + }, + faq: [ { question: "What is a SERP API?", diff --git a/surfsense_web/lib/connectors-marketing/reddit.tsx b/surfsense_web/lib/connectors-marketing/reddit.tsx index eddaec3c9..c9ce8e79a 100644 --- a/surfsense_web/lib/connectors-marketing/reddit.tsx +++ b/surfsense_web/lib/connectors-marketing/reddit.tsx @@ -153,6 +153,138 @@ export const reddit: ConnectorPageContent = { }, }, + schema: { + requestNote: + "Provide at least one source: urls, search_queries, or community. Up to 20 sources per call.", + request: [ + { + name: "urls", + type: "string[]", + defaultValue: "[]", + description: + "Reddit URLs to scrape: a post, a subreddit (/r/name), a user (/user/name), or a search URL. Max 20.", + }, + { + name: "search_queries", + type: "string[]", + defaultValue: "[]", + description: + "Search terms to run on Reddit. Each returns up to max_items results. Scope to one subreddit with community. Max 20.", + }, + { + name: "community", + type: "string", + description: + "Subreddit name without the r/ prefix, e.g. 'python'. Scopes search_queries to that subreddit; with no search_queries, its listing is scraped.", + }, + { + name: "sort", + type: "string", + defaultValue: '"new"', + description: "Result ordering: relevance, hot, top, new, rising, or comments.", + }, + { + name: "time_filter", + type: "string", + description: "Time window for top sorts: hour, day, week, month, year, or all.", + }, + { + name: "include_nsfw", + type: "boolean", + defaultValue: "true", + description: "Include posts flagged over-18 in the results.", + }, + { + name: "skip_comments", + type: "boolean", + defaultValue: "false", + description: "Skip fetching comment trees. Faster when you only need posts or listings.", + }, + { + name: "max_items", + type: "integer", + defaultValue: "10", + description: "Max total items to return across all sources. 1 to 100.", + }, + { + name: "max_posts", + type: "integer", + defaultValue: "10", + description: "Max posts to pull per subreddit, user, or search target.", + }, + { + name: "max_comments", + type: "integer", + defaultValue: "10", + description: "Max comments to pull per post. 0 disables comments.", + }, + { + name: "post_date_limit", + type: "string", + description: "ISO date. Only return posts newer than this, for incremental scrapes.", + }, + { + name: "comment_date_limit", + type: "string", + description: "ISO date. Only return comments newer than this, for incremental scrapes.", + }, + ], + responseNote: + "The response is { items: [...] } with one flat item per result, keyed by dataType. Fields that do not apply to a given dataType are null. One returned item is one billable unit.", + response: [ + { + name: "dataType", + type: "string", + description: "What this item is: post, comment, community, or user.", + }, + { + name: "id / url / username", + type: "string", + description: "Identity and provenance: the Reddit ID, permalink, and author username.", + }, + { + name: "title / body", + type: "string", + description: "Post title and full text body (or comment text for comments).", + }, + { + name: "communityName", + type: "string", + description: "The subreddit the item belongs to, plus numberOfMembers for communities.", + }, + { + name: "upVotes / upVoteRatio", + type: "integer / number", + description: "Score and upvote ratio, the engagement signal for ranking what matters.", + }, + { + name: "numberOfComments", + type: "integer", + description: "Comment count on a post; numberOfReplies for comments.", + }, + { + name: "flair / over18 / isVideo", + type: "string / boolean", + description: "Post flair and content flags.", + }, + { + name: "thumbnailUrl / imageUrls / videoUrls", + type: "string / string[]", + description: "Media attached to the post.", + }, + { + name: "postId / parentId", + type: "string", + description: "Threading for comments: the parent post and parent comment IDs.", + }, + { + name: "createdAt / scrapedAt", + type: "string", + description: "ISO timestamps for when the item was posted and when it was scraped.", + }, + ], + }, + faq: [ { question: "Is scraping Reddit legal?", diff --git a/surfsense_web/lib/connectors-marketing/types.ts b/surfsense_web/lib/connectors-marketing/types.ts index 40bc71179..a4d789efa 100644 --- a/surfsense_web/lib/connectors-marketing/types.ts +++ b/surfsense_web/lib/connectors-marketing/types.ts @@ -67,6 +67,32 @@ export interface ApiSample { requestBody: Record; } +/** One request parameter or response field in the schema reference tables. */ +export interface SchemaField { + name: string; + /** Short type label, e.g. "string[]", "integer", "boolean", "object[]". */ + type: string; + /** Rendered as a "required" pill next to the type. */ + required?: boolean; + /** Default value shown for optional request parameters, e.g. '"new"', "10". */ + defaultValue?: string; + description: string; +} + +/** + * The request/response contract rendered as the "API schema" section. Fields + * map 1:1 to the backend capability `ScrapeInput`/`ScrapeOutput` models so the + * reference is truthful. + */ +export interface ApiSchema { + /** One-liner above the request table (e.g. which sources are required). */ + requestNote: string; + request: SchemaField[]; + /** One-liner above the response table (envelope shape + billable unit). */ + responseNote: string; + response: SchemaField[]; +} + /** Everything needed to render one connector marketing page. */ export interface ConnectorPageContent { slug: string; @@ -99,6 +125,7 @@ export interface ConnectorPageContent { rows: ComparisonRow[]; }; api: ApiSample; + schema: ApiSchema; faq: FaqItem[]; related: RelatedLink[]; } diff --git a/surfsense_web/lib/connectors-marketing/web-crawl.tsx b/surfsense_web/lib/connectors-marketing/web-crawl.tsx index b4affe9db..efb35034e 100644 --- a/surfsense_web/lib/connectors-marketing/web-crawl.tsx +++ b/surfsense_web/lib/connectors-marketing/web-crawl.tsx @@ -157,6 +157,103 @@ export const webCrawl: ConnectorPageContent = { }, }, + schema: { + requestNote: + "Only startUrls is required. With maxCrawlDepth 0 the call scrapes exactly those URLs; with a higher depth it spiders the site from them.", + request: [ + { + name: "startUrls", + type: "string[]", + required: true, + description: + "Seed URLs to crawl, 1 to 20. With maxCrawlDepth 0 only these are fetched; otherwise they are the spider's entry points.", + }, + { + name: "maxCrawlDepth", + type: "integer", + defaultValue: "0", + description: + "Link-hops to follow from each start URL, 0 to 5. 0 scrapes only the start URLs; 1 also fetches their linked pages. The spider stays on the start URL's site.", + }, + { + name: "maxCrawlPages", + type: "integer", + defaultValue: "10", + description: + "Max pages to fetch in total, start URLs included, 1 to 200. The crawl stops at this ceiling.", + }, + { + name: "maxLength", + type: "integer", + defaultValue: "50000", + description: "Max characters of cleaned markdown kept per page. Longer pages truncate.", + }, + { + name: "includeUrlPatterns", + type: "string[]", + defaultValue: "[]", + description: + "Regex patterns a discovered link must match to be followed. Empty follows every same-site link. Start URLs are always fetched. Max 25.", + }, + { + name: "excludeUrlPatterns", + type: "string[]", + defaultValue: "[]", + description: + "Regex patterns that exclude a discovered link from being followed. Wins over includeUrlPatterns. Max 25.", + }, + ], + responseNote: + "The response is { items: [...], contacts: {...} }: one item per fetched page in crawl order, plus site-wide deduplicated contacts. Only successful pages are billed.", + response: [ + { + name: "items[].url / status", + type: "string", + description: + "The requested URL and its outcome: success, empty (fetched but no content), or failed.", + }, + { + name: "items[].markdown", + type: "string", + description: "The page's content as cleaned markdown, ready for an LLM or a diff.", + }, + { + name: "items[].metadata", + type: "object", + description: "Page metadata such as title and description.", + }, + { + name: "items[].crawl", + type: "object", + description: + "Crawl provenance: the URL actually loaded, its link depth, and the referrer page it was discovered on.", + }, + { + name: "items[].links", + type: "object[]", + description: + "Every link on the page with its anchor text and kind: internal, external, social, email, or tel.", + }, + { + name: "items[].contacts", + type: "object", + description: + "Emails, phones, and social profile URLs harvested from the page's raw HTML, including footer boilerplate the markdown omits.", + }, + { + name: "items[].error", + type: "string", + description: "Failure reason when status is not success.", + }, + { + name: "contacts", + type: "object", + description: + "Site-wide contact rollup: every email, phone, and social URL deduplicated across pages, each with the pages it appeared on and a siteWide flag separating company boilerplate from page-local finds like team members.", + }, + ], + }, + faq: [ { question: "What formats does the Web Crawl API return?", diff --git a/surfsense_web/lib/connectors-marketing/youtube.tsx b/surfsense_web/lib/connectors-marketing/youtube.tsx index 0a211caea..ca4b4579f 100644 --- a/surfsense_web/lib/connectors-marketing/youtube.tsx +++ b/surfsense_web/lib/connectors-marketing/youtube.tsx @@ -150,6 +150,101 @@ export const youtube: ConnectorPageContent = { }, }, + schema: { + requestNote: "Provide at least one source: urls or search_queries. Up to 20 of each per call.", + request: [ + { + name: "urls", + type: "string[]", + defaultValue: "[]", + description: + "YouTube URLs to scrape: a video, channel (/@handle or /channel/UC...), playlist (?list=...), shorts, or hashtag page. Max 20.", + }, + { + name: "search_queries", + type: "string[]", + defaultValue: "[]", + description: + "Search terms to run on YouTube. Each returns up to max_results videos. Max 20.", + }, + { + name: "max_results", + type: "integer", + defaultValue: "10", + description: + "Max items per source and per content type, 1 to 1,000. For a channel, videos, shorts, and streams are capped independently.", + }, + { + name: "download_subtitles", + type: "boolean", + defaultValue: "false", + description: "Also fetch each video's subtitle track. Slower.", + }, + { + name: "subtitles_language", + type: "string", + defaultValue: '"en"', + description: + "Subtitle language code, e.g. 'en', 'fr'. Used when download_subtitles is true.", + }, + ], + responseNote: + "The response is { items: [...] } with one item per video, short, or stream. One returned item is one billable unit.", + response: [ + { + name: "title / id / url", + type: "string", + description: "Video title, YouTube ID, and watch URL.", + }, + { + name: "type", + type: "string", + description: "What this item is: video, short, or stream.", + }, + { + name: "viewCount / likes / commentsCount", + type: "integer", + description: "Engagement metrics for ranking what resonates.", + }, + { + name: "date / duration", + type: "string", + description: "Publish date and video length.", + }, + { + name: "text / descriptionLinks / hashtags", + type: "string / object[] / string[]", + description: "Full description, the links it contains, and its hashtags.", + }, + { + name: "subtitles", + type: "object[]", + description: + "Full transcript tracks with SRT content when download_subtitles is true. The raw material for content analysis.", + }, + { + name: "thumbnailUrl", + type: "string", + description: "The video's thumbnail image.", + }, + { + name: "channelName / channelUrl / channelId", + type: "string", + description: "The publishing channel's name, URL, and ID.", + }, + { + name: "numberOfSubscribers / isChannelVerified", + type: "integer / boolean", + description: "Channel reach and verification status.", + }, + { + name: "input / fromYTUrl / order", + type: "string / integer", + description: "Provenance: which source produced this item and its position.", + }, + ], + }, + faq: [ { question: "Is scraping YouTube legal?",