apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-08 22:22:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(marketing): CI-first homepage, connector API pages, and hero chat demo

Browse source 
- Homepage rebuilt around competitive-intelligence positioning: connector grid,
  how-it-works, use-case art, compare table, FAQ, and a scripted new-chat hero
  demo (typewriter prompt, agent timeline, streamed answer, /login on send)
- Marketing pages for the Reddit/YouTube/Maps/SERP/Web Crawl scrape APIs at
  /<slug>, a /connectors hub, and a bespoke /mcp-connector page
- Remove stale app/dashboard/[search_space_id] tree that broke the build
  (canonical route is [workspace_id])

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-05 20:28:27 -07:00
parent 80927a2872
commit fb5b0da816
137 changed files with 4161 additions and 16017 deletions
Download patch file Download diff file Expand all files Collapse all files

182
surfsense_web/lib/connectors-marketing/google-maps.tsx Normal file
View file

@ -0,0 +1,182 @@
import { IconMapPin } from "@tabler/icons-react";
import type { ConnectorPageContent } from "./types";
export const googleMaps: ConnectorPageContent = {
slug: "google-maps",
name: "Google Maps",
icon: IconMapPin,
metaTitle: "Google Maps Scraper API for Lead Generation | SurfSense",
metaDescription:
"Turn Google Maps into a B2B lead engine with the SurfSense Google Maps Scraper API. Extract business data, ratings, and reviews at scale for your AI agents. Start now.",
keywords: [
"google maps scraper",
"google maps scraper api",
"google maps data extraction",
"google maps lead generation",
"google maps lead scraper",
"businesses without websites",
"google maps reviews api",
"local business data api",
"google maps scraper python",
],
h1: "Google Maps API for Lead Generation and Local Market Intelligence",
heroLede:
"The SurfSense Google Maps API turns Maps into a B2B lead engine. Extract business names, categories, phones, websites, ratings, and reviews at scale, then point your AI agents at any territory or category to build lead lists and map a local market in minutes.",
transcript: {
prompt: "Find every dentist in Austin without a website",
toolCall:
'google_maps.scrape({ search_queries: ["dentist"], location: "Austin, TX",\n max_places: 100, include_details: true })',
rows: [
{
primary: "Barton Springs Family Dental",
secondary: "4.9 ★ · 212 reviews · (512) 555-0148",
tag: "no website",
},
{
primary: "Congress Ave Dentistry",
secondary: "4.7 ★ · 88 reviews · (512) 555-0192",
tag: "no website",
},
{
primary: "East Austin Smiles",
secondary: "4.8 ★ · 141 reviews · (512) 555-0175",
tag: "no website",
},
],
resultSummary: "100 places · 23 without a website · surfaced in 4.0s",
},
extractIntro:
"Give the API a search query plus a location, a Maps URL, or a known place ID. It returns one structured item per place, with reviews and images attached on request.",
extractFields: [
{
label: "Business identity",
description: "Name, category, description, and Google place ID for every result.",
},
{
label: "Contact",
description: "Phone number and website when Google lists them, ready for outreach lists.",
},
{
label: "Address and geo",
description: "Full address, neighborhood, city, postal code, and latitude/longitude.",
},
{
label: "Ratings",
description: "Star rating, review count, and the full one-to-five-star distribution.",
},
{
label: "Reviews",
description: "Attach up to the review count you set per place, with text, stars, and author.",
},
{
label: "Details",
description: "Opening hours, popular times, and images when you enable detail pages.",
},
],
useCasesHeading: "What teams do with the Google Maps API",
useCases: [
{
title: "B2B lead lists",
description:
"Point an agent at a category and a territory and get back a clean list of businesses with phone numbers, ratings, and websites, ready to load into your CRM or outreach tool.",
},
{
title: "Businesses without websites",
description:
"Filter results down to places Google lists with no website, the classic warm lead for agencies and web shops. It is the highest-converting local lead-gen angle there is.",
},
{
title: "Competitor and review analysis",
description:
"Pull a competitor's rating, review count, and review text to see exactly what their customers praise and complain about, then brief an agent to summarize the gaps you can win on.",
},
{
title: "Territory and market mapping",
description:
"Scrape an entire category across a city or region to size a market, spot white space, and plan coverage before your sales team ever picks up the phone.",
},
],
comparison: {
heading: "A Google Maps API alternative for lead gen",
intro:
"Google's official Places API is metered per call and returns only a handful of reviews. Here is how SurfSense compares for lead generation and market research.",
columnLabel: "Official Places API",
rows: [
{
feature: "Results per query",
official: "Paginated and capped; bulk pulls are painful",
surfsense: "Up to 1,000 places per search query",
},
{
feature: "Reviews",
official: "Only about five reviews per place",
surfsense: "Attach up to the review count you set, or use the reviews verb for depth",
},
{
feature: "Pricing",
official: "Metered per API call, per field mask",
surfsense: "Pay per place returned, with a free tier to start",
},
{
feature: "Setup",
official: "Google Cloud project, billing, and API key",
surfsense: "One API key, or add the MCP server to your agent",
},
{
feature: "Agent-ready",
official: "No; you build the harness yourself",
surfsense: "MCP server exposes google_maps.scrape as a native tool",
},
],
},
api: {
platform: "google_maps",
verb: "scrape",
mcpTool: "google_maps.scrape",
requestBody: {
search_queries: ["dentist"],
location: "Austin, TX",
max_places: 100,
include_details: true,
},
},
faq: [
{
question: "Is scraping Google Maps legal?",
answer:
"SurfSense reads only public Google Maps listings, the business data any visitor can see. It collects no private data and cannot access anything behind a login. Review Google's terms and your own compliance needs, especially around personal data in reviews, before running at scale.",
},
{
question: "How many results can I get per search?",
answer:
"Up to 1,000 places per search query, and you can send up to 20 queries, URLs, or place IDs in a single call. Pair a query like 'dentist' with a location such as 'Austin, TX' to scope the search to one market.",
},
{
question: "Does it include reviews and contact details?",
answer:
"Yes. Every place includes its phone number and website when Google lists them, plus address and coordinates. Set max_reviews to attach review text and stars per place, or use the dedicated reviews verb when you need deep review history for one location.",
},
{
question: "How is this different from the official Places API?",
answer:
"The Places API is metered per call and returns only about five reviews per place. SurfSense returns up to 1,000 places per query with deeper reviews, no Cloud project to set up, and an MCP server so your agents can call it as a native tool.",
},
],
related: [
{ label: "Reddit API", href: "/reddit" },
{ label: "YouTube API", href: "/youtube" },
{ label: "SERP API", href: "/google-search" },
{ label: "Web Crawl API", href: "/web-crawl" },
{ label: "MCP Connector", href: "/mcp-connector" },
{ label: "Read the docs", href: "/docs" },
],
};

189
surfsense_web/lib/connectors-marketing/google-search.tsx Normal file
View file

@ -0,0 +1,189 @@
import { IconBrandGoogle } from "@tabler/icons-react";
import type { ConnectorPageContent } from "./types";
export const googleSearch: ConnectorPageContent = {
slug: "google-search",
name: "Google Search",
cardTitle: "SERP API",
icon: IconBrandGoogle,
metaTitle: "SERP API for Rank Tracking and Competitor Intel | SurfSense",
metaDescription:
"Track rankings, competitor ads, and Google AI Overviews with the SurfSense SERP API. Scrape Google Search results as structured JSON via API or MCP. Start free.",
keywords: [
"serp api",
"serp api alternative",
"cheap serp api",
"scrape google search results",
"google search scraper",
"google search api",
"rank tracking api",
"serp scraper api",
"ai overview monitoring",
"serp api python",
"competitor rank tracking",
"serp analysis",
],
h1: "SERP API for Rank Tracking and Competitor SERP Intelligence",
heroLede:
"The SurfSense SERP API scrapes Google Search results as structured JSON: organic rankings, paid ads, AI Overviews, and People Also Ask. Point your AI agents at the queries that matter to your market and know the moment a rank moves, an ad appears, or an AI answer starts citing a competitor.",
transcript: {
prompt: "Track who's ranking and running ads for 'competitor pricing' in the US",
toolCall:
'google_search.scrape({ queries: ["competitor pricing"], country_code: "us",\n max_pages_per_query: 2 })',
rows: [
{
primary: "competitor.com — #3 organic, up from #7",
secondary: "google.com/search · desktop · US",
tag: "+4 spots",
},
{
primary: "AI Overview cites competitor.com and 2 others",
secondary: "AI Overview · 3 sources",
tag: "GEO risk",
},
{
primary: "rival.com is running a paid ad at position 1",
secondary: "paid result · sitelinks · US",
tag: "new ad",
},
],
resultSummary: "20 organic · 3 ads · 1 AI Overview · surfaced in 2.4s",
},
extractIntro:
"Give the API a list of search terms or full Google URLs and a country. It returns one structured item per SERP page, with every block Google renders parsed into its own array.",
extractFields: [
{
label: "Organic results",
description:
"Title, URL, displayed URL, description, date, and rank position for every listing.",
},
{
label: "Paid ads",
description:
"Sponsored results and shopping ads with title, URL, sitelinks, prices, and ad position.",
},
{
label: "AI Overviews",
description:
"The AI Overview answer text and the exact pages Google cites, for GEO and AI-visibility tracking.",
},
{
label: "People Also Ask",
description: "The related questions and answers Google expands beneath the results.",
},
{
label: "Related searches",
description:
"Suggested and related queries, to map how your market actually phrases its search.",
},
{
label: "SERP metadata",
description:
"Total result count, device, page number, and the country and language of each result.",
},
],
useCasesHeading: "What teams do with the SERP API",
useCases: [
{
title: "Rank tracking at agent speed",
description:
"Track your rankings and your competitors' across the queries and countries you care about. Feed the positions to an agent that diffs each run and pings you the moment something moves.",
},
{
title: "Competitor ad monitoring",
description:
"See who is bidding on your keywords, what their ad copy and sitelinks say, and where they sit on the page. Watch a rival's paid strategy without a single manual search.",
},
{
title: "AI Overview and GEO monitoring",
description:
"Capture whether an AI Overview appears for a query and exactly which sources it cites. It is the cleanest way to measure your visibility in AI search, a signal almost nobody is tracking yet.",
},
{
title: "SERP and keyword research",
description:
"Pull People Also Ask and related queries at scale to see the questions and phrasings behind a topic, then brief an agent to turn them into a content plan.",
},
],
comparison: {
heading: "A SERP API alternative built for agents",
intro:
"Most SERP APIs bill per search, gate AI Overviews behind add-ons, and leave the rank-tracking logic to you. Here is how SurfSense compares.",
columnLabel: "Typical SERP API",
rows: [
{
feature: "Pricing",
official: "Per-search pricing that climbs fast at scale",
surfsense: "Pay per SERP page returned, with a free tier to start",
},
{
feature: "AI Overviews",
official: "Often a paid add-on, or not supported at all",
surfsense: "Parsed inline with the answer text and the sources it cites",
},
{
feature: "Rank tracking",
official: "You build the diffing and alerting yourself",
surfsense: "Structured positions your agent can diff and alert on",
},
{
feature: "Setup",
official: "A new vendor account and key management",
surfsense: "One API key, or add the MCP server to your agent",
},
{
feature: "Agent-ready",
official: "No; you wire the harness yourself",
surfsense: "MCP server exposes google_search.scrape as a native tool",
},
],
},
api: {
platform: "google_search",
verb: "scrape",
mcpTool: "google_search.scrape",
requestBody: {
queries: ["competitor pricing"],
country_code: "us",
max_pages_per_query: 2,
},
},
faq: [
{
question: "What is a SERP API?",
answer:
"A SERP API returns the contents of a search engine results page as structured data instead of raw HTML. The SurfSense SERP API scrapes Google Search and gives you organic results, ads, AI Overviews, and People Also Ask as clean JSON your agents can read.",
},
{
question: "Can I track keyword rankings with it?",
answer:
"Yes. Each organic result includes its rank position, and every item is stamped with the query, country, and language it came from. Run the same queries on a schedule and diff the positions to build rank tracking, without maintaining scrapers or proxies.",
},
{
question: "Does it capture Google AI Overviews?",
answer:
"It does. When an AI Overview appears for a query, the response includes its answer text and the list of pages Google cites as sources. That makes it straightforward to monitor whether AI search is surfacing you or your competitors.",
},
{
question: "How is this different from SerpApi or the Google Search API?",
answer:
"You get one API key instead of a new vendor relationship, AI Overviews parsed inline rather than as a paid add-on, and an MCP server so your agents can call google_search.scrape as a native tool. Pricing is per SERP page returned, with a free tier.",
},
],
related: [
{ label: "Web Crawl API", href: "/web-crawl" },
{ label: "Google Maps API", href: "/google-maps" },
{ label: "Reddit API", href: "/reddit" },
{ label: "MCP Connector", href: "/mcp-connector" },
{ label: "Read the docs", href: "/docs" },
],
};

33
surfsense_web/lib/connectors-marketing/index.ts Normal file
View file

@ -0,0 +1,33 @@
import { googleMaps } from "./google-maps";
import { googleSearch } from "./google-search";
import { reddit } from "./reddit";
import type { ConnectorPageContent } from "./types";
import { webCrawl } from "./web-crawl";
import { youtube } from "./youtube";
export type { ConnectorPageContent } from "./types";
/** Registry order controls sitemap and cross-link ordering. */
const CONNECTOR_LIST: ConnectorPageContent[] = [
reddit,
youtube,
googleMaps,
googleSearch,
webCrawl,
];
const CONNECTORS_BY_SLUG: Record<string, ConnectorPageContent> = Object.fromEntries(
CONNECTOR_LIST.map((c) => [c.slug, c])
);
export function getConnector(slug: string): ConnectorPageContent | undefined {
return CONNECTORS_BY_SLUG[slug];
}
export function getAllConnectors(): ConnectorPageContent[] {
return CONNECTOR_LIST;
}
export function getAllConnectorSlugs(): string[] {
return CONNECTOR_LIST.map((c) => c.slug);
}

187
surfsense_web/lib/connectors-marketing/reddit.tsx Normal file
View file

@ -0,0 +1,187 @@
import { IconBrandReddit } from "@tabler/icons-react";
import type { ConnectorPageContent } from "./types";
export const reddit: ConnectorPageContent = {
slug: "reddit",
name: "Reddit",
icon: IconBrandReddit,
metaTitle: "Reddit Scraper API for Brand Monitoring | SurfSense",
metaDescription:
"Track every brand and competitor mention across Reddit with the SurfSense Reddit Scraper API. No official API rate limits or pricing, plus a free tier. Start now.",
keywords: [
"reddit scraper",
"reddit scraper api",
"reddit api",
"reddit api alternative",
"scrape reddit",
"reddit data api",
"reddit comment scraper",
"reddit monitoring tool",
"reddit sentiment analysis",
"reddit mcp",
"social listening",
"brand monitoring tool",
],
h1: "Reddit API for Brand Monitoring and Market Research",
heroLede:
"The SurfSense Reddit API extracts posts, comments, subreddits, and user data without the official Reddit API's rate limits or per-call pricing. Give your AI agents a live feed of what your market says about your brand, your competitors, and your category, so you hear it first.",
transcript: {
prompt: "See what r/webscraping is saying about our competitor this week",
toolCall:
'reddit.scrape({ search_queries: ["competitor"], community: "webscraping",\n sort: "top", time_filter: "week", max_comments: 50 })',
rows: [
{
primary: "Switched off Competitor after the API price hike",
secondary: "r/webscraping · 312 upvotes · 0.94 ratio",
tag: "pricing pain",
},
{
primary: "Anyone got a Competitor alternative that does comments?",
secondary: "r/webscraping · 148 upvotes · 61 comments",
tag: "buying intent",
},
{
primary: "Competitor keeps rate-limiting my agent, help",
secondary: "r/webscraping · 89 upvotes · 44 comments",
tag: "churn signal",
},
],
resultSummary: "37 posts · 1,204 comments · surfaced in 2.1s",
},
extractIntro:
"Every call returns structured items keyed by type: posts, comments, communities, and users. Point the API at a subreddit, a search query, a post, or a user profile.",
extractFields: [
{
label: "Posts",
description: "Title, body text, upvotes, upvote ratio, comment count, flair, and permalink.",
},
{
label: "Comments",
description: "Full comment trees with body, vote counts, reply counts, and parent threading.",
},
{
label: "Communities",
description: "Subreddit name, member count, and listing metadata for any /r/ community.",
},
{
label: "Users",
description: "Public profile and authored posts or comments for any /user/ handle.",
},
{
label: "Media",
description: "Image and video URLs, thumbnails, and NSFW flags attached to each post.",
},
{
label: "Timestamps",
description:
"Created and scraped times, with date limits for incremental, delta-only scrapes.",
},
],
useCasesHeading: "What teams do with the Reddit API",
useCases: [
{
title: "Brand and competitor monitoring",
description:
"Track every mention of your brand, your competitors, and your category across Reddit. Feed the stream to an agent that flags churn signals, pricing complaints, and buying intent the moment they surface.",
},
{
title: "Voice-of-customer and pain-point mining",
description:
"Mine real complaints and feature requests to find your next product idea. Reddit is where people say what they actually think, unfiltered by a survey or a support ticket.",
},
{
title: "Reddit sentiment analysis",
description:
"Pull comment trees at depth and score them for sentiment, so you can measure how a launch, a price change, or a competitor's move actually landed with your market.",
},
{
title: "Community research for go-to-market",
description:
"Map the subreddits, language, and objections of your buyers before you spend on ads. Turn community research into positioning your GTM team can act on.",
},
],
comparison: {
heading: "A Reddit API alternative built for agents",
intro:
"Reddit's official Data API added rate limits and per-call pricing in 2023. If you are hitting the paywall or the throttle, here is how SurfSense compares.",
columnLabel: "Official Reddit API",
rows: [
{
feature: "Rate limits",
official: "Per-OAuth-app quotas that throttle bulk and agent use",
surfsense: "Managed for you; scale reads without minding the throttle",
},
{
feature: "Pricing",
official: "Per-call pricing since the 2023 changes",
surfsense: "Pay per item returned, with a free tier to start",
},
{
feature: "Comment depth",
official: "Paginated; deep trees are slow and quota-heavy",
surfsense: "Full comment trees in one call, up to 100 items",
},
{
feature: "Setup",
official: "Register an OAuth app and manage tokens",
surfsense: "One API key, or add the MCP server to your agent",
},
{
feature: "Agent-ready",
official: "No; you build the harness yourself",
surfsense: "MCP server exposes reddit.scrape as a native tool",
},
],
},
api: {
platform: "reddit",
verb: "scrape",
mcpTool: "reddit.scrape",
requestBody: {
search_queries: ["your brand"],
community: "webscraping",
sort: "top",
time_filter: "week",
max_comments: 50,
},
},
faq: [
{
question: "Is scraping Reddit legal?",
answer:
"SurfSense reads only public Reddit data, the same posts and comments any logged-out visitor can see. It never logs in and cannot access private or deleted content. As always, review Reddit's terms and your own compliance needs before you run at scale.",
},
{
question: "Does this bypass the official Reddit API?",
answer:
"It is an independent alternative, not a wrapper. You do not register an OAuth app or manage Reddit tokens. You call the SurfSense API with one key, or add the MCP server to your agent, and get structured posts and comments back without the official API's limits.",
},
{
question: "What are the rate limits?",
answer:
"Each call caps at 100 returned items across all sources, with up to 20 URLs or search queries per request. SurfSense manages the underlying request budget for you, so you scale reads without registering apps or watching an OAuth quota.",
},
{
question: "How much historical data can I pull?",
answer:
"You can scrape a subreddit, user, or search back through its available public history, then use post and comment date limits to run incremental, delta-only scrapes. That keeps repeat monitoring cheap: pull only what is new since your last run.",
},
],
related: [
{ label: "YouTube API", href: "/youtube" },
{ label: "Google Maps API", href: "/google-maps" },
{ label: "SERP API", href: "/google-search" },
{ label: "Web Crawl API", href: "/web-crawl" },
{ label: "MCP Connector", href: "/mcp-connector" },
{ label: "Read the docs", href: "/docs" },
],
};

104
surfsense_web/lib/connectors-marketing/types.ts Normal file
View file

@ -0,0 +1,104 @@
import type { ComponentType } from "react";
/** One extractable data point, rendered in the "What you can extract" grid. */
export interface ExtractField {
label: string;
description: string;
}
/** An outcome-framed use case, rendered as an H3 under the use-cases H2. */
export interface UseCase {
title: string;
description: string;
}
/** A single row in the "official API vs SurfSense" comparison table. */
export interface ComparisonRow {
feature: string;
/** The incumbent / DIY column value. */
official: string;
/** The SurfSense column value. */
surfsense: string;
}
/** A FAQ entry. Answers are written to 40-60 words for featured-snippet fit. */
export interface FaqItem {
question: string;
answer: string;
}
/** One structured result row revealed in the agent-transcript hero artifact. */
export interface TranscriptRow {
primary: string;
secondary: string;
/** Optional short tag rendered as a pill (e.g. "no website", "+92%"). */
tag?: string;
}
/** The signature hero artifact: a realistic agent task and its structured output. */
export interface AgentTranscript {
/** Natural-language task the agent is asked to perform. */
prompt: string;
/** The tool invocation the agent makes, mirroring the real capability surface. */
toolCall: string;
rows: TranscriptRow[];
/** One-line summary shown under the rows (e.g. "100 places · 23 without a website"). */
resultSummary: string;
}
/** A cross-link (to another connector, the homepage, or docs). */
export interface RelatedLink {
label: string;
href: string;
}
/**
* The real capability surface for the API/MCP code samples. Values map 1:1 to
* the backend `ScrapeInput`/`CommentsInput` contracts so examples are truthful.
*/
export interface ApiSample {
/** Platform segment of the REST path, e.g. "google_maps". */
platform: string;
/** Verb segment of the REST path, e.g. "scrape". */
verb: string;
/** MCP tool name, e.g. "google_maps.scrape". */
mcpTool: string;
/** Request body keys/values, serialized into the cURL and Python samples. */
requestBody: Record<string, unknown>;
}
/** Everything needed to render one connector marketing page. */
export interface ConnectorPageContent {
slug: string;
/** Platform display name, e.g. "Reddit". */
name: string;
/** Clean product label for cards and headings. Defaults to `${name} API`. */
cardTitle?: string;
icon: ComponentType<{ className?: string }>;
// SEO
/** <= 60 chars, primary keyword front-loaded. */
metaTitle: string;
/** 150-160 chars, ends with a CTA. */
metaDescription: string;
keywords: string[];
// Hero
h1: string;
/** Direct-answer paragraph placed in the first 150 words, keyword-rich. */
heroLede: string;
transcript: AgentTranscript;
// Body
extractIntro: string;
extractFields: ExtractField[];
useCasesHeading: string;
useCases: UseCase[];
comparison: {
heading: string;
intro: string;
/** Label for the incumbent column, e.g. "Official Reddit API". */
columnLabel: string;
rows: ComparisonRow[];
};
api: ApiSample;
faq: FaqItem[];
related: RelatedLink[];
}

190
surfsense_web/lib/connectors-marketing/web-crawl.tsx Normal file
View file

@ -0,0 +1,190 @@
import { IconWorldWww } from "@tabler/icons-react";
import type { ConnectorPageContent } from "./types";
export const webCrawl: ConnectorPageContent = {
slug: "web-crawl",
name: "Web Crawl",
cardTitle: "Web Crawl API",
icon: IconWorldWww,
metaTitle: "Web Crawl API for AI Agents and Deep Research | SurfSense",
metaDescription:
"Point your agents at any URL and get clean, LLM-ready markdown, links, and contacts back. The SurfSense Web Crawl API for deep research and competitor monitoring. Start free.",
keywords: [
"web crawl api",
"web crawler api",
"web crawling api",
"firecrawl alternative",
"llm web scraping",
"agentic web scraping",
"deep research api",
"web scraping mcp",
"competitor price monitoring",
"crawl website for ai",
"website to markdown",
"web data for ai",
],
h1: "Web Crawl API: Turn Any Website into Agent-Ready Context",
heroLede:
"The SurfSense Web Crawl API points your AI agents at any URL and returns clean, LLM-ready markdown, structured links, and harvested contacts. Scrape a single page or spider a whole site, with the same agent harness that powers our platform connectors.",
transcript: {
prompt: "Crawl our top competitor's site and pull their pricing and contacts",
toolCall:
'web.crawl({ startUrls: ["https://competitor.com"], maxCrawlDepth: 2,\n maxCrawlPages: 50 })',
rows: [
{
primary: "/pricing — Pro plan now $49/mo, up from $39",
secondary: "markdown · depth 1 · success",
tag: "price change",
},
{
primary: "sales@competitor.com",
secondary: "found on 12 pages · site-wide",
tag: "contact",
},
{
primary: "linkedin.com/in/their-head-of-sales",
secondary: "team page · anchor: 'VP Sales'",
tag: "social",
},
],
resultSummary: "50 pages · 41 markdown · 6 contacts · surfaced in 8.3s",
},
extractIntro:
"Give the API one or more start URLs. With depth 0 it scrapes just those pages; with a higher depth it spiders the site, bounded by your page cap, and returns one structured item per page.",
extractFields: [
{
label: "Clean markdown",
description:
"Cleaned, LLM-ready markdown for every page, truncated to the character limit you set.",
},
{
label: "Page metadata",
description:
"Title, description, and other metadata, plus the URL, fetch status, and crawl depth.",
},
{
label: "Links",
description:
"Every link with its anchor text and kind: internal, external, social, email, or tel.",
},
{
label: "Contacts",
description:
"Emails, phone numbers, and social profiles harvested from each page's raw HTML.",
},
{
label: "Site-wide contacts",
description:
"A deduplicated roll-up that separates company boilerplate from page-local people, with provenance.",
},
{
label: "Crawl control",
description:
"Depth, page cap, and include or exclude URL regex patterns to keep the spider on target.",
},
],
useCasesHeading: "What teams do with the Web Crawl API",
useCases: [
{
title: "Deep research over the open web",
description:
"Let an agent crawl dozens of sources on a topic, collect clean markdown, and synthesize an answer with citations. A deep-research API that reads the live web, not a stale index.",
},
{
title: "Competitor website monitoring",
description:
"Crawl a rival's pricing, product, and changelog pages on a schedule and diff them. Catch a price change, a new feature, or a repositioning the day it ships, not the quarter after.",
},
{
title: "Lead and contact enrichment",
description:
"Spider a company site and pull emails, phone numbers, and social profiles with provenance, so your agent knows which contact belongs to the company and which to a specific person.",
},
{
title: "RAG and knowledge-base ingestion",
description:
"Turn any set of sites into clean markdown ready to embed. Feed a knowledge base or a RAG pipeline without writing a parser or babysitting a headless browser.",
},
],
comparison: {
heading: "A Firecrawl alternative with an agent harness",
intro:
"Commodity crawlers hand you markdown and leave the rest to you. SurfSense adds the harness, the contact extraction, and the platform connectors around it.",
columnLabel: "Commodity crawler",
rows: [
{
feature: "Output",
official: "Markdown or HTML you post-process yourself",
surfsense: "Markdown plus parsed links, contacts, and site-wide roll-ups",
},
{
feature: "Contacts",
official: "Not extracted; you write the regex",
surfsense: "Emails, phones, and socials harvested with page provenance",
},
{
feature: "Scope",
official: "One site at a time",
surfsense: "Crawl the open web, then combine with platform connectors in one workflow",
},
{
feature: "Pricing",
official: "Per-page credits that add up quickly",
surfsense: "Pay per successful page, with a free tier to start",
},
{
feature: "Agent-ready",
official: "An SDK you wire into your own harness",
surfsense: "MCP server exposes web.crawl as a native tool",
},
],
},
api: {
platform: "web",
verb: "crawl",
mcpTool: "web.crawl",
requestBody: {
startUrls: ["https://competitor.com"],
maxCrawlDepth: 2,
maxCrawlPages: 50,
},
},
faq: [
{
question: "What formats does the Web Crawl API return?",
answer:
"Every page comes back as cleaned, LLM-ready markdown, alongside structured JSON for its metadata, links, and contacts. You set the maximum characters kept per page, so you control how much content each result carries.",
},
{
question: "Can it crawl a whole site or just one page?",
answer:
"Both. Set maxCrawlDepth to 0 to scrape only the URLs you pass in, or raise it to spider the site, following links hop by hop. maxCrawlPages caps the total pages fetched, and the spider stays on the start URL's site.",
},
{
question: "Does it handle JavaScript-heavy sites?",
answer:
"Yes. The underlying engine renders dynamic pages and waits for content to load automatically, so you get the same markdown a browser would show. You do not configure a headless browser or manage proxies yourself.",
},
{
question: "How is this different from Firecrawl?",
answer:
"You get more than markdown: parsed links, harvested contacts with provenance, and an MCP server that exposes web.crawl as a native agent tool. It shares one harness and workspace with the SurfSense platform connectors, so a crawl and a Reddit or Maps pull live in the same workflow.",
},
],
related: [
{ label: "SERP API", href: "/google-search" },
{ label: "Google Maps API", href: "/google-maps" },
{ label: "Reddit API", href: "/reddit" },
{ label: "MCP Connector", href: "/mcp-connector" },
{ label: "Read the docs", href: "/docs" },
],
};

184
surfsense_web/lib/connectors-marketing/youtube.tsx Normal file
View file

@ -0,0 +1,184 @@
import { IconBrandYoutube } from "@tabler/icons-react";
import type { ConnectorPageContent } from "./types";
export const youtube: ConnectorPageContent = {
slug: "youtube",
name: "YouTube",
icon: IconBrandYoutube,
metaTitle: "YouTube Scraper API for Audience Sentiment | SurfSense",
metaDescription:
"Scrape YouTube videos, comments, and transcripts at scale with the SurfSense YouTube Scraper API. No Data API quotas, plus a free tier for AI agents. Start now.",
keywords: [
"youtube scraper",
"youtube scraper api",
"youtube comment scraper",
"youtube comments api",
"youtube data api",
"youtube data api alternative",
"youtube channel scraper",
"youtube transcript scraper",
"youtube mcp server",
"youtube sentiment analysis",
"audience research tools",
],
h1: "YouTube API for Audience Sentiment and Content Intelligence",
heroLede:
"The SurfSense YouTube API pulls videos, channel stats, comments, and transcripts at scale, without YouTube Data API quotas. Feed your AI agents every comment and transcript that matters to your market, so you know what people actually think about your product and your competitors' content.",
transcript: {
prompt: "Summarize how people reacted in the comments on our launch video",
toolCall:
'youtube.comments({ urls: ["youtube.com/watch?v=launch"],\n max_comments: 500, sort_by: "TOP_COMMENTS" })',
rows: [
{
primary: '"Finally an API that does not throttle my agent"',
secondary: "@devbuilds · 214 likes · creator hearted",
tag: "positive",
},
{
primary: '"How does pricing compare to the incumbent?"',
secondary: "@growthlee · 96 likes · 12 replies",
tag: "buying intent",
},
{
primary: '"Does it handle transcripts too or just metadata?"',
secondary: "@ml_ana · 71 likes · 8 replies",
tag: "feature ask",
},
],
resultSummary: "500 comments · 78% positive · surfaced in 3.4s",
},
extractIntro:
"Give the API video URLs, channel or playlist URLs, or search queries. It returns structured video items; a separate comments call returns full comment and reply trees for any video.",
extractFields: [
{
label: "Video metadata",
description: "Title, view count, likes, duration, publish date, hashtags, and description.",
},
{
label: "Channel stats",
description: "Subscriber count, total views, total videos, join date, and verified status.",
},
{
label: "Comments and replies",
description: "Author, comment text, vote counts, reply counts, and creator-heart flags.",
},
{
label: "Transcripts",
description: "Subtitle tracks per video in the language you request, ready as agent context.",
},
{
label: "Content types",
description: "Standard videos, Shorts, and live streams, capped independently per source.",
},
{
label: "Engagement",
description: "Likes, comment counts, and monetization or members-only flags per video.",
},
],
useCasesHeading: "What teams do with the YouTube API",
useCases: [
{
title: "Comment sentiment at scale",
description:
"Read every comment on your launch video or a competitor's ad, not a sample. Score product feedback and ad reaction so your team knows what actually resonated with the audience.",
},
{
title: "Competitor channel analysis",
description:
"Pull a rival channel's full catalog with views, likes, and cadence to see which content wins in your niche, then brief an agent to spot the patterns you should copy.",
},
{
title: "Influencer vetting",
description:
"Before you sponsor a creator, scrape their real view counts, comment sentiment, and posting history so you pay for genuine reach instead of a vanity subscriber number.",
},
{
title: "Transcripts as agent context",
description:
"Download transcripts to feed RAG pipelines and research agents. Turn hours of video into searchable, citable context your agents can reason over in seconds.",
},
],
comparison: {
heading: "A YouTube Data API alternative without the quota",
intro:
"The official YouTube Data API gives you 10,000 units a day, and reading comments burns through it fast. Here is how SurfSense compares.",
columnLabel: "YouTube Data API",
rows: [
{
feature: "Quota",
official: "10,000 units per day; comment reads drain it fast",
surfsense: "No daily unit quota; pay per item returned",
},
{
feature: "Comments",
official: "Paginated and quota-heavy at depth",
surfsense: "Up to 100,000 comments and replies per video",
},
{
feature: "Transcripts",
official: "Not available through the official API",
surfsense: "Subtitle tracks included, in the language you pick",
},
{
feature: "Setup",
official: "Google Cloud project, OAuth, and API key",
surfsense: "One API key, or add the MCP server to your agent",
},
{
feature: "Agent-ready",
official: "No; you build the harness yourself",
surfsense: "MCP server exposes youtube.scrape as a native tool",
},
],
},
api: {
platform: "youtube",
verb: "scrape",
mcpTool: "youtube.scrape",
requestBody: {
search_queries: ["your topic"],
max_results: 50,
download_subtitles: true,
subtitles_language: "en",
},
},
faq: [
{
question: "Is scraping YouTube legal?",
answer:
"SurfSense reads only public YouTube data, the videos, channels, and comments any visitor can see. It never signs in and cannot access private or unlisted content. Review YouTube's terms and your own compliance needs before running large jobs.",
},
{
question: "Are there quota limits like the YouTube Data API?",
answer:
"No daily unit quota. Each scrape call takes up to 20 sources with up to 1,000 results each, and a comments call returns up to 100,000 comments and replies per video. You pay per item returned instead of budgeting Google's 10,000 daily units.",
},
{
question: "Can it download transcripts, and in which languages?",
answer:
"Yes. Set download_subtitles to true and pass a language code such as en or fr, and each video comes back with its subtitle track. Transcripts make excellent context for RAG pipelines and research agents that need to reason over video content.",
},
{
question: "Does it support YouTube Shorts?",
answer:
"Yes. Channel sources return videos, Shorts, and live streams, each capped independently so one source can return all three types. You can also pass Shorts URLs directly, or use search queries to discover them across the platform.",
},
],
related: [
{ label: "Reddit API", href: "/reddit" },
{ label: "Google Maps API", href: "/google-maps" },
{ label: "SERP API", href: "/google-search" },
{ label: "Web Crawl API", href: "/web-crawl" },
{ label: "MCP Connector", href: "/mcp-connector" },
{ label: "Read the docs", href: "/docs" },
],
};