Merge remote-tracking branch 'upstream/dev' into feature-walmart-scraper

# Conflicts:
#	docker/.env.example
#	surfsense_backend/app/capabilities/core/billing.py
#	surfsense_backend/app/capabilities/core/types.py
#	surfsense_backend/app/config/__init__.py
#	surfsense_mcp/mcp_server/features/scrapers/__init__.py
#	surfsense_web/content/docs/connectors/index.mdx
#	surfsense_web/content/docs/connectors/native/index.mdx
#	surfsense_web/content/docs/how-to/mcp-server.mdx
This commit is contained in:
CREDO23 2026-07-21 02:42:41 +02:00
commit ca317c4686
68 changed files with 4365 additions and 20 deletions

View file

@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li
<Card
icon={<Zap />}
title="Native Connectors"
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST"
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, Walmart, and Web Crawl — usable in chat, the API Playground, or via REST"
href="/docs/connectors/native"
/>
<Card

View file

@ -0,0 +1,51 @@
---
title: Indeed
description: Scrape public Indeed job postings by search, company, or URL
---
The Indeed scraper pulls structured public job postings from Indeed. Give it search terms and/or URLs (a search page, a company jobs page, or a single job), and it returns jobs with title, company, location, salary, job types, benefits, remote/hybrid flag, posting age, and description.
## Endpoint
```bash
POST /api/v1/workspaces/{workspace_id}/scrapers/indeed/scrape
```
## Inputs
At least one of `urls` or `search_queries` is required.
| Field | Default | Description |
|-------|---------|-------------|
| `urls` | — | Indeed URLs: a search page (`/jobs?q=&l=`), a company jobs page (`/cmp/<slug>/jobs`), or a single job (`/viewjob?jk=`) (max 20 sources per call, combined with queries) |
| `search_queries` | — | Job search terms, e.g. `["data analyst"]` |
| `country` | `us` | Country code selecting the Indeed domain, e.g. `us`, `gb`, `de` |
| `location` | — | Where to search, e.g. `Remote`, `New York, NY` |
| `radius` | — | Search radius in miles/km around `location` |
| `job_type` | — | `fulltime`, `parttime`, `contract`, `internship`, `temporary`, `permanent`, `seasonal`, or `freelance` |
| `level` | — | `entry_level`, `mid_level`, or `senior_level` |
| `remote` | — | `remote` or `hybrid` |
| `from_days` | — | Only return jobs posted within the last N days |
| `sort` | `relevance` | `relevance` or `date` |
| `scrape_job_details` | `false` | Fetch each job's detail page for the full description (slower: one extra page load per job) |
| `max_items` | `25` | Max total jobs returned across all sources (hard cap 100) |
| `max_items_per_query` | `25` | Max jobs per search/company target |
## Example
```bash
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/indeed/scrape" \
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"search_queries": ["data analyst"],
"location": "Remote",
"remote": "remote",
"sort": "date",
"max_items": 30
}'
```
The response is `{ "items": [...] }` — one item per job posting. Billing is per returned job.
For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Indeed → Scrape** in your workspace.

View file

@ -1,6 +1,6 @@
---
title: Native Connectors
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, Walmart, and the web
---
import { Card, Cards } from 'fumadocs-ui/components/card';
@ -38,6 +38,11 @@ Native connectors are SurfSense's own scraper APIs — built into the platform,
description="Structured SERPs: organic results, people-also-ask, AI overviews"
href="/docs/connectors/native/google-search"
/>
<Card
title="Indeed"
description="Job postings by search, company, or URL — with salary and full description"
href="/docs/connectors/native/indeed"
/>
<Card
title="Amazon"
description="Public product data: prices, ratings, offers, sellers, and best-seller ranks"

View file

@ -7,6 +7,7 @@
"tiktok",
"google-maps",
"google-search",
"indeed",
"amazon",
"walmart",
"web-crawl"

View file

@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
# SurfSense MCP Server
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 27 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 28 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, Amazon, Walmart, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector.
Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted.
@ -264,7 +264,7 @@ For self-host (stdio), all settings are environment variables passed by the clie
| Group | Tools |
|-------|-------|
| Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` |
| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_tiktok_comments`, `surfsense_tiktok_user_search`, `surfsense_tiktok_trending`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` |
| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_tiktok_comments`, `surfsense_tiktok_user_search`, `surfsense_tiktok_trending`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_indeed_scrape`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` |
| Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` |
Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**.

View file

@ -44,6 +44,7 @@ const PUBLIC_ROUTE_PREFIXES = [
"/youtube",
"/google-maps",
"/google-search",
"/indeed",
"/web-crawl",
"/amazon",
];

View file

@ -306,6 +306,7 @@ export const googleMaps: ConnectorPageContent = {
{ label: "Instagram API", href: "/instagram" },
{ label: "SERP API", href: "/google-search" },
{ label: "Web Crawl API", href: "/web-crawl" },
{ label: "Indeed API", href: "/indeed" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
{ label: "Read the docs", href: "/docs" },
],

View file

@ -260,6 +260,7 @@ export const googleSearch: ConnectorPageContent = {
{ label: "Google Maps API", href: "/google-maps" },
{ label: "Reddit API", href: "/reddit" },
{ label: "Instagram API", href: "/instagram" },
{ label: "Indeed API", href: "/indeed" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
{ label: "Read the docs", href: "/docs" },
],

View file

@ -0,0 +1,327 @@
import { IconBriefcase } from "@tabler/icons-react";
import type { ConnectorPageContent } from "./types";
export const indeed: ConnectorPageContent = {
slug: "indeed",
name: "Indeed",
icon: IconBriefcase,
metaTitle: "Indeed Scraper API for Jobs and Hiring Data | SurfSense",
metaDescription:
"Scrape public Indeed job postings with the SurfSense Indeed Scraper API: titles, companies, salaries, and full descriptions by search or company. No Indeed API. Start free.",
keywords: [
"indeed scraper",
"indeed scraper api",
"indeed api",
"indeed jobs api",
"scrape indeed",
"indeed job scraper",
"job posting scraper",
"salary data api",
"hiring data api",
"indeed mcp",
"labor market data",
"recruiting data tool",
],
h1: "Indeed Scraper API for Job Postings and Hiring Data",
heroLede:
"The SurfSense Indeed API extracts public job postings, salaries, companies, and full descriptions by search query, company page, or job URL, without Indeed's official API. Give your AI agents a live feed of who is hiring, for what, at what pay, so you track the labor market as it moves.",
transcript: {
prompt: "Find remote data analyst roles posted this week and what they pay",
toolCall:
'indeed.scrape({ search_queries: ["data analyst"], location: "Remote",\n remote: "remote", from_days: 7, sort: "date", max_items: 30 })',
rows: [
{
primary: "Senior Data Analyst · Acme Corp",
secondary: "Remote (US) · $120k$145k/year · posted 2 days ago",
tag: "salary listed",
},
{
primary: "Data Analyst, Growth · Globex",
secondary: "Remote · $95k$110k/year · Indeed Apply",
tag: "buying signal",
},
{
primary: "Marketing Data Analyst · Initech",
secondary: "Remote (US) · estimated $88k$102k · posted today",
tag: "new today",
},
],
resultSummary: "30 jobs · 22 with salary · surfaced in 3.4s",
},
extractIntro:
"Every call returns structured job items. Point the API at a search query, an Indeed search or company page, or a single job URL, and set scrape_job_details for the full description per job.",
extractFields: [
{
label: "Job",
description: "Title, job key, listing URL, apply URL, and whether Indeed Apply is enabled.",
},
{
label: "Company",
description: "Company name, profile URL, star rating, and review count where available.",
},
{
label: "Location",
description: "Formatted location, city, state, postal code, country, and remote or hybrid flags.",
},
{
label: "Salary",
description:
"Pay text, min and max bounds, currency, period, and whether the figure is an Indeed estimate.",
},
{
label: "Description",
description:
"Listing snippet by default; the full text and HTML description with scrape_job_details.",
},
{
label: "Signals",
description:
"Job types, benefits, sponsored, urgently hiring, new, and expired flags, plus post age.",
},
],
useCasesHeading: "What teams do with the Indeed API",
useCases: [
{
title: "Competitor hiring intelligence",
description:
"Track what your competitors are hiring for and where. A spike in sales or ML roles is a roadmap signal months before it ships. Feed the stream to an agent that flags the moves that matter.",
},
{
title: "Salary and compensation benchmarking",
description:
"Pull real posted salaries for a title in a location and benchmark your own bands against the live market, instead of a survey that is a year stale.",
},
{
title: "Labor market and sector research",
description:
"Measure hiring demand for a role, skill, or sector over time. Turn thousands of postings into a demand index your analysts and clients can act on.",
},
{
title: "Recruiting and lead sourcing",
description:
"Find companies actively hiring for a role and reach them while the need is hot. Job postings are a public, timely buying signal for staffing and B2B sales.",
},
],
comparison: {
heading: "An Indeed API alternative built for agents",
intro:
"Indeed retired its public Publisher jobs API and gates data behind partner programs. If you cannot get access or need clean structured jobs now, here is how SurfSense compares.",
columnLabel: "DIY Indeed scraping",
rows: [
{
feature: "Access",
official: "Publisher API retired; partner-gated and approval-only",
surfsense: "One API key; scrape public postings without an approval process",
},
{
feature: "Anti-bot",
official: "You fight Cloudflare, fingerprinting, and CAPTCHAs yourself",
surfsense: "Warmed, rotated sessions managed for you; no proxy plumbing",
},
{
feature: "Pricing",
official: "Proxy, browser, and maintenance costs you own",
surfsense: "Pay per job returned, with a free tier to start",
},
{
feature: "Descriptions",
official: "Extra page fetch and parsing you build and maintain",
surfsense: "Full description per job with one scrape_job_details flag",
},
{
feature: "Agent-ready",
official: "No; you build the harness yourself",
surfsense: "MCP server exposes indeed.scrape as a native tool",
},
],
},
api: {
platform: "indeed",
verb: "scrape",
mcpTool: "indeed.scrape",
requestBody: {
search_queries: ["data analyst"],
location: "Remote",
remote: "remote",
from_days: 7,
sort: "date",
max_items: 30,
},
},
schema: {
requestNote:
"Provide at least one source: urls or search_queries. Up to 20 sources per call.",
request: [
{
name: "urls",
type: "string[]",
defaultValue: "[]",
description:
"Indeed URLs: a search page (/jobs?q=&l=), a company jobs page (/cmp/<slug>/jobs), or a single job (/viewjob?jk=...). Max 20.",
},
{
name: "search_queries",
type: "string[]",
defaultValue: "[]",
description:
"Job search terms. Each returns up to max_items_per_query results, shaped by the filters below. Max 20.",
},
{
name: "country",
type: "string",
defaultValue: '"us"',
description: "Country code selecting the Indeed domain, e.g. 'us', 'gb', 'de'.",
},
{
name: "location",
type: "string",
description: "Where to search, e.g. 'Remote', 'New York, NY'.",
},
{
name: "radius",
type: "integer",
description: "Search radius in miles or km around location.",
},
{
name: "job_type",
type: "string",
description: "Employment type: fulltime, parttime, contract, internship, and more.",
},
{
name: "level",
type: "string",
description: "Experience level: entry_level, mid_level, or senior_level.",
},
{
name: "remote",
type: "string",
description: "Work model filter: remote or hybrid.",
},
{
name: "from_days",
type: "integer",
description: "Only return jobs posted within the last N days.",
},
{
name: "sort",
type: "string",
defaultValue: '"relevance"',
description: "Result ordering: relevance or date.",
},
{
name: "scrape_job_details",
type: "boolean",
defaultValue: "false",
description:
"Fetch each job's detail page for the full description. Slower: one extra page load per job.",
},
{
name: "max_items",
type: "integer",
defaultValue: "25",
description: "Max total jobs to return across all sources. 1 to 100.",
},
{
name: "max_items_per_query",
type: "integer",
defaultValue: "25",
description: "Max jobs to pull per search or company target.",
},
],
responseNote:
"The response is { items: [...] } with one flat item per job. Fields Indeed omits are null. One returned job is one billable unit.",
response: [
{
name: "jobKey / jobUrl / applyUrl",
type: "string",
description: "Indeed job key, listing URL, and third-party apply URL.",
},
{
name: "title",
type: "string",
description: "The job title as posted.",
},
{
name: "company / companyUrl",
type: "string",
description: "Company name and its Indeed profile URL.",
},
{
name: "companyRating / companyReviewCount",
type: "number / integer",
description: "Employer star rating and number of reviews, where Indeed shows them.",
},
{
name: "formattedLocation / isRemote / remoteType",
type: "string / boolean",
description: "Location string plus remote and hybrid flags.",
},
{
name: "salary",
type: "object",
description:
"salaryText, salaryMin, salaryMax, currency, period, and isEstimated when the pay is an Indeed estimate.",
},
{
name: "jobTypes / benefits",
type: "string[]",
description: "Employment types and listed benefits parsed from the posting.",
},
{
name: "descriptionText / descriptionHtml",
type: "string",
description: "Snippet by default; the full description when scrape_job_details is set.",
},
{
name: "sponsored / urgentlyHiring / isNew / expired",
type: "boolean",
description: "Listing flags for ranking and filtering.",
},
{
name: "age / datePublished / scrapedAt",
type: "string",
description: "Relative post age, ISO publish date, and when the job was scraped.",
},
],
},
faq: [
{
question: "Is scraping Indeed legal?",
answer:
"SurfSense reads only public Indeed job postings, the same listings any logged-out visitor can see. It never logs in and cannot access private or applicant data. As always, review Indeed's terms and your own compliance needs before you run at scale.",
},
{
question: "Does Indeed have an official jobs API?",
answer:
"Indeed retired its public Publisher jobs API and now gates job data behind partner and approval programs. SurfSense is an independent alternative: you call one API, or add the MCP server to your agent, and get structured public postings back.",
},
{
question: "Can I get the full job description?",
answer:
"Yes. By default each job returns the listing snippet, which is fast. Set scrape_job_details to true and SurfSense fetches each job's detail page for the full description text and HTML, at the cost of one extra page load per job.",
},
{
question: "What are the rate limits?",
answer:
"Each call returns up to 100 jobs across all sources, with up to 20 URLs or search queries per request. SurfSense manages the anti-bot request budget for you, so you scale reads without running proxies or a headless browser yourself.",
},
],
related: [
{ label: "Reddit API", href: "/reddit" },
{ label: "YouTube API", href: "/youtube" },
{ label: "Google Maps API", href: "/google-maps" },
{ label: "SERP API", href: "/google-search" },
{ label: "Web Crawl API", href: "/web-crawl" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
],
};

View file

@ -1,6 +1,7 @@
import { amazon } from "./amazon";
import { googleMaps } from "./google-maps";
import { googleSearch } from "./google-search";
import { indeed } from "./indeed";
import { instagram } from "./instagram";
import { reddit } from "./reddit";
import { tiktok } from "./tiktok";
@ -19,6 +20,7 @@ const CONNECTOR_LIST: ConnectorPageContent[] = [
tiktok,
googleMaps,
googleSearch,
indeed,
amazon,
walmart,
webCrawl,

View file

@ -288,6 +288,7 @@ export const instagram: ConnectorPageContent = {
{ label: "Reddit API", href: "/reddit" },
{ label: "Google Maps API", href: "/google-maps" },
{ label: "SERP API", href: "/google-search" },
{ label: "Indeed API", href: "/indeed" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
],
};

View file

@ -315,6 +315,7 @@ export const reddit: ConnectorPageContent = {
{ label: "Google Maps API", href: "/google-maps" },
{ label: "SERP API", href: "/google-search" },
{ label: "Web Crawl API", href: "/web-crawl" },
{ label: "Indeed API", href: "/indeed" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
],
};

View file

@ -284,6 +284,7 @@ export const tiktok: ConnectorPageContent = {
{ label: "Google Maps API", href: "/google-maps" },
{ label: "SERP API", href: "/google-search" },
{ label: "Web Crawl API", href: "/web-crawl" },
{ label: "Indeed API", href: "/indeed" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
],
};

View file

@ -283,6 +283,7 @@ export const webCrawl: ConnectorPageContent = {
{ label: "Google Maps API", href: "/google-maps" },
{ label: "Reddit API", href: "/reddit" },
{ label: "Instagram API", href: "/instagram" },
{ label: "Indeed API", href: "/indeed" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
{ label: "Read the docs", href: "/docs" },
],

View file

@ -275,6 +275,7 @@ export const youtube: ConnectorPageContent = {
{ label: "Google Maps API", href: "/google-maps" },
{ label: "SERP API", href: "/google-search" },
{ label: "Web Crawl API", href: "/web-crawl" },
{ label: "Indeed API", href: "/indeed" },
{ label: "SurfSense MCP Server", href: "/mcp-server" },
],
};

View file

@ -3,6 +3,7 @@ import {
AmazonIcon,
GoogleMapsIcon,
GoogleSearchIcon,
IndeedIcon,
InstagramIcon,
RedditIcon,
TikTokIcon,
@ -91,6 +92,12 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [
icon: GoogleSearchIcon,
verbs: [{ name: "google_search.scrape", verb: "scrape", label: "Scrape" }],
},
{
id: "indeed",
label: "Indeed",
icon: IndeedIcon,
verbs: [{ name: "indeed.scrape", verb: "scrape", label: "Scrape" }],
},
{
id: "amazon",
label: "Amazon",

View file

@ -30,4 +30,5 @@ export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram")
export const TikTokIcon = brandIcon("/connectors/tiktok.svg", "TikTok");
export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps");
export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search");
export const IndeedIcon = brandIcon("/connectors/indeed.svg", "Indeed");
export const WebIcon = brandIcon("/connectors/web.svg", "Web");

View file

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Indeed" viewBox="0 0 24 24" width="24" height="24" fill="#003A9B"><title>Indeed</title><path d="M11.566 21.5633v-8.762c.2553.0231.5009.0346.758.0346 1.2225 0 2.3739-.3206 3.3506-.8928v9.6182c0 .8219-.1957 1.4287-.5757 1.8338-.378.4033-.8808.6049-1.491.6049-.6007 0-1.0766-.2016-1.468-.6183-.3781-.4032-.5739-1.01-.5739-1.8184zM11.589.5659c2.5447-.8929 5.4424-.8449 7.6186.987.405.3687.8673.8334 1.0515 1.3806.2207.6913-.7695-.073-.9057-.167-.71-.4532-1.4182-.8334-2.2127-1.0946C12.8614.3873 8.8122 2.709 6.2945 6.315c-1.0516 1.5939-1.7367 3.2721-2.299 5.1174-.0614.2017-.1094.4647-.2207.6413-.1113.2036-.048-.5453-.048-.5702.0845-.7623.2438-1.4997.4414-2.237C5.3292 5.3375 7.897 2.0655 11.5891.5658zm4.9281 7.0587c0 1.6686-1.353 3.0224-3.0205 3.0224-1.6677 0-3.0186-1.3538-3.0186-3.0224 0-1.6687 1.351-3.0224 3.0186-3.0224 1.6676 0 3.0205 1.3518 3.0205 3.0224Z"/></svg>

After

Width:  |  Height:  |  Size: 933 B