From 249bba08868b81d3e7c0b2796a47f2c10c89e86e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 01/56] feat: indeed scraper input/output schemas --- .../platforms/indeed_jobs/schemas.py | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/schemas.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/schemas.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/schemas.py new file mode 100644 index 000000000..27be45d9a --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/schemas.py @@ -0,0 +1,122 @@ +# ruff: noqa: N815 - field names intentionally use the public camelCase API +"""Input/output models for the Indeed scraper. + +Anonymous scraper: there is no auth field. Fields absent from a listing (full +description, benefits) stay ``None``/``[]`` until a detail fetch fills them. +""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +IndeedSort = Literal["relevance", "date"] +IndeedJobType = Literal[ + "fulltime", + "parttime", + "contract", + "internship", + "temporary", + "permanent", + "seasonal", + "freelance", +] +IndeedLevel = Literal["entry_level", "mid_level", "senior_level"] +IndeedRemote = Literal["remote", "hybrid"] +SalaryPeriod = Literal["hour", "day", "week", "month", "year"] + + +class StartUrl(BaseModel): + """A direct URL entry; extra keys ignored.""" + + model_config = ConfigDict(extra="allow") + + url: str + + +class IndeedScrapeInput(BaseModel): + """Indeed scraper input. Caps are collector policy, enforced by ``scrape_indeed``.""" + + model_config = ConfigDict(extra="allow") + + # Discovery: direct URLs and/or search queries. + startUrls: list[StartUrl] = Field(default_factory=list) + queries: list[str] = Field(default_factory=list) + + # Search parameters applied to ``queries``. + country: str = "us" + location: str | None = None + radius: int | None = None + jobType: IndeedJobType | None = None + level: IndeedLevel | None = None + remote: IndeedRemote | None = None + fromDays: int | None = None + sort: IndeedSort = "relevance" + + # Fetch each job's detail page for the full description. + scrapeJobDetails: bool = False + + maxItems: int = Field(default=25, ge=0) + maxItemsPerQuery: int = Field(default=25, ge=0) + + +class Salary(BaseModel): + """Salary block; fields are ``None`` when Indeed omits pay.""" + + model_config = ConfigDict(extra="allow") + + salaryText: str | None = None + salaryMin: float | None = None + salaryMax: float | None = None + currency: str | None = None + period: SalaryPeriod | None = None + isEstimated: bool | None = None + + +class IndeedItem(BaseModel): + """One job posting. ``extra="allow"`` keeps the contract additive.""" + + model_config = ConfigDict(extra="allow") + + jobKey: str | None = None + title: str | None = None + jobUrl: str | None = None + applyUrl: str | None = None + + company: str | None = None + companyUrl: str | None = None + companyRating: float | None = None + companyReviewCount: int | None = None + + formattedLocation: str | None = None + city: str | None = None + state: str | None = None + postalCode: str | None = None + country: str | None = None + isRemote: bool | None = None + remoteType: str | None = None + + jobTypes: list[str] = Field(default_factory=list) + salary: Salary = Field(default_factory=Salary) + benefits: list[str] = Field(default_factory=list) + + descriptionText: str | None = None + descriptionHtml: str | None = None + + sponsored: bool | None = None + isNew: bool | None = None + urgentlyHiring: bool | None = None + expired: bool | None = None + indeedApplyEnabled: bool | None = None + + age: str | None = None + datePublished: str | None = None + createdAt: str | None = None + scrapedAt: str | None = None + + source: str = "indeed" + + def to_output(self) -> dict[str, Any]: + """Serialize to the flat output dict, keeping extras.""" + return self.model_dump(exclude_none=False) From 01c3f097e6bfc53fa0eb961f7555566052f2ca26 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 02/56] feat: indeed jobcards blob extraction and job parser --- .../platforms/indeed_jobs/parsers.py | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py new file mode 100644 index 000000000..ab2b97cf9 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py @@ -0,0 +1,228 @@ +"""Pure HTML/JSON -> item mapping for the Indeed scraper. + +I/O-free and deterministic so it can be unit-tested against captured fixtures; +the orchestrator stamps ``scrapedAt``. + +Indeed embeds job data as a JS assignment:: + + window.mosaic.providerData["mosaic-provider-jobcards"]={"metaData":{...}}; + +The literal recurs hundreds of times in the page, so :func:`extract_jobcards_blob` +anchors on the assignment (``...]=``) and brace-matches the balanced object. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from html import unescape +from re import sub as _re_sub +from typing import Any + +_DEFAULT_BASE = "https://www.indeed.com" + +_JOBCARDS_ANCHOR = 'window.mosaic.providerData["mosaic-provider-jobcards"]=' + +# Indeed's extractedSalary.type -> our SalaryPeriod. +_SALARY_PERIODS = { + "HOURLY": "hour", + "DAILY": "day", + "WEEKLY": "week", + "MONTHLY": "month", + "YEARLY": "year", +} + + +def _brace_match(text: str, start: int) -> str | None: + """Return the balanced ``{...}``/``[...]`` blob at ``text[start]``, quote-aware.""" + open_ch = text[start] if start < len(text) else "" + close_ch = {"[": "]", "{": "}"}.get(open_ch) + if close_ch is None: + return None + depth = 0 + i = start + n = len(text) + while i < n: + ch = text[i] + if ch == open_ch: + depth += 1 + elif ch == close_ch: + depth -= 1 + if depth == 0: + return text[start : i + 1] + elif ch == '"': + i += 1 + while i < n and text[i] != '"': + if text[i] == "\\": + i += 1 + i += 1 + i += 1 + return None + + +def extract_jobcards_blob(html: str) -> dict | None: + """Decode the ``mosaic-provider-jobcards`` assignment, or ``None`` if absent.""" + import json + + idx = html.find(_JOBCARDS_ANCHOR) + if idx == -1: + return None + brace = html.find("{", idx + len(_JOBCARDS_ANCHOR)) + if brace == -1: + return None + blob = _brace_match(html, brace) + if not blob: + return None + try: + data = json.loads(blob) + except ValueError: + return None + return data if isinstance(data, dict) else None + + +def job_results(blob: dict | None) -> list[dict[str, Any]]: + """Return the raw job records from a decoded blob.""" + if not isinstance(blob, dict): + return [] + results = ( + blob.get("metaData", {}).get("mosaicProviderJobCardsModel", {}).get("results") + ) + if not isinstance(results, list): + return [] + return [r for r in results if isinstance(r, dict)] + + +def _utc_from_ms(value: Any) -> str | None: + """Epoch milliseconds -> millisecond ISO string.""" + if isinstance(value, bool) or not isinstance(value, int | float): + return None + dt = datetime.fromtimestamp(float(value) / 1000.0, tz=UTC) + return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def _int(value: Any) -> int | None: + """Coerce to int, dropping bools.""" + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + return None + + +def _abs_url(path: Any, base_url: str) -> str | None: + """Resolve an Indeed-relative path against ``base_url``; keep absolute URLs.""" + if not isinstance(path, str) or not path: + return None + if path.startswith("http"): + return path + return f"{base_url}{path if path.startswith('/') else '/' + path}" + + +def _clean_snippet(snippet: Any) -> str | None: + """Strip tags and decode entities into plain text.""" + if not isinstance(snippet, str) or not snippet: + return None + text = _re_sub(r"<[^>]+>", " ", snippet) + text = unescape(text) + return _re_sub(r"\s+", " ", text).strip() or None + + +def _taxonomy(raw: dict[str, Any]) -> dict[str, list[str]]: + """Flatten ``taxonomyAttributes`` into ``{group label: [attribute labels]}``.""" + out: dict[str, list[str]] = {} + for group in raw.get("taxonomyAttributes") or []: + if not isinstance(group, dict): + continue + label = group.get("label") + attrs = group.get("attributes") + if isinstance(label, str) and isinstance(attrs, list): + out[label] = [ + a["label"] + for a in attrs + if isinstance(a, dict) and isinstance(a.get("label"), str) + ] + return out + + +def _job_types(raw: dict[str, Any], taxo: dict[str, list[str]]) -> list[str]: + """Job types from ``jobTypes`` then the taxonomy, deduped and order-stable.""" + seen: dict[str, None] = {} + for jt in raw.get("jobTypes") or []: + if isinstance(jt, str): + seen.setdefault(jt, None) + for label in ("job-types", "job-types-cc"): + for jt in taxo.get(label, []): + seen.setdefault(jt, None) + return list(seen) + + +def _salary(raw: dict[str, Any]) -> dict[str, Any]: + """Flatten salary from ``salarySnippet`` (text) + ``extractedSalary`` (bounds).""" + snippet = raw.get("salarySnippet") or {} + extracted = raw.get("extractedSalary") or {} + estimated = raw.get("estimatedSalary") or {} + source = extracted or estimated + text = snippet.get("text") if isinstance(snippet, dict) else None + return { + "salaryText": text if isinstance(text, str) else None, + "salaryMin": source.get("min") if isinstance(source, dict) else None, + "salaryMax": source.get("max") if isinstance(source, dict) else None, + "currency": snippet.get("currency") if isinstance(snippet, dict) else None, + "period": _SALARY_PERIODS.get( + source.get("type") if isinstance(source, dict) else None + ), + "isEstimated": bool(estimated) and not extracted, + } + + +def _is_remote(raw: dict[str, Any], taxo: dict[str, list[str]]) -> bool: + """Resolve remote/hybrid across Indeed's several signals.""" + if raw.get("remoteLocation") is True: + return True + if isinstance(raw.get("remoteWorkModel"), dict): + return True + return bool(taxo.get("remote")) + + +def parse_job(raw: dict[str, Any], *, base_url: str = _DEFAULT_BASE) -> dict[str, Any]: + """Map one raw ``results[]`` record to a flat item dict. + + ``base_url`` is the country domain the record came from, so job and company + URLs resolve to the right host. + """ + taxo = _taxonomy(raw) + job_key = raw.get("jobkey") + remote_model = raw.get("remoteWorkModel") + return { + "jobKey": job_key if isinstance(job_key, str) else None, + "title": raw.get("displayTitle") or raw.get("title"), + "jobUrl": f"{base_url}/viewjob?jk={job_key}" if job_key else None, + "applyUrl": raw.get("thirdPartyApplyUrl") or None, + "company": raw.get("company") or raw.get("truncatedCompany"), + "companyUrl": _abs_url(raw.get("companyOverviewLink"), base_url), + "companyRating": raw.get("companyRating"), + "companyReviewCount": _int(raw.get("companyReviewCount")), + "formattedLocation": raw.get("formattedLocation"), + "city": raw.get("jobLocationCity"), + "state": raw.get("jobLocationState"), + "postalCode": raw.get("jobLocationPostal"), + "country": raw.get("country"), + "isRemote": _is_remote(raw, taxo), + "remoteType": remote_model.get("type") + if isinstance(remote_model, dict) + else None, + "jobTypes": _job_types(raw, taxo), + "salary": _salary(raw), + "benefits": taxo.get("benefits", []), + "descriptionText": _clean_snippet(raw.get("snippet")), + "descriptionHtml": None, + "sponsored": raw.get("sponsored"), + "isNew": raw.get("newJob"), + "urgentlyHiring": raw.get("urgentlyHiring"), + "expired": raw.get("expired"), + "indeedApplyEnabled": raw.get("indeedApplyEnabled"), + "age": raw.get("formattedRelativeTime"), + "datePublished": _utc_from_ms(raw.get("pubDate")), + "createdAt": _utc_from_ms(raw.get("createDate")), + } From d87c86dcc41bd4d8a4013d2ff954f31bc86b9663 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 03/56] feat: indeed url classifier and search url builder --- .../platforms/indeed_jobs/url_resolver.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/url_resolver.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/url_resolver.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/url_resolver.py new file mode 100644 index 000000000..c69d3fe19 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/url_resolver.py @@ -0,0 +1,124 @@ +"""Classify Indeed URLs and build search URLs. + +Recognizes search pages (``/jobs?q=&l=``), company pages (``/cmp//jobs``), +and single jobs (``/viewjob?jk=``); other hosts resolve to ``None``. Also owns +the country->domain map so classification and URL building share one source. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal +from urllib.parse import parse_qs, urlencode, urlparse + +ResolvedKind = Literal["search", "company", "job"] + +# Locale subdomains that deviate from the ISO code; others map to .indeed.com. +_DOMAIN_OVERRIDES = {"us": "www", "gb": "uk"} + +_JT_VALUES = frozenset( + { + "fulltime", + "parttime", + "contract", + "internship", + "temporary", + "permanent", + "seasonal", + "freelance", + } +) + + +@dataclass(frozen=True) +class ResolvedUrl: + kind: ResolvedKind + value: str # search query, company slug, or job key + url: str + domain: str + location: str | None = None + params: dict[str, str] = field(default_factory=dict) + + +def _is_indeed_host(hostname: str | None) -> bool: + if not hostname: + return False + h = hostname.lower() + return h == "indeed.com" or h.endswith(".indeed.com") + + +def country_domain(country: str) -> str: + """Country code -> Indeed host, e.g. ``us`` -> ``www.indeed.com``.""" + cc = (country or "us").strip().lower() + return f"{_DOMAIN_OVERRIDES.get(cc, cc)}.indeed.com" + + +def resolve_url(url: str) -> ResolvedUrl | None: + """Classify an Indeed URL into a scrape job, or ``None`` if unrecognized.""" + parsed = urlparse(url) + if not _is_indeed_host(parsed.hostname): + return None + domain = parsed.hostname or "www.indeed.com" + path = (parsed.path or "").rstrip("/") + query = parse_qs(parsed.query) + segments = [s for s in path.split("/") if s] + + # /viewjob?jk= + if path.endswith("/viewjob") or segments[:1] == ["viewjob"]: + jk = query.get("jk", [None])[0] + return ResolvedUrl("job", jk, url, domain) if jk else None + + # /cmp//jobs + if segments[:1] == ["cmp"] and "jobs" in segments and len(segments) >= 2: + return ResolvedUrl("company", segments[1], url, domain) + + # /jobs?q=&l= + if path.endswith("/jobs") or segments[-1:] == ["jobs"]: + q = query.get("q", [""])[0] + loc = query.get("l", [None])[0] + extra = { + k: v[0] + for k, v in query.items() + if k in ("radius", "sort", "fromage", "jt", "explvl") and v + } + return ResolvedUrl("search", q, url, domain, location=loc, params=extra) + + return None + + +def build_search_url( + query: str, + *, + country: str = "us", + location: str | None = None, + radius: int | None = None, + job_type: str | None = None, + level: str | None = None, + remote: str | None = None, + from_days: int | None = None, + sort: str = "relevance", + start: int = 0, +) -> str: + """Build an Indeed ``/jobs`` search URL. + + Remote/hybrid is passed as a query keyword; Indeed's structured ``sc`` + attribute codes rotate and aren't stable to hardcode. + """ + domain = country_domain(country) + q = f"{query} {remote}".strip() if remote else query + params: dict[str, str] = {"q": q} + if location: + params["l"] = location + if radius is not None: + params["radius"] = str(radius) + if job_type in _JT_VALUES: + params["jt"] = job_type # type: ignore[assignment] + if level: + params["explvl"] = level + if from_days is not None: + params["fromage"] = str(from_days) + if sort == "date": + params["sort"] = "date" + if start: + params["start"] = str(start) + return f"https://{domain}/jobs?{urlencode(params)}" From 44f9f0301c81f7c6e778e8985b178f667112ab67 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 04/56] feat: indeed warmed browser session with rotate-on-block --- .../platforms/indeed_jobs/fetch.py | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py new file mode 100644 index 000000000..9f913a579 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py @@ -0,0 +1,184 @@ +"""Browser-session fetch seam for the Indeed scraper. + +Indeed fronts its origin with Cloudflare plus an anonymous-bot check that bounces +cold sessions to ``secure.indeed.com/auth``. The working recipe: a persistent +camoufox session that solves Cloudflare, warms on the domain home page, then +navigates to ``/jobs`` in the same context so the clearance carries. + +:class:`IndeedSession` warms per domain once, retries a blocked page on a fresh +residential IP, and caps each navigation with a hard timeout so a stuck solve +can't stall a run. All egress is through the residential proxy. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from contextlib import asynccontextmanager, suppress +from datetime import UTC, datetime +from typing import Any, Protocol +from urllib.parse import urlparse + +from app.utils.proxy import get_proxy_url + +logger = logging.getLogger(__name__) + + +class IndeedAccessBlockedError(RuntimeError): + """Every rotated IP was bounced to Indeed's security wall.""" + + +# Per navigation; a stuck Cloudflare solve otherwise hangs the whole run. +_PAGE_TIMEOUT_S = 75.0 +# Browser-internal timeout; kept above the page timeout so ours fires first. +_SESSION_TIMEOUT_MS = 90_000 +_MAX_ROTATIONS = 3 + +# Markers of a Cloudflare / security-check interstitial served instead of jobs. +_BLOCK_MARKERS = ( + "secure.indeed.com", + "bot-detection", + "security check", + "challenge-platform", + "just a moment", + "verify you are human", + "hcaptcha", +) + + +def now_iso() -> str: + """UTC timestamp in the millisecond ISO shape used by scraper output.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +class _Session(Protocol): + """Minimal browser-session surface used here (real or fake).""" + + async def start(self) -> Any: ... + async def fetch(self, url: str, **kwargs: Any) -> Any: ... + async def close(self) -> Any: ... + + +def _default_session_factory() -> _Session: + """Build a proxied, Cloudflare-solving camoufox session. + + ``disable_resources`` skips images/fonts/media; job data is inline in the + document, so this only trims bandwidth. + """ + from scrapling.fetchers import AsyncStealthySession + + return AsyncStealthySession( + headless=True, + solve_cloudflare=True, + network_idle=True, + block_webrtc=True, + disable_resources=True, + timeout=_SESSION_TIMEOUT_MS, + proxy=get_proxy_url(), + ) + + +def _html(page: Any) -> str: + """Best-effort HTML body across scrapling response shapes.""" + for attr in ("html_content", "body", "text"): + val = getattr(page, attr, None) + if isinstance(val, bytes): + val = val.decode("utf-8", "replace") + if isinstance(val, str) and val: + return val + return "" + + +def _looks_blocked(html: str, final_url: str) -> bool: + """Whether a response is an interstitial rather than a real page.""" + if not html: + return True + haystack = (final_url + " " + html[:6000]).lower() + return any(marker in haystack for marker in _BLOCK_MARKERS) + + +class IndeedSession: + """One warmed browser session that rotates its exit IP when blocked.""" + + def __init__( + self, session_factory: Callable[[], _Session] = _default_session_factory + ) -> None: + self._factory = session_factory + self._session: _Session | None = None + self._warmed: set[str] = set() + self.rotations = 0 + + async def start(self) -> None: + self._session = self._factory() + await self._session.start() + + async def close(self) -> None: + if self._session is not None: + with suppress(Exception): + await self._session.close() + self._session = None + self._warmed.clear() + + async def _rotate(self) -> None: + """Drop the session for a fresh exit IP; clears warmed domains.""" + await self.close() + self.rotations += 1 + await self.start() + logger.info("[indeed] rotated session (rotation #%d)", self.rotations) + + async def _timed_fetch(self, url: str, **kwargs: Any) -> Any: + assert self._session is not None + coro: Awaitable[Any] = self._session.fetch(url, **kwargs) + return await asyncio.wait_for(coro, timeout=_PAGE_TIMEOUT_S) + + async def _ensure_warm(self, domain: str) -> None: + """Land on the domain home with a Google referer before scraping it.""" + if domain in self._warmed: + return + with suppress(Exception): + await self._timed_fetch(f"https://{domain}/", google_search=True) + self._warmed.add(domain) + + async def fetch_html(self, url: str) -> str: + """Return a search/company/job page's HTML through the warmed session. + + Rotates the IP and re-warms on a security-wall bounce or timeout; raises + :class:`IndeedAccessBlockedError` once every rotation is exhausted. + """ + if self._session is None: + await self.start() + domain = urlparse(url).hostname or "www.indeed.com" + attempt = 0 + while True: + try: + await self._ensure_warm(domain) + page = await self._timed_fetch(url) + html = _html(page) + if not _looks_blocked(html, str(getattr(page, "url", "") or "")): + return html + logger.info("[indeed] blocked on %s", url) + except TimeoutError: + logger.warning("[indeed] fetch timed out on %s", url) + except Exception as e: + logger.warning("[indeed] fetch failed on %s: %s", url, e) + + if attempt >= _MAX_ROTATIONS: + raise IndeedAccessBlockedError( + f"Indeed refused {url} on {attempt} rotated IPs" + ) + attempt += 1 + await self._rotate() + + +@asynccontextmanager +async def open_session( + session_factory: Callable[[], _Session] = _default_session_factory, +): + """Open an :class:`IndeedSession` and guarantee teardown.""" + session = IndeedSession(session_factory) + await session.start() + try: + yield session + finally: + await session.close() From 824041ef7fef6c3ec09c2509098ee21e059ef405 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 05/56] feat: indeed scraper orchestrator with pagination and dedupe --- .../platforms/indeed_jobs/scraper.py | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py new file mode 100644 index 000000000..4f66f2ebf --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py @@ -0,0 +1,156 @@ +"""Orchestrator for the Indeed scraper. + +:func:`iter_indeed` streams flat job items from one warmed browser session: +``startUrls`` (search/company pages) or ``queries`` are paged by the ``start`` +offset, deduped by ``jobKey``, and stopped on the first page that adds nothing. +:func:`scrape_indeed` collects the stream under a caller ``limit``. + +Targets run sequentially on a single session — a browser per target would cost +far more than the sequential paging it would save. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse + +from .fetch import IndeedSession, now_iso, open_session +from .parsers import extract_jobcards_blob, job_results, parse_job +from .schemas import IndeedItem, IndeedScrapeInput +from .url_resolver import build_search_url, resolve_url + +logger = logging.getLogger(__name__) + +__all__ = ["iter_indeed", "scrape_indeed"] + +# Indeed's search pagination increments the ``start`` offset by 10. +_PAGE_STEP = 10 +# Indeed stops serving useful results past ~1000; cap pages well before that. +_MAX_PAGES = 10 + + +def _emit(partial: dict[str, Any]) -> dict[str, Any]: + """Stamp ``scrapedAt`` and normalize through the output model.""" + return IndeedItem(**{**partial, "scrapedAt": now_iso()}).to_output() + + +def _with_start(url: str, start: int) -> str: + """Return ``url`` with its ``start`` query param set (removed when 0).""" + parsed = urlparse(url) + params = [(k, v) for k, v in parse_qsl(parsed.query) if k != "start"] + if start: + params.append(("start", str(start))) + return urlunparse(parsed._replace(query=urlencode(params))) + + +async def _paginate( + session: IndeedSession, first_url: str, *, domain: str, max_items: int +) -> AsyncIterator[dict[str, Any]]: + """Yield items across pages of one search/company URL, deduped by ``jobKey``.""" + if max_items <= 0: + return + base_url = f"https://{domain}" + seen: set[str] = set() + emitted = 0 + for page in range(_MAX_PAGES): + html = await session.fetch_html(_with_start(first_url, page * _PAGE_STEP)) + raws = job_results(extract_jobcards_blob(html)) + if not raws: + return + added = 0 + for raw in raws: + item = parse_job(raw, base_url=base_url) + job_key = item.get("jobKey") + if isinstance(job_key, str): + if job_key in seen: + continue + seen.add(job_key) + yield _emit(item) + emitted += 1 + added += 1 + if emitted >= max_items: + return + if added == 0: # a whole page of duplicates: end of useful results + return + + +def _targets(input_model: IndeedScrapeInput) -> list[tuple[str, str]]: + """Resolve inputs to ``(url, domain)`` search/company targets. + + ``startUrls`` take precedence over ``queries``. Job (``/viewjob``) URLs are + skipped here; detail enrichment is a separate flow. + """ + if input_model.startUrls: + out: list[tuple[str, str]] = [] + for entry in input_model.startUrls: + resolved = resolve_url(entry.url) + if resolved is None or resolved.kind == "job": + logger.warning("[indeed] skipping unsupported URL: %s", entry.url) + continue + out.append((resolved.url, resolved.domain)) + return out + + domain = None + urls: list[tuple[str, str]] = [] + for query in input_model.queries: + url = build_search_url( + query, + country=input_model.country, + location=input_model.location, + radius=input_model.radius, + job_type=input_model.jobType, + level=input_model.level, + remote=input_model.remote, + from_days=input_model.fromDays, + sort=input_model.sort, + ) + domain = domain or urlparse(url).hostname or "www.indeed.com" + urls.append((url, domain)) + return urls + + +async def iter_indeed( + input_model: IndeedScrapeInput, session: IndeedSession +) -> AsyncIterator[dict[str, Any]]: + """Stream flat job items for every target, deduped by ``jobKey`` across all.""" + global_seen: set[str] = set() + for url, domain in _targets(input_model): + async for item in _paginate( + session, url, domain=domain, max_items=input_model.maxItemsPerQuery + ): + job_key = item.get("jobKey") + if isinstance(job_key, str): + if job_key in global_seen: + continue + global_seen.add(job_key) + yield item + + +async def scrape_indeed( + input_model: IndeedScrapeInput, + *, + limit: int | None = None, + session: IndeedSession | None = None, +) -> list[dict[str, Any]]: + """Collect :func:`iter_indeed` into a list under an optional ``limit``. + + Opens a warmed session when one is not supplied. ``limit`` is a request-time + guard, not a ceiling baked into the stream. + """ + from app.capabilities.core.progress import emit_progress + + async def _collect(sess: IndeedSession) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + async for item in iter_indeed(input_model, sess): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + break + return results + + if session is not None: + return await _collect(session) + async with open_session() as sess: + return await _collect(sess) From 1d367148d009bf3eab4a3fc320eb7c1d7038e90a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 06/56] feat: indeed_jobs package exports --- .../proprietary/platforms/indeed_jobs/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/__init__.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/__init__.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/__init__.py new file mode 100644 index 000000000..eb59671d7 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/__init__.py @@ -0,0 +1,13 @@ +"""Platform-native Indeed jobs scraper (anonymous, warmed browser session).""" + +from .fetch import IndeedAccessBlockedError +from .schemas import IndeedItem, IndeedScrapeInput +from .scraper import iter_indeed, scrape_indeed + +__all__ = [ + "IndeedAccessBlockedError", + "IndeedItem", + "IndeedScrapeInput", + "iter_indeed", + "scrape_indeed", +] From 794b127136b411060a8aba037ad40ae8280ac0ae Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 07/56] test: indeed_jobs test package --- surfsense_backend/tests/unit/platforms/indeed_jobs/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/__init__.py diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/__init__.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/__init__.py new file mode 100644 index 000000000..e69de29bb From 3d6da91bf36112bf0995628395ff7625ce061bb7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 08/56] test: indeed trimmed jobcards fixture --- .../indeed_jobs/fixtures/sample_jobcards.json | 1633 +++++++++++++++++ 1 file changed, 1633 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_jobcards.json diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_jobcards.json b/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_jobcards.json new file mode 100644 index 000000000..9faa61ef7 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_jobcards.json @@ -0,0 +1,1633 @@ +{ + "metaData": { + "mosaicProviderJobCardsModel": { + "results": [ + { + "adBlob": "-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N1lyYEY1nzdWnmTq7hlQAqMT4EZoUMRYAQrNEOj5nb4ZE_F_ZCNcUyWwXqUnsb3IFHbnllXq0f5pYhflUexN3LD6mf9j8U6fPKOFmTQuul154CxQeo8aExEApf08b2iFfAx7Eci6fsR4i5wlRjvKA5EumK1U1Eauxzua2oUUaLKdAW9137H0CYgdcoAMF84PbbBCj5eW5R0p7Tfgi9NmbhJFRaEZv-u8JEg9U1zcdmaIh6RuPDU7bjQIuak5MO3DwaEJlHXEg4v7-XpR0uTAehVEkbNEFQdl0EwMnI32HxxwGJ_r7Xy7viI3kh6lEdrpZeF1ca18i9FlF_nce_H7BdzO2qmsM11dLrzIwGnipd9-NCmqh-jZJbJLooc2Yl3D8379SRHWpbfMz_a9aa1x_miN7s9YFssX_W8wrIqe3eHIaV-pRMzXMJYIxrcyPztopcC5udht92l9RFkJ8GjCK-zMHwiJRuCIqjML2cMsQblSxIpv5pb9M88dlUMlDd6dMpQBDYRrFe1PYyRu_QTWc_ggES6VZu7PJ7BcMPSFZ7cSuzuPT49lMRw7lYs3P1aakqoiFThrgWJAWATgalPtzZy754uEQG6ez5Gh-KW-nvDhDDfUS-fhvIMcDFftBf6gyU6_wEuH9GI7sOTR8b7jnHS285ryLd212LhBy0GJjliQdMxRmWK0qDqFr8z9q7Ufi9AHPafSIWTFw==", + "adId": "464623129", + "advn": "9533474731693465", + "appliedOrGreater": false, + "applyCount": -1, + "assistedApply": false, + "autoSourcerJob": false, + "bidPosition": 1, + "blobKey": "SoBj6_M3hwflOvyN8p0LbzkdCdPP", + "company": "TherapyNotes.com", + "companyBrandingAttributes": { + "brandingReasons": [ + "PAID_BRANDING" + ], + "brandingReasonsAsString": [ + "PAID_BRANDING" + ], + "headerImageUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_headerimage/1960x400/9d05368a0956c3266ea4291e95d34331", + "logoUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_squarelogo/256x256/eac29e66f882990ae917c959915eb291", + "paidBrandingDealIds": [ + "58664", + "95349" + ], + "showJobBranding": true, + "shownForBrandedJobPackage": false + }, + "companyIdEncrypted": "d10e9cc0e75a9595", + "companyOverviewLink": "/cmp/Therapynotes", + "companyOverviewLinkCampaignId": "serp-linkcompanyname-content", + "companyRating": 3.8, + "companyReviewCount": 30, + "companyReviewLink": "/cmp/Therapynotes/reviews", + "companyReviewLinkCampaignId": "cmplinktst2", + "country": "US", + "createDate": 1774276267415, + "d2iEnabled": false, + "displayTitle": "Software Developer", + "dradisJob": false, + "employerAssistEnabled": false, + "employerResponseTime": 1, + "employerResponsive": true, + "encryptedFccompanyId": "c5ef9b9375a8bbf1", + "encryptedResultData": "VwIPTVJ1cTn5AN7Q-tSqGRXGNe2wB2UYx73qSczFnGU", + "enhancedAttributesModel": {}, + "enticers": [], + "expired": false, + "extractTrackingUrls": "", + "extractedSalary": { + "max": 115000, + "min": 65000, + "type": "YEARLY" + }, + "fccompanyId": -1, + "featuredCompanyAttributes": {}, + "featuredEmployer": false, + "featuredEmployerCandidate": false, + "feedId": 822778, + "formattedLocation": "Philadelphia, PA", + "formattedRelativeTime": "30+ days ago", + "gatedVjp": false, + "hideMetaData": false, + "highVolumeHiringModel": { + "highVolumeHiring": false + }, + "hiringEventJob": false, + "homepageJobFeedSectionId": "0", + "indeedApplyEnabled": true, + "indeedApplyFinishAppUrlEnabled": false, + "indeedApplyResumeType": "required", + "indeedApplyable": true, + "isJobVisited": false, + "isMobileThirdPartyApplyable": false, + "isNoResumeJob": false, + "isSubsidiaryJob": false, + "isTopRatedEmployer": false, + "jobCardRequirementsModel": { + "additionalRequirementsCount": 0, + "jobOnlyRequirements": [], + "jobTagRequirements": [], + "requirementsHeaderShown": false, + "screenerQuestionRequirements": [] + }, + "jobFlairPackageEnabled": false, + "jobLocationCity": "Philadelphia", + "jobLocationState": "PA", + "jobSeekerMatchSummaryModel": { + "sortedEntityDisplayText": [], + "sortedMatchingEntityDisplayText": [], + "sortedMisMatchingEntityDisplayText": [ + ".NET Core", + "Agile software development", + "Bachelor's degree", + "Communication skills", + "Computer Science", + "Computer science", + "Design (software development lifecycle)", + "Developing and maintaining backend systems", + "Entity Framework", + "Event-driven architecture", + "Front-end component implementation", + "HTML", + "JavaScript frameworks", + "Planning (software development lifecycle)", + "PostgreSQL", + "Project development phase management", + "Providing code feedback", + "Responsive design", + "SASS/LESS", + "SQL databases", + "Scalability", + "Scalable systems", + "Software Engineering", + "Software engineering", + "TypeScript", + "Web applications", + "Web services design" + ], + "taxoEntityMatchesNegative": [ + { + "displayText": "SASS/LESS", + "id": "", + "rawName": "SASS/LESS", + "source": "JOB", + "strictness": "", + "suid": "FE6SC" + }, + { + "displayText": "Computer science", + "id": "", + "rawName": "Computer science", + "source": "JOB", + "strictness": "", + "suid": "4E4WW" + }, + { + "displayText": "JavaScript frameworks", + "id": "", + "rawName": "JavaScript frameworks", + "source": "JOB", + "strictness": "", + "suid": "EUPT5" + }, + { + "displayText": "HTML", + "id": "", + "rawName": "HTML", + "source": "JOB", + "strictness": "", + "suid": "Y7U37" + }, + { + "displayText": "Developing and maintaining backend systems", + "id": "", + "rawName": "Developing and maintaining backend systems", + "source": "JOB", + "strictness": "", + "suid": "Q3QNF" + }, + { + "displayText": "Design (software development lifecycle)", + "id": "", + "rawName": "Design (software development lifecycle)", + "source": "JOB", + "strictness": "", + "suid": "XA6CW" + }, + { + "displayText": "Responsive design", + "id": "", + "rawName": "Responsive design", + "source": "JOB", + "strictness": "", + "suid": "36ZHE" + }, + { + "displayText": "Scalable systems", + "id": "", + "rawName": "Scalable systems", + "source": "JOB", + "strictness": "", + "suid": "DP9CS" + }, + { + "displayText": "SQL databases", + "id": "", + "rawName": "SQL databases", + "source": "JOB", + "strictness": "", + "suid": "7G8UN" + }, + { + "displayText": "Software Engineering", + "id": "", + "rawName": "Software Engineering", + "source": "JOB", + "strictness": "", + "suid": "7F6H9" + }, + { + "displayText": "Web applications", + "id": "", + "rawName": "Web applications", + "source": "JOB", + "strictness": "", + "suid": "QM3S8" + }, + { + "displayText": "Entity Framework", + "id": "", + "rawName": "Entity Framework", + "source": "JOB", + "strictness": "", + "suid": "X5HXD" + }, + { + "displayText": "Computer Science", + "id": "", + "rawName": "Computer Science", + "source": "JOB", + "strictness": "", + "suid": "6XNCP" + }, + { + "displayText": "Providing code feedback", + "id": "", + "rawName": "Providing code feedback", + "source": "JOB", + "strictness": "", + "suid": "Z4F5N" + }, + { + "displayText": "PostgreSQL", + "id": "", + "rawName": "PostgreSQL", + "source": "JOB", + "strictness": "", + "suid": "JCKB4" + }, + { + "displayText": ".NET Core", + "id": "", + "rawName": ".NET Core", + "source": "JOB", + "strictness": "", + "suid": "AGW49" + }, + { + "displayText": "TypeScript", + "id": "", + "rawName": "TypeScript", + "source": "JOB", + "strictness": "", + "suid": "WD7PP" + }, + { + "displayText": "Scalability", + "id": "", + "rawName": "Scalability", + "source": "JOB", + "strictness": "", + "suid": "PYTWK" + }, + { + "displayText": "Agile software development", + "id": "", + "rawName": "Agile software development", + "source": "JOB", + "strictness": "", + "suid": "QWGDE" + }, + { + "displayText": "Web services design", + "id": "", + "rawName": "Web services design", + "source": "JOB", + "strictness": "", + "suid": "ZN5SR" + }, + { + "displayText": "Project development phase management", + "id": "", + "rawName": "Project development phase management", + "source": "JOB", + "strictness": "", + "suid": "4UEV6" + }, + { + "displayText": "Bachelor's degree", + "id": "", + "rawName": "Bachelor's degree", + "source": "JOB", + "strictness": "", + "suid": "HFDVW" + }, + { + "displayText": "Software engineering", + "id": "", + "rawName": "Software engineering", + "source": "JOB", + "strictness": "", + "suid": "4R4AM" + }, + { + "displayText": "Event-driven architecture", + "id": "", + "rawName": "Event-driven architecture", + "source": "JOB", + "strictness": "", + "suid": "G82AR" + }, + { + "displayText": "Front-end component implementation", + "id": "", + "rawName": "Front-end component implementation", + "source": "JOB", + "strictness": "", + "suid": "RP4CB" + }, + { + "displayText": "Communication skills", + "id": "", + "rawName": "Communication skills", + "source": "JOB", + "strictness": "", + "suid": "WSBNK" + }, + { + "displayText": "Planning (software development lifecycle)", + "id": "", + "rawName": "Planning (software development lifecycle)", + "source": "JOB", + "strictness": "", + "suid": "X9WB7" + } + ], + "taxoEntityMatchesPositive": [], + "trafficLight": "yellow" + }, + "jobTypes": [], + "jobkey": "7b993d015f4b257a", + "jsiEnabled": false, + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N1lyYEY1nzdWnmTq7hlQAqMT4EZoUMRYAQrNEOj5nb4ZE_F_ZCNcUyWwXqUnsb3IFHbnllXq0f5pYhflUexN3LD6mf9j8U6fPKOFmTQuul154CxQeo8aExEApf08b2iFfAx7Eci6fsR4i5wlRjvKA5EumK1U1Eauxzua2oUUaLKdAW9137H0CYgdcoAMF84PbbBCj5eW5R0p7Tfgi9NmbhJFRaEZv-u8JEg9U1zcdmaIh6RuPDU7bjQIuak5MO3DwaEJlHXEg4v7-XpR0uTAehVEkbNEFQdl0EwMnI32HxxwGJ_r7Xy7viI3kh6lEdrpZeF1ca18i9FlF_nce_H7BdzO2qmsM11dLrzIwGnipd9-NCmqh-jZJbJLooc2Yl3D8379SRHWpbfMz_a9aa1x_miN7s9YFssX_W8wrIqe3eHIaV-pRMzXMJYIxrcyPztopcC5udht92l9RFkJ8GjCK-zMHwiJRuCIqjML2cMsQblSxIpv5pb9M88dlUMlDd6dMpQBDYRrFe1PYyRu_QTWc_ggES6VZu7PJ7BcMPSFZ7cSuzuPT49lMRw7lYs3P1aakqoiFThrgWJAWATgalPtzZy754uEQG6ez5Gh-KW-nvDhDDfUS-fhvIMcDFftBf6gyU6_wEuH9GI7sOTR8b7jnHS285ryLd212LhBy0GJjliQdMxRmWK0qDqFr8z9q7Ufi9AHPafSIWTFw==&xkcb=SoBj6_M3hwflOvyN8p0LbzkdCdPP&p=0&camk=C3EPSzFlQw-JWkl1H7Zuog==&vjs=3", + "locationCount": 1, + "minimumCount": 1, + "mobtk": "1jtgbqmn921k6001", + "moreLocUrl": "/jobs?q=software+engineer&nl=&l=Remote&radius=50&jtid=6804610d02b67694&jcid=d10e9cc0e75a9595&grp=tcl", + "mouseDownHandlerOption": { + "adId": 464623129, + "advn": "9533474731693465", + "extractTrackingUrls": [], + "from": "vjs", + "jobKey": "7b993d015f4b257a", + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N1lyYEY1nzdWnmTq7hlQAqMT4EZoUMRYAQrNEOj5nb4ZE_F_ZCNcUyWwXqUnsb3IFHbnllXq0f5pYhflUexN3LD6mf9j8U6fPKOFmTQuul154CxQeo8aExEApf08b2iFfAx7Eci6fsR4i5wlRjvKA5EumK1U1Eauxzua2oUUaLKdAW9137H0CYgdcoAMF84PbbBCj5eW5R0p7Tfgi9NmbhJFRaEZv-u8JEg9U1zcdmaIh6RuPDU7bjQIuak5MO3DwaEJlHXEg4v7-XpR0uTAehVEkbNEFQdl0EwMnI32HxxwGJ_r7Xy7viI3kh6lEdrpZeF1ca18i9FlF_nce_H7BdzO2qmsM11dLrzIwGnipd9-NCmqh-jZJbJLooc2Yl3D8379SRHWpbfMz_a9aa1x_miN7s9YFssX_W8wrIqe3eHIaV-pRMzXMJYIxrcyPztopcC5udht92l9RFkJ8GjCK-zMHwiJRuCIqjML2cMsQblSxIpv5pb9M88dlUMlDd6dMpQBDYRrFe1PYyRu_QTWc_ggES6VZu7PJ7BcMPSFZ7cSuzuPT49lMRw7lYs3P1aakqoiFThrgWJAWATgalPtzZy754uEQG6ez5Gh-KW-nvDhDDfUS-fhvIMcDFftBf6gyU6_wEuH9GI7sOTR8b7jnHS285ryLd212LhBy0GJjliQdMxRmWK0qDqFr8z9q7Ufi9AHPafSIWTFw==&xkcb=SoBj6_M3hwflOvyN8p0LbzkdCdPP&p=0&camk=C3EPSzFlQw-JWkl1H7Zuog==&vjs=3", + "tk": "1jtgbqmn921k6001" + }, + "newJob": false, + "normTitle": "Software Engineer", + "numHires": 0, + "openInterviewsInterviewsOnTheSpot": false, + "openInterviewsJob": false, + "openInterviewsOffersOnTheSpot": false, + "openInterviewsPhoneJob": false, + "organicApplyStartCount": 6979, + "overrideIndeedApplyText": true, + "packageTier": "NONE", + "preciseLocationModel": { + "obfuscateLocation": false, + "overrideJCMPreciseLocationModel": true + }, + "pubDate": 1774242000000, + "rankedBenefits": { + "GENERIC": [ + "YQ98H", + "RZAT2", + "FQJ2X", + "CFRGS", + "4C2ZW" + ], + "OCCUPATION_TFIDF": [ + "4C2ZW", + "CFRGS", + "FQJ2X", + "RZAT2", + "YQ98H" + ] + }, + "rankingScoresModel": { + "bid": 123996, + "bidPosition": 1, + "eApply": 0.020990705, + "eAttainability": 0.08190364, + "eQualified": 0 + }, + "recommendationReasonModel": { + "reason": null + }, + "redirectToThirdPartySite": false, + "remoteLocation": false, + "remoteWorkModel": { + "inlineText": true, + "text": "Remote", + "type": "REMOTE_ALWAYS" + }, + "resultBeforeExpansion": false, + "resumeMatch": false, + "salarySnippet": { + "currency": "USD", + "salaryTextFormatted": false, + "source": "EXTRACTION", + "text": "$65,000 - $115,000 a year" + }, + "saved": false, + "savedApplication": false, + "screenerQuestionsURL": "iqp://JOB/69c158406b0a663247e222ee?sourceUrl=iqp%3A%2F%2FINDEXEDJOB%2F10074441540%3Fcountry%3DUS&advertiserId=32113445&aggJobId=10074441540&country=US&language=en", + "searchUID": "1jtgbqmn921k6001", + "showCommutePromo": false, + "showEarlyApply": false, + "showJobType": false, + "showRelativeDate": true, + "showSponsoredLabel": false, + "showStrongerAppliedLabel": false, + "smartFillEnabled": false, + "smbD2iEnabled": false, + "snippet": "
    \n
  • We are looking for a passionate engineer skilled in building scalable and responsive web applications and services using Angular and ASP.
    • \n
", + "sourceId": 3368918, + "sponsored": true, + "taxoAttributes": [ + "Profit sharing" + ], + "taxoAttributesDisplayLimit": 3, + "taxoLogAttributes": [ + "4C2ZW" + ], + "taxonomyAttributes": [ + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types" + }, + { + "attributes": [], + "label": "shifts" + }, + { + "attributes": [ + { + "label": "Remote", + "suid": "DSQF7" + } + ], + "label": "remote" + }, + { + "attributes": [ + { + "label": "Dental insurance", + "suid": "FQJ2X" + }, + { + "label": "Disability insurance", + "suid": "CFRGS" + }, + { + "label": "Retirement plan", + "suid": "YQ98H" + }, + { + "label": "Vision insurance", + "suid": "RZAT2" + }, + { + "label": "Profit sharing", + "suid": "4C2ZW" + } + ], + "label": "benefits" + }, + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types-cc" + }, + { + "attributes": [], + "label": "schedules" + } + ], + "thirdPartyApplyUrl": "http://www.indeed.com//applystart?jk=7b993d015f4b257a&from=jobsearch&mvj=0&jobsearchTk=1jtgbqmn921k6001&spon=1&adid=464623129&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N1lyYEY1nzdWnmTq7hlQAqMT4EZoUMRYAQrNEOj5nb4ZE_F_ZCNcUyWwXqUnsb3IFHbnllXq0f5pYhflUexN3LD6mf9j8U6fPKOFmTQuul154CxQeo8aExEApf08b2iFfAx7Eci6fsR4i5wlRjvKA5EumK1U1Eauxzua2oUUaLKdAW9137H0CYgdcoAMF84PbbBCj5eW5R0p7Tfgi9NmbhJFRaEZv-u8JEg9U1zcdmaIh6RuPDU7bjQIuak5MO3DwaEJlHXEg4v7-XpR0uTAehVEkbNEFQdl0EwMnI32HxxwGJ_r7Xy7viI3kh6lEdrpZeF1ca18i9FlF_nce_H7BdzO2qmsM11dLrzIwGnipd9-NCmqh-jZJbJLooc2Yl3D8379SRHWpbfMz_a9aa1x_miN7s9YFssX_W8wrIqe3eHIaV-pRMzXMJYIxrcyPztopcC5udht92l9RFkJ8GjCK-zMHwiJRuCIqjML2cMsQblSxIpv5pb9M88dlUMlDd6dMpQBDYRrFe1PYyRu_QTWc_ggES6VZu7PJ7BcMPSFZ7cSuzuPT49lMRw7lYs3P1aakqoiFThrgWJAWATgalPtzZy754uEQG6ez5Gh-KW-nvDhDDfUS-fhvIMcDFftBf6gyU6_wEuH9GI7sOTR8b7jnHS285ryLd212LhBy0GJjliQdMxRmWK0qDqFr8z9q7Ufi9AHPafSIWTFw%3D%3D&xkcb=SoBj6_M3hwflOvyN8p0LbzkdCdPP&sjdu=Xbbwb0dbVlAPs0TIG7JumEX25uTOF1Oj2WnXk16AywHWA2PUcrZQMHMXHdXk2eTvbAixkyUF3j-OOmJlYeYtIjeDAvq7rEASgcC0zRMWutBvuyCgKBpkTMsm3zcuFYK1uIzH-8PzvPSrzMEzGtRTg5jv-iXDq5yVFQaIv8kgOWkCg9YikGZNJgeIivR2iCfdxWXtDkF3x7BjrIiI14W0QsCgmbR1Dg8qS-_CO9vND2NLAJoHK8df0UY8vlo7n1P7mzaDYyEyfb0INk3TiuEWMemQii8wUThkfLKJY9xDEST1O0JfmjdhQ3CuSF3QpabBfwaJIpgPfWh5SU8ES2tfW40LHQ6gsm9PLZCCJd2RWBsAg-ynqMVmNvHBgTO9S5FFBEcILkSKcC_MyVjNUK-8qfhKnQQPXgamiQj-imh4f5H-01jn0qSbiIaNBBbWs8WaiEDX1JaFE1ANmzavVGRxShPILHL_US7TCQ871Vr89ga7Y0v9ivrNvS2rdS5oF4sLUvcuVq9FhTVrjZpfmMNrEcehBwAJB_oy0hwGoit3I9QwJBE7yTcTRuabRpUOY8NGnuBwzl_YpEii8y3zKUv_S56bGdV6hXrIYnlrC_lQUZISI14iKUD-Dc7DUmh-vDl7MpfMvKQUktHpIeGdJMdxqweC1QdHFvhFbZkfRgjDhnNc0jNtP6Mr34zH5R5LQ8XbdLc6hlCebmo1Cv6kdjNRaLtdI6Mq1Y9Gq0gPMRVyTIPtovc7LsBlfXwcpxl21GixCJ_-bI6UaPK3TmPi0p_29vNanWx2E5_SGwELeYDW3egDxwPLtz4Smx_kFWcH6inYTsQbi68YonwAEahac1GjyjGvrjYKsoS680WKuzZS0Mutx0IkRyRloqlBvNHBK06L-Wx8ojMF4axEl43YmtkRZeyfVWrwEc9a9vcONhKAJbgOACubfzVac9cjNyYppXgW-XK5SWGo9lcjkVg5r63SM4pv1skycGMDMD2AbX8kTiFoukUeCElgDzeLhXKgJGiMwyYHrbKUQRUDVrgVJ81WasDodKiQH-yZ_-PiXCHPergRl5I31036jtOI1ylLDUgKg6ut2knm_9eRe_tC2EEhxXtT7ZaLatnvaE4b2G0o_Q-izpmOx92rL3gkpo33T97hPc7J1sZW7kwt9HXHyZBXX5xZ6pdvi2b3fmxnjLYK0Ro&vjfrom=vjs&astse=d6aae2e110970c59&assa=588", + "tier": { + "matchedPreferences": { + "longMatchedPreferences": [], + "stringMatchedPreferences": [] + }, + "type": "DEFAULT" + }, + "title": "Software Developer", + "translatedAttributes": [], + "translatedCmiJobTags": [], + "truncatedCompany": "TherapyNotes.com", + "urgentlyHiring": false, + "viewJobLink": "/viewjob?jk=7b993d015f4b257a&from=vjs&tk=1jtgbqmn921k6001&viewtype=embedded&advn=9533474731693465&adid=464623129&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N1lyYEY1nzdWnmTq7hlQAqMT4EZoUMRYAQrNEOj5nb4ZE_F_ZCNcUyWwXqUnsb3IFHbnllXq0f5pYhflUexN3LD6mf9j8U6fPKOFmTQuul154CxQeo8aExEApf08b2iFfAx7Eci6fsR4i5wlRjvKA5EumK1U1Eauxzua2oUUaLKdAW9137H0CYgdcoAMF84PbbBCj5eW5R0p7Tfgi9NmbhJFRaEZv-u8JEg9U1zcdmaIh6RuPDU7bjQIuak5MO3DwaEJlHXEg4v7-XpR0uTAehVEkbNEFQdl0EwMnI32HxxwGJ_r7Xy7viI3kh6lEdrpZeF1ca18i9FlF_nce_H7BdzO2qmsM11dLrzIwGnipd9-NCmqh-jZJbJLooc2Yl3D8379SRHWpbfMz_a9aa1x_miN7s9YFssX_W8wrIqe3eHIaV-pRMzXMJYIxrcyPztopcC5udht92l9RFkJ8GjCK-zMHwiJRuCIqjML2cMsQblSxIpv5pb9M88dlUMlDd6dMpQBDYRrFe1PYyRu_QTWc_ggES6VZu7PJ7BcMPSFZ7cSuzuPT49lMRw7lYs3P1aakqoiFThrgWJAWATgalPtzZy754uEQG6ez5Gh-KW-nvDhDDfUS-fhvIMcDFftBf6gyU6_wEuH9GI7sOTR8b7jnHS285ryLd212LhBy0GJjliQdMxRmWK0qDqFr8z9q7Ufi9AHPafSIWTFw%3D%3D&xkcb=SoBj6_M3hwflOvyN8p0LbzkdCdPP&continueUrl=%2Fjobs%3Fq%3Dsoftware%2Bengineer%26l%3DRemote", + "vjFeaturedEmployerCandidate": false + }, + { + "adBlob": "-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N2MhaGUtj2kz1cGhiRJCPZR4-FTtJq5aUy42k66aUDfG-jeElTJaJF_uhS-sSFhT8XjUynnZ9RdOuIClMgX3_QeGQ1eZ8ZugnnoExqs4lHbEtd3kidPweUJRF0MyFycMxMCn9PPBMAr2RWtUUo1uwl1XqX3eSw72oEa1W-RXDX2rtTZXHOprn3W7VUJ7KBfEuz9dn-7yLd2EqAKhhT-rHN9Lm1ES3UePr384UySNQNOC05nSAXRD7sWGEFSLN7BMYsDwZMQA39fRPGioSyhsoNKnGbkEiEgeUpRce_grNxxR7lU_oVHb-QemsQv_jZWwwZ9Q2j74LtVfpNdO_U1is58Hju2-BHih_ov1gUxcwRJGEIQcmQGtqsaarwGZalEQeCafsMzlN1QpmPGpH55FsGE3d_71A39-xRsCn0gFkouk_KEV0G8g_DoU5G9ZMEGdSbPDpLHYwszFA6QFKKjcTqzcy7hntV6BtA5gVnr_pQWNY3XNhrT59XGBm34NnzOfGGT8kCDoyd8juFITsfyMFHEMDkIrek0JfU-1Y-FPzmaLCpVNxPkRC3e8xXPqqzzcXshv5_ZRxpTNgBuUrd_EdhZtv-yKLdC0FtfGSSl8rm9Aqexw0CS7Wivy6DdNTGpHNsOybkoisYqEDeYaDN_zuSYmznXafHCwT-FgULG5_GSI2d9kU__07ht6bD0RzHr-Jk=", + "adId": "464623129", + "advn": "9533474731693465", + "appliedOrGreater": false, + "applyCount": -1, + "assistedApply": false, + "autoSourcerJob": false, + "bidPosition": 4, + "blobKey": "SoDX6_M3hwflOvyN8p0KbzkdCdPP", + "company": "TherapyNotes.com", + "companyBrandingAttributes": { + "brandingReasons": [ + "PAID_BRANDING" + ], + "brandingReasonsAsString": [ + "PAID_BRANDING" + ], + "headerImageUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_headerimage/1960x400/9d05368a0956c3266ea4291e95d34331", + "logoUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_squarelogo/256x256/eac29e66f882990ae917c959915eb291", + "paidBrandingDealIds": [ + "58664", + "95349" + ], + "showJobBranding": true, + "shownForBrandedJobPackage": false + }, + "companyIdEncrypted": "d10e9cc0e75a9595", + "companyOverviewLink": "/cmp/Therapynotes", + "companyOverviewLinkCampaignId": "serp-linkcompanyname-content", + "companyRating": 3.8, + "companyReviewCount": 30, + "companyReviewLink": "/cmp/Therapynotes/reviews", + "companyReviewLinkCampaignId": "cmplinktst2", + "country": "US", + "createDate": 1782827792000, + "d2iEnabled": false, + "displayTitle": "Senior Software Developer (Agentic Development)", + "dradisJob": false, + "employerAssistEnabled": false, + "employerResponseTime": 1, + "employerResponsive": true, + "encryptedFccompanyId": "c5ef9b9375a8bbf1", + "encryptedResultData": "VwIPTVJ1cTn5AN7Q-tSqGRXGNe2wB2UYx73qSczFnGU", + "enhancedAttributesModel": {}, + "enticers": [], + "expired": false, + "extractTrackingUrls": "", + "extractedSalary": { + "max": 135000, + "min": 110000, + "type": "YEARLY" + }, + "fccompanyId": -1, + "featuredCompanyAttributes": {}, + "featuredEmployer": false, + "featuredEmployerCandidate": false, + "feedId": 822778, + "formattedLocation": "Philadelphia, PA", + "formattedRelativeTime": "13 days ago", + "gatedVjp": false, + "hideMetaData": false, + "highVolumeHiringModel": { + "highVolumeHiring": false + }, + "hiringEventJob": false, + "homepageJobFeedSectionId": "0", + "indeedApplyEnabled": true, + "indeedApplyFinishAppUrlEnabled": false, + "indeedApplyResumeType": "required", + "indeedApplyable": true, + "isJobVisited": false, + "isMobileThirdPartyApplyable": false, + "isNoResumeJob": false, + "isSubsidiaryJob": false, + "isTopRatedEmployer": false, + "jobCardRequirementsModel": { + "additionalRequirementsCount": 0, + "jobOnlyRequirements": [], + "jobTagRequirements": [], + "requirementsHeaderShown": false, + "screenerQuestionRequirements": [] + }, + "jobFlairPackageEnabled": false, + "jobLocationCity": "Philadelphia", + "jobLocationState": "PA", + "jobSeekerMatchSummaryModel": { + "sortedEntityDisplayText": [], + "sortedMatchingEntityDisplayText": [], + "sortedMisMatchingEntityDisplayText": [ + ".NET Core", + "AI models", + "AI platforms (beyond public GPTs)", + "AI use case identification", + "AI-driven automation", + "API integrations", + "Agile software development", + "Automation software", + "Bachelor's degree", + "Collaboration with product development teams", + "Communication skills", + "Computer Science", + "Computer science", + "Continuous improvement for developers", + "Cross-functional communication", + "Design (software development lifecycle)", + "Developer tools", + "Employee onboarding", + "Engineering process optimization", + "Entity Framework", + "Full-stack development", + "Generative AI", + "HTML", + "IT quality control", + "JavaScript frameworks", + "Mentoring", + "PostgreSQL", + "Process design", + "Product management", + "Prompt engineering", + "Providing code feedback", + "Quality assurance experience within IT", + "Quality control", + "Release management", + "SASS/LESS", + "SQL databases", + "SaaS", + "Scalability", + "Scalable systems", + "Software Engineering", + "Software engineering", + "Software testing", + "System design", + "System design for system development", + "Team training", + "Technology security practices", + "Tooling", + "TypeScript" + ], + "taxoEntityMatchesNegative": [ + { + "displayText": "SASS/LESS", + "id": "", + "rawName": "SASS/LESS", + "source": "JOB", + "strictness": "", + "suid": "FE6SC" + }, + { + "displayText": "Software testing", + "id": "", + "rawName": "Software testing", + "source": "JOB", + "strictness": "", + "suid": "PZP9P" + }, + { + "displayText": "Computer science", + "id": "", + "rawName": "Computer science", + "source": "JOB", + "strictness": "", + "suid": "4E4WW" + }, + { + "displayText": "JavaScript frameworks", + "id": "", + "rawName": "JavaScript frameworks", + "source": "JOB", + "strictness": "", + "suid": "EUPT5" + }, + { + "displayText": "HTML", + "id": "", + "rawName": "HTML", + "source": "JOB", + "strictness": "", + "suid": "Y7U37" + }, + { + "displayText": "System design", + "id": "", + "rawName": "System design", + "source": "JOB", + "strictness": "", + "suid": "C5C2F" + }, + { + "displayText": "Mentoring", + "id": "", + "rawName": "Mentoring", + "source": "JOB", + "strictness": "", + "suid": "MGSEB" + }, + { + "displayText": "Cross-functional communication", + "id": "", + "rawName": "Cross-functional communication", + "source": "JOB", + "strictness": "", + "suid": "Y8657" + }, + { + "displayText": "Process design", + "id": "", + "rawName": "Process design", + "source": "JOB", + "strictness": "", + "suid": "C4MUU" + }, + { + "displayText": "AI use case identification", + "id": "", + "rawName": "AI use case identification", + "source": "JOB", + "strictness": "", + "suid": "4KV6M" + }, + { + "displayText": "Design (software development lifecycle)", + "id": "", + "rawName": "Design (software development lifecycle)", + "source": "JOB", + "strictness": "", + "suid": "XA6CW" + }, + { + "displayText": "Scalable systems", + "id": "", + "rawName": "Scalable systems", + "source": "JOB", + "strictness": "", + "suid": "DP9CS" + }, + { + "displayText": "Quality control", + "id": "", + "rawName": "Quality control", + "source": "JOB", + "strictness": "", + "suid": "K9VX6" + }, + { + "displayText": "AI models", + "id": "", + "rawName": "AI models", + "source": "JOB", + "strictness": "", + "suid": "22JPW" + }, + { + "displayText": "SQL databases", + "id": "", + "rawName": "SQL databases", + "source": "JOB", + "strictness": "", + "suid": "7G8UN" + }, + { + "displayText": "Continuous improvement for developers", + "id": "", + "rawName": "Continuous improvement for developers", + "source": "JOB", + "strictness": "", + "suid": "TZQ7R" + }, + { + "displayText": "Software Engineering", + "id": "", + "rawName": "Software Engineering", + "source": "JOB", + "strictness": "", + "suid": "7F6H9" + }, + { + "displayText": "Quality assurance experience within IT", + "id": "", + "rawName": "Quality assurance experience within IT", + "source": "JOB", + "strictness": "", + "suid": "EC9V5" + }, + { + "displayText": "Entity Framework", + "id": "", + "rawName": "Entity Framework", + "source": "JOB", + "strictness": "", + "suid": "X5HXD" + }, + { + "displayText": "Computer Science", + "id": "", + "rawName": "Computer Science", + "source": "JOB", + "strictness": "", + "suid": "6XNCP" + }, + { + "displayText": "Product management", + "id": "", + "rawName": "Product management", + "source": "JOB", + "strictness": "", + "suid": "JT9DE" + }, + { + "displayText": "Providing code feedback", + "id": "", + "rawName": "Providing code feedback", + "source": "JOB", + "strictness": "", + "suid": "Z4F5N" + }, + { + "displayText": "Engineering process optimization", + "id": "", + "rawName": "Engineering process optimization", + "source": "JOB", + "strictness": "", + "suid": "CCQ9J" + }, + { + "displayText": "Collaboration with product development teams", + "id": "", + "rawName": "Collaboration with product development teams", + "source": "JOB", + "strictness": "", + "suid": "GEYN2" + }, + { + "displayText": "PostgreSQL", + "id": "", + "rawName": "PostgreSQL", + "source": "JOB", + "strictness": "", + "suid": "JCKB4" + }, + { + "displayText": "AI-driven automation", + "id": "", + "rawName": "AI-driven automation", + "source": "JOB", + "strictness": "", + "suid": "TX6QF" + }, + { + "displayText": "IT quality control", + "id": "", + "rawName": "IT quality control", + "source": "JOB", + "strictness": "", + "suid": "AHSXR" + }, + { + "displayText": "Release management", + "id": "", + "rawName": "Release management", + "source": "JOB", + "strictness": "", + "suid": "J3CT2" + }, + { + "displayText": "Prompt engineering", + "id": "", + "rawName": "Prompt engineering", + "source": "JOB", + "strictness": "", + "suid": "EV2FP" + }, + { + "displayText": "Tooling", + "id": "", + "rawName": "Tooling", + "source": "JOB", + "strictness": "", + "suid": "93YM8" + }, + { + "displayText": "AI platforms (beyond public GPTs)", + "id": "", + "rawName": "AI platforms (beyond public GPTs)", + "source": "JOB", + "strictness": "", + "suid": "DP8RP" + }, + { + "displayText": ".NET Core", + "id": "", + "rawName": ".NET Core", + "source": "JOB", + "strictness": "", + "suid": "AGW49" + }, + { + "displayText": "TypeScript", + "id": "", + "rawName": "TypeScript", + "source": "JOB", + "strictness": "", + "suid": "WD7PP" + }, + { + "displayText": "Developer tools", + "id": "", + "rawName": "Developer tools", + "source": "JOB", + "strictness": "", + "suid": "WE46Y" + }, + { + "displayText": "SaaS", + "id": "", + "rawName": "SaaS", + "source": "JOB", + "strictness": "", + "suid": "UC5ZW" + }, + { + "displayText": "Team training", + "id": "", + "rawName": "Team training", + "source": "JOB", + "strictness": "", + "suid": "JSNTU" + }, + { + "displayText": "Full-stack development", + "id": "", + "rawName": "Full-stack development", + "source": "JOB", + "strictness": "", + "suid": "8PA9N" + }, + { + "displayText": "Scalability", + "id": "", + "rawName": "Scalability", + "source": "JOB", + "strictness": "", + "suid": "PYTWK" + }, + { + "displayText": "Automation software", + "id": "", + "rawName": "Automation software", + "source": "JOB", + "strictness": "", + "suid": "ANHQ8" + }, + { + "displayText": "Agile software development", + "id": "", + "rawName": "Agile software development", + "source": "JOB", + "strictness": "", + "suid": "QWGDE" + }, + { + "displayText": "System design for system development", + "id": "", + "rawName": "System design for system development", + "source": "JOB", + "strictness": "", + "suid": "FNWKU" + }, + { + "displayText": "Generative AI", + "id": "", + "rawName": "Generative AI", + "source": "JOB", + "strictness": "", + "suid": "Y6B2U" + }, + { + "displayText": "Bachelor's degree", + "id": "", + "rawName": "Bachelor's degree", + "source": "JOB", + "strictness": "", + "suid": "HFDVW" + }, + { + "displayText": "API integrations", + "id": "", + "rawName": "API integrations", + "source": "JOB", + "strictness": "", + "suid": "TXVT6" + }, + { + "displayText": "Software engineering", + "id": "", + "rawName": "Software engineering", + "source": "JOB", + "strictness": "", + "suid": "4R4AM" + }, + { + "displayText": "Communication skills", + "id": "", + "rawName": "Communication skills", + "source": "JOB", + "strictness": "", + "suid": "WSBNK" + }, + { + "displayText": "Technology security practices", + "id": "", + "rawName": "Technology security practices", + "source": "JOB", + "strictness": "", + "suid": "EKUZN" + }, + { + "displayText": "Employee onboarding", + "id": "", + "rawName": "Employee onboarding", + "source": "JOB", + "strictness": "", + "suid": "3ERWX" + } + ], + "taxoEntityMatchesPositive": [], + "trafficLight": "yellow" + }, + "jobTypes": [], + "jobkey": "1964995584dc3547", + "jsiEnabled": false, + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N2MhaGUtj2kz1cGhiRJCPZR4-FTtJq5aUy42k66aUDfG-jeElTJaJF_uhS-sSFhT8XjUynnZ9RdOuIClMgX3_QeGQ1eZ8ZugnnoExqs4lHbEtd3kidPweUJRF0MyFycMxMCn9PPBMAr2RWtUUo1uwl1XqX3eSw72oEa1W-RXDX2rtTZXHOprn3W7VUJ7KBfEuz9dn-7yLd2EqAKhhT-rHN9Lm1ES3UePr384UySNQNOC05nSAXRD7sWGEFSLN7BMYsDwZMQA39fRPGioSyhsoNKnGbkEiEgeUpRce_grNxxR7lU_oVHb-QemsQv_jZWwwZ9Q2j74LtVfpNdO_U1is58Hju2-BHih_ov1gUxcwRJGEIQcmQGtqsaarwGZalEQeCafsMzlN1QpmPGpH55FsGE3d_71A39-xRsCn0gFkouk_KEV0G8g_DoU5G9ZMEGdSbPDpLHYwszFA6QFKKjcTqzcy7hntV6BtA5gVnr_pQWNY3XNhrT59XGBm34NnzOfGGT8kCDoyd8juFITsfyMFHEMDkIrek0JfU-1Y-FPzmaLCpVNxPkRC3e8xXPqqzzcXshv5_ZRxpTNgBuUrd_EdhZtv-yKLdC0FtfGSSl8rm9Aqexw0CS7Wivy6DdNTGpHNsOybkoisYqEDeYaDN_zuSYmznXafHCwT-FgULG5_GSI2d9kU__07ht6bD0RzHr-Jk=&xkcb=SoDX6_M3hwflOvyN8p0KbzkdCdPP&p=1&camk=C3EPSzFlQw-JWkl1H7Zuog==&vjs=3", + "locationCount": 1, + "minimumCount": 1, + "mobtk": "1jtgbqmn921k6001", + "moreLocUrl": "/jobs?q=software+engineer&nl=&l=Remote&radius=50&jtid=7ca63af8b5ec590b&jcid=d10e9cc0e75a9595&grp=tcl", + "mouseDownHandlerOption": { + "adId": 464623129, + "advn": "9533474731693465", + "extractTrackingUrls": [], + "from": "vjs", + "jobKey": "1964995584dc3547", + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N2MhaGUtj2kz1cGhiRJCPZR4-FTtJq5aUy42k66aUDfG-jeElTJaJF_uhS-sSFhT8XjUynnZ9RdOuIClMgX3_QeGQ1eZ8ZugnnoExqs4lHbEtd3kidPweUJRF0MyFycMxMCn9PPBMAr2RWtUUo1uwl1XqX3eSw72oEa1W-RXDX2rtTZXHOprn3W7VUJ7KBfEuz9dn-7yLd2EqAKhhT-rHN9Lm1ES3UePr384UySNQNOC05nSAXRD7sWGEFSLN7BMYsDwZMQA39fRPGioSyhsoNKnGbkEiEgeUpRce_grNxxR7lU_oVHb-QemsQv_jZWwwZ9Q2j74LtVfpNdO_U1is58Hju2-BHih_ov1gUxcwRJGEIQcmQGtqsaarwGZalEQeCafsMzlN1QpmPGpH55FsGE3d_71A39-xRsCn0gFkouk_KEV0G8g_DoU5G9ZMEGdSbPDpLHYwszFA6QFKKjcTqzcy7hntV6BtA5gVnr_pQWNY3XNhrT59XGBm34NnzOfGGT8kCDoyd8juFITsfyMFHEMDkIrek0JfU-1Y-FPzmaLCpVNxPkRC3e8xXPqqzzcXshv5_ZRxpTNgBuUrd_EdhZtv-yKLdC0FtfGSSl8rm9Aqexw0CS7Wivy6DdNTGpHNsOybkoisYqEDeYaDN_zuSYmznXafHCwT-FgULG5_GSI2d9kU__07ht6bD0RzHr-Jk=&xkcb=SoDX6_M3hwflOvyN8p0KbzkdCdPP&p=1&camk=C3EPSzFlQw-JWkl1H7Zuog==&vjs=3", + "tk": "1jtgbqmn921k6001" + }, + "newJob": false, + "normTitle": "Senior Software Engineer", + "numHires": 0, + "openInterviewsInterviewsOnTheSpot": false, + "openInterviewsJob": false, + "openInterviewsOffersOnTheSpot": false, + "openInterviewsPhoneJob": false, + "organicApplyStartCount": 55, + "overrideIndeedApplyText": true, + "packageTier": "NONE", + "preciseLocationModel": { + "obfuscateLocation": false, + "overrideJCMPreciseLocationModel": true + }, + "pubDate": 1782795600000, + "rankedBenefits": { + "GENERIC": [ + "YQ98H", + "RZAT2", + "FQJ2X", + "CFRGS", + "4C2ZW" + ], + "OCCUPATION_TFIDF": [ + "4C2ZW", + "CFRGS", + "FQJ2X", + "RZAT2", + "YQ98H" + ] + }, + "rankingScoresModel": { + "bid": 90638, + "bidPosition": 4, + "eApply": 0.011701469, + "eAttainability": 0, + "eQualified": 0 + }, + "recommendationReasonModel": { + "reason": null + }, + "redirectToThirdPartySite": false, + "remoteLocation": false, + "remoteWorkModel": { + "inlineText": true, + "text": "Remote", + "type": "REMOTE_ALWAYS" + }, + "resultBeforeExpansion": false, + "resumeMatch": false, + "salarySnippet": { + "currency": "USD", + "salaryTextFormatted": false, + "source": "EXTRACTION", + "text": "$110,000 - $135,000 a year" + }, + "saved": false, + "savedApplication": false, + "screenerQuestionsURL": "iqp://INDEXEDJOB/10183863070?country=US", + "searchUID": "1jtgbqmn921k6001", + "showCommutePromo": false, + "showEarlyApply": false, + "showJobType": false, + "showRelativeDate": true, + "showSponsoredLabel": false, + "showStrongerAppliedLabel": false, + "smartFillEnabled": false, + "smbD2iEnabled": false, + "snippet": "
    \n
  • Ability to design reliable, maintainable, and scalable software architectures.
    • \n
  • You engineer tools, processes and workflows that meaningfully reduce friction in\u2026
    • \n
", + "sourceId": 3368918, + "sponsored": true, + "taxoAttributes": [ + "Profit sharing" + ], + "taxoAttributesDisplayLimit": 3, + "taxoLogAttributes": [ + "4C2ZW" + ], + "taxonomyAttributes": [ + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types" + }, + { + "attributes": [], + "label": "shifts" + }, + { + "attributes": [ + { + "label": "Remote", + "suid": "DSQF7" + } + ], + "label": "remote" + }, + { + "attributes": [ + { + "label": "Dental insurance", + "suid": "FQJ2X" + }, + { + "label": "Disability insurance", + "suid": "CFRGS" + }, + { + "label": "Vision insurance", + "suid": "RZAT2" + }, + { + "label": "Retirement plan", + "suid": "YQ98H" + }, + { + "label": "Profit sharing", + "suid": "4C2ZW" + } + ], + "label": "benefits" + }, + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types-cc" + }, + { + "attributes": [], + "label": "schedules" + } + ], + "thirdPartyApplyUrl": "http://www.indeed.com//applystart?jk=1964995584dc3547&from=jobsearch&mvj=0&jobsearchTk=1jtgbqmn921k6001&spon=1&adid=464623129&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N2MhaGUtj2kz1cGhiRJCPZR4-FTtJq5aUy42k66aUDfG-jeElTJaJF_uhS-sSFhT8XjUynnZ9RdOuIClMgX3_QeGQ1eZ8ZugnnoExqs4lHbEtd3kidPweUJRF0MyFycMxMCn9PPBMAr2RWtUUo1uwl1XqX3eSw72oEa1W-RXDX2rtTZXHOprn3W7VUJ7KBfEuz9dn-7yLd2EqAKhhT-rHN9Lm1ES3UePr384UySNQNOC05nSAXRD7sWGEFSLN7BMYsDwZMQA39fRPGioSyhsoNKnGbkEiEgeUpRce_grNxxR7lU_oVHb-QemsQv_jZWwwZ9Q2j74LtVfpNdO_U1is58Hju2-BHih_ov1gUxcwRJGEIQcmQGtqsaarwGZalEQeCafsMzlN1QpmPGpH55FsGE3d_71A39-xRsCn0gFkouk_KEV0G8g_DoU5G9ZMEGdSbPDpLHYwszFA6QFKKjcTqzcy7hntV6BtA5gVnr_pQWNY3XNhrT59XGBm34NnzOfGGT8kCDoyd8juFITsfyMFHEMDkIrek0JfU-1Y-FPzmaLCpVNxPkRC3e8xXPqqzzcXshv5_ZRxpTNgBuUrd_EdhZtv-yKLdC0FtfGSSl8rm9Aqexw0CS7Wivy6DdNTGpHNsOybkoisYqEDeYaDN_zuSYmznXafHCwT-FgULG5_GSI2d9kU__07ht6bD0RzHr-Jk%3D&xkcb=SoDX6_M3hwflOvyN8p0KbzkdCdPP&sjdu=Xbbwb0dbVlAPs0TIG7JumEX25uTOF1Oj2WnXk16AywHWA2PUcrZQMHMXHdXk2eTvbAixkyUF3j-OOmJlYeYtIjeDAvq7rEASgcC0zRMWutD3E9nQwSeMuCzdft7Qzt3FrQrTab10PIp0rWyA5dpjRE_z_G8OF7DNUc4gvnFFI-gpZwJRNYrOmZ5GZc_T8c2uHtcA7yEHKanT-ZgKiAexbe5qjDvO2kXqUnLLpW7YdwVJyc9eBIFg_vfY1-syU0GBwcsCuyrZU8nv0Xq8G93mw-7llCMyNJkF-WtwsM9Ue6aeLchHe0XGbSYl1Qcgf0sY-JQpXW7ju_Bk-MDoBCtis20YCc1bVmoQthnzEBWFp_9TYbQKJLr11DQp75s5Vdmsf-d5IQTi2PMp0iW7xymdYEQ3LDGdq2iavfaMS8A8OQ6rOQXa9jJFvJk9Ta5XeU_mpy5fSFg1oxkr0xp2QA2Utj3433zpiWFKDsr3Df6bWNvXjJoRzwIWKUpx2esmkKj7JybAVwMqmDoSkpfVABPfjuxY1hcCETtdEgmRbv6rBt0LOZX4H3srbX-kBOf6bWogp0PCe91mjDN2aw61THWKieSLyEzPUCYuoYr2onoda8SvUcksZX8zNPm_6b_s-yzzK9gO8ayTD6J-4jLltcKnpVPDYlxxCOFzCE00ROrg3Fdsv5xMIDVLwW7-zd6Rj8UrSVI77Ngt6Xuv4QptvZuF1trAUcMXIh9eq5jBrwb3_p3iAChwWpQm2LoaxnbeXmY4lWLNx-GuVCMQC9PPCdO2IZVUI2AHBR-WjvuESBVvRXlAXYLKObNGVDkrcUxV8nvoCcVndtP9TF5ZORGRAdWScvy63d0aKgnOkxJ3fSSHwTlRDOEY_tIYg31V8b1f21PG3DeLB17KWd6yFJNAPxgz99fRLl22swix5fq0lRJsIkN8K5eNtzDQ8b2M5QNT_-vr7ydAs3NiWqZbkns80Rhva-g4ppYeirl4F-oE27jkbsyR0mT19gIbooC-0xxw5rhbJVd3xcU-Qns5G3Nr8utN_PCj8MNuo8AomjLddrI-OA_DFCxJ9dMx4OQltXVjCk4TJI5pSLZxWVsDOsIirhwPN4O6LTFptkPn8PQP0M-MqfFUyGuzU73dybEBIAltr8aORGKf6dZx5anCwoZfBAU3zQ&vjfrom=vjs&astse=da7fff3285780cc4&assa=589", + "tier": { + "matchedPreferences": { + "longMatchedPreferences": [], + "stringMatchedPreferences": [] + }, + "type": "DEFAULT" + }, + "title": "Senior Software Developer (Agentic Development)", + "translatedAttributes": [], + "translatedCmiJobTags": [], + "truncatedCompany": "TherapyNotes.com", + "urgentlyHiring": false, + "viewJobLink": "/viewjob?jk=1964995584dc3547&from=vjs&tk=1jtgbqmn921k6001&viewtype=embedded&advn=9533474731693465&adid=464623129&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N2MhaGUtj2kz1cGhiRJCPZR4-FTtJq5aUy42k66aUDfG-jeElTJaJF_uhS-sSFhT8XjUynnZ9RdOuIClMgX3_QeGQ1eZ8ZugnnoExqs4lHbEtd3kidPweUJRF0MyFycMxMCn9PPBMAr2RWtUUo1uwl1XqX3eSw72oEa1W-RXDX2rtTZXHOprn3W7VUJ7KBfEuz9dn-7yLd2EqAKhhT-rHN9Lm1ES3UePr384UySNQNOC05nSAXRD7sWGEFSLN7BMYsDwZMQA39fRPGioSyhsoNKnGbkEiEgeUpRce_grNxxR7lU_oVHb-QemsQv_jZWwwZ9Q2j74LtVfpNdO_U1is58Hju2-BHih_ov1gUxcwRJGEIQcmQGtqsaarwGZalEQeCafsMzlN1QpmPGpH55FsGE3d_71A39-xRsCn0gFkouk_KEV0G8g_DoU5G9ZMEGdSbPDpLHYwszFA6QFKKjcTqzcy7hntV6BtA5gVnr_pQWNY3XNhrT59XGBm34NnzOfGGT8kCDoyd8juFITsfyMFHEMDkIrek0JfU-1Y-FPzmaLCpVNxPkRC3e8xXPqqzzcXshv5_ZRxpTNgBuUrd_EdhZtv-yKLdC0FtfGSSl8rm9Aqexw0CS7Wivy6DdNTGpHNsOybkoisYqEDeYaDN_zuSYmznXafHCwT-FgULG5_GSI2d9kU__07ht6bD0RzHr-Jk%3D&xkcb=SoDX6_M3hwflOvyN8p0KbzkdCdPP&continueUrl=%2Fjobs%3Fq%3Dsoftware%2Bengineer%26l%3DRemote", + "vjFeaturedEmployerCandidate": false + }, + { + "adBlob": "-6NYlbfkN0CnM4TERr6XuVBW3VTRACvDXDx9S3Pbn6a0SwhkmoLTiLCFUpITpg4QVB7SR39MoP0BLR841cgbHYPJopI9fiQa67MfGt-TS3gF6m2xSij4aAarTk90jQV5vKgeHuaG-dJ4P0bDI4Zj_HA51wzDk11uApXr5YRiDtyBwiQNLf5d9Lyctn13orSbPQeRCVH0zqauP5hiYL4ZkfdYeGe3I1Wbq855uSspMHp95AmMxgY4r8gBmfjoH2ZckgGERlYYLnGOotBKSI7i1CVIHtUDpAaF8MlAFpdDsF7JJ3NyS5QmqxHMQDQvH-MTyCDpC3bADAnfw6-khs-18fT3MQkZxaiHCnFCeCieu5HX8FLJW0Vbld4azqHzvV6iSucVFTJi1AKqwERerK9DWIfC0zose7qxVHyZkCqyCO7MtebmgcFgz8G-TMEKd3KEBdjQHW4g989DJ5vUEw0r_96qW6vxxph2br8GDqh0gRdNttZbE8iRAL2dJQGEiGkx-SEOJbw4SoVYUwpjBejBIJ-rdG4cyC5ppwrlxPXTlnAcqq9Ie16k5dyg-_YEGJgAXpTkGiS_jUefF4WFy9TLA5mJmWV1UGK991Seolocw55nIqu8uewnGZrKlzn_URF3fqCCkPEu048MFp3V5B32llPFdw6ecDZM4ixH_Y6Ex2ja_KQrD-VhNLIsQT0l0xiRsuD4NxCSVzIO3oh4AR-qxkF088JN5s0b7vYNrPpc20UUPEMmP9UTsO9TnPKrxhEPZX0qoIvEdXQwWA__4p652c_mTlKFK9F-VOq0Af-rMFWJFT2H9Cstjh4J7RyLp70DgUD5IJYLysP3eEEvknyDAOnB3IbnpMF4vePgU0vTSD-QHvvd-GxNquZ1d-hGjAfcdG70mwq736OitfXAGrZBSH7tm4yOCxSllF5t1z0TIE-LuM1UKKvYFt1H8B8TI-mhgtXwx6FhLAytzRKYOQlO5A==", + "adId": "461480873", + "advn": "7831672186689475", + "appliedOrGreater": false, + "applyCount": -1, + "assistedApply": false, + "autoSourcerJob": false, + "bidPosition": 3, + "blobKey": "SoBK6_M3hwflOvyN8p0JbzkdCdPP", + "company": "DataAnnotation", + "companyBrandingAttributes": { + "brandingReasons": [ + "PAID_BRANDING" + ], + "brandingReasonsAsString": [ + "PAID_BRANDING" + ], + "headerImageUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_headerimage/1960x400/e42fbb2b9f70343758932438638f973c", + "logoUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_squarelogo/256x256/cc8dc489da5211695773d9f9020812b0", + "paidBrandingDealIds": [ + "88129" + ], + "showJobBranding": true, + "shownForBrandedJobPackage": false + }, + "companyIdEncrypted": "342de9268ba1a972", + "companyOverviewLink": "/cmp/Dataannotation", + "companyOverviewLinkCampaignId": "serp-linkcompanyname-content", + "companyRating": 4.1, + "companyReviewCount": 1661, + "companyReviewLink": "/cmp/Dataannotation/reviews", + "companyReviewLinkCampaignId": "cmplinktst2", + "country": "US", + "createDate": 1778281962938, + "d2iEnabled": false, + "displayTitle": "Java Developer - AI Trainer", + "dradisJob": false, + "employerAssistEnabled": false, + "employerResponsive": false, + "encryptedFccompanyId": "e3735f59dc6a6354", + "encryptedResultData": "VwIPTVJ1cTn5AN7Q-tSqGRXGNe2wB2UYx73qSczFnGU", + "enhancedAttributesModel": {}, + "enticers": [], + "expired": false, + "extractTrackingUrls": "", + "extractedSalary": { + "max": 100, + "min": 0, + "type": "HOURLY" + }, + "fccompanyId": -1, + "featuredCompanyAttributes": {}, + "featuredEmployer": false, + "featuredEmployerCandidate": false, + "feedId": 915432, + "formattedLocation": "Remote", + "formattedRelativeTime": "30+ days ago", + "gatedVjp": false, + "hideMetaData": false, + "highVolumeHiringModel": { + "highVolumeHiring": false + }, + "hiringEventJob": false, + "homepageJobFeedSectionId": "0", + "indeedApplyEnabled": true, + "indeedApplyFinishAppUrlEnabled": true, + "indeedApplyResumeType": "required", + "indeedApplyable": true, + "isJobVisited": false, + "isMobileThirdPartyApplyable": false, + "isNoResumeJob": false, + "isSubsidiaryJob": false, + "isTopRatedEmployer": false, + "jobCardRequirementsModel": { + "additionalRequirementsCount": 0, + "jobOnlyRequirements": [], + "jobTagRequirements": [], + "requirementsHeaderShown": false, + "screenerQuestionRequirements": [] + }, + "jobFlairPackageEnabled": false, + "jobLocationCity": "Remote", + "jobSeekerMatchSummaryModel": { + "sortedEntityDisplayText": [], + "sortedMatchingEntityDisplayText": [], + "sortedMisMatchingEntityDisplayText": [ + "AI models", + "Bachelor's degree", + "C#", + "Data analytics", + "Data visualization", + "English", + "Go", + "Grammar Experience", + "JavaScript frameworks", + "Kotlin", + "Model training", + "Providing code feedback", + "Software coding", + "Software design", + "Software engineering", + "Swift", + "TypeScript", + "Writing skills" + ], + "taxoEntityMatchesNegative": [ + { + "displayText": "Software design", + "id": "", + "rawName": "Software design", + "source": "JOB", + "strictness": "", + "suid": "7DZKQ" + }, + { + "displayText": "Go", + "id": "", + "rawName": "Go", + "source": "JOB", + "strictness": "", + "suid": "6ETQT" + }, + { + "displayText": "JavaScript frameworks", + "id": "", + "rawName": "JavaScript frameworks", + "source": "JOB", + "strictness": "", + "suid": "EUPT5" + }, + { + "displayText": "English", + "id": "", + "rawName": "English", + "source": "JOB", + "strictness": "", + "suid": "D866K" + }, + { + "displayText": "TypeScript", + "id": "", + "rawName": "TypeScript", + "source": "JOB", + "strictness": "", + "suid": "WD7PP" + }, + { + "displayText": "Model training", + "id": "", + "rawName": "Model training", + "source": "JOB", + "strictness": "", + "suid": "QZ89D" + }, + { + "displayText": "AI models", + "id": "", + "rawName": "AI models", + "source": "JOB", + "strictness": "", + "suid": "22JPW" + }, + { + "displayText": "Kotlin", + "id": "", + "rawName": "Kotlin", + "source": "JOB", + "strictness": "", + "suid": "4KE5P" + }, + { + "displayText": "Data visualization", + "id": "", + "rawName": "Data visualization", + "source": "JOB", + "strictness": "", + "suid": "TKG4S" + }, + { + "displayText": "Bachelor's degree", + "id": "", + "rawName": "Bachelor's degree", + "source": "JOB", + "strictness": "", + "suid": "HFDVW" + }, + { + "displayText": "Software engineering", + "id": "", + "rawName": "Software engineering", + "source": "JOB", + "strictness": "", + "suid": "4R4AM" + }, + { + "displayText": "Data analytics", + "id": "", + "rawName": "Data analytics", + "source": "JOB", + "strictness": "", + "suid": "NGW4T" + }, + { + "displayText": "Software coding", + "id": "", + "rawName": "Software coding", + "source": "JOB", + "strictness": "", + "suid": "9MWQS" + }, + { + "displayText": "Writing skills", + "id": "", + "rawName": "Writing skills", + "source": "JOB", + "strictness": "", + "suid": "A7SFW" + }, + { + "displayText": "Grammar Experience", + "id": "", + "rawName": "Grammar Experience", + "source": "JOB", + "strictness": "", + "suid": "VTSX9" + }, + { + "displayText": "Providing code feedback", + "id": "", + "rawName": "Providing code feedback", + "source": "JOB", + "strictness": "", + "suid": "Z4F5N" + }, + { + "displayText": "C#", + "id": "", + "rawName": "C#", + "source": "JOB", + "strictness": "", + "suid": "AP3T4" + }, + { + "displayText": "Swift", + "id": "", + "rawName": "Swift", + "source": "JOB", + "strictness": "", + "suid": "U5AXZ" + } + ], + "taxoEntityMatchesPositive": [], + "trafficLight": "yellow" + }, + "jobTypes": [], + "jobkey": "40da1d788bab69b0", + "jsiEnabled": false, + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0CnM4TERr6XuVBW3VTRACvDXDx9S3Pbn6a0SwhkmoLTiLCFUpITpg4QVB7SR39MoP0BLR841cgbHYPJopI9fiQa67MfGt-TS3gF6m2xSij4aAarTk90jQV5vKgeHuaG-dJ4P0bDI4Zj_HA51wzDk11uApXr5YRiDtyBwiQNLf5d9Lyctn13orSbPQeRCVH0zqauP5hiYL4ZkfdYeGe3I1Wbq855uSspMHp95AmMxgY4r8gBmfjoH2ZckgGERlYYLnGOotBKSI7i1CVIHtUDpAaF8MlAFpdDsF7JJ3NyS5QmqxHMQDQvH-MTyCDpC3bADAnfw6-khs-18fT3MQkZxaiHCnFCeCieu5HX8FLJW0Vbld4azqHzvV6iSucVFTJi1AKqwERerK9DWIfC0zose7qxVHyZkCqyCO7MtebmgcFgz8G-TMEKd3KEBdjQHW4g989DJ5vUEw0r_96qW6vxxph2br8GDqh0gRdNttZbE8iRAL2dJQGEiGkx-SEOJbw4SoVYUwpjBejBIJ-rdG4cyC5ppwrlxPXTlnAcqq9Ie16k5dyg-_YEGJgAXpTkGiS_jUefF4WFy9TLA5mJmWV1UGK991Seolocw55nIqu8uewnGZrKlzn_URF3fqCCkPEu048MFp3V5B32llPFdw6ecDZM4ixH_Y6Ex2ja_KQrD-VhNLIsQT0l0xiRsuD4NxCSVzIO3oh4AR-qxkF088JN5s0b7vYNrPpc20UUPEMmP9UTsO9TnPKrxhEPZX0qoIvEdXQwWA__4p652c_mTlKFK9F-VOq0Af-rMFWJFT2H9Cstjh4J7RyLp70DgUD5IJYLysP3eEEvknyDAOnB3IbnpMF4vePgU0vTSD-QHvvd-GxNquZ1d-hGjAfcdG70mwq736OitfXAGrZBSH7tm4yOCxSllF5t1z0TIE-LuM1UKKvYFt1H8B8TI-mhgtXwx6FhLAytzRKYOQlO5A==&xkcb=SoBK6_M3hwflOvyN8p0JbzkdCdPP&p=2&camk=C3EPSzFlQw95L2GYPjH5nw==&vjs=3", + "locationCount": 1, + "minimumCount": 1, + "mobtk": "1jtgbqmn921k6001", + "moreLocUrl": "/jobs?q=software+engineer&nl=&l=Remote&radius=50&jtid=c1b76b0304ad72c6&jcid=342de9268ba1a972&grp=tcl", + "mouseDownHandlerOption": { + "adId": 461480873, + "advn": "7831672186689475", + "extractTrackingUrls": [], + "from": "vjs", + "jobKey": "40da1d788bab69b0", + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0CnM4TERr6XuVBW3VTRACvDXDx9S3Pbn6a0SwhkmoLTiLCFUpITpg4QVB7SR39MoP0BLR841cgbHYPJopI9fiQa67MfGt-TS3gF6m2xSij4aAarTk90jQV5vKgeHuaG-dJ4P0bDI4Zj_HA51wzDk11uApXr5YRiDtyBwiQNLf5d9Lyctn13orSbPQeRCVH0zqauP5hiYL4ZkfdYeGe3I1Wbq855uSspMHp95AmMxgY4r8gBmfjoH2ZckgGERlYYLnGOotBKSI7i1CVIHtUDpAaF8MlAFpdDsF7JJ3NyS5QmqxHMQDQvH-MTyCDpC3bADAnfw6-khs-18fT3MQkZxaiHCnFCeCieu5HX8FLJW0Vbld4azqHzvV6iSucVFTJi1AKqwERerK9DWIfC0zose7qxVHyZkCqyCO7MtebmgcFgz8G-TMEKd3KEBdjQHW4g989DJ5vUEw0r_96qW6vxxph2br8GDqh0gRdNttZbE8iRAL2dJQGEiGkx-SEOJbw4SoVYUwpjBejBIJ-rdG4cyC5ppwrlxPXTlnAcqq9Ie16k5dyg-_YEGJgAXpTkGiS_jUefF4WFy9TLA5mJmWV1UGK991Seolocw55nIqu8uewnGZrKlzn_URF3fqCCkPEu048MFp3V5B32llPFdw6ecDZM4ixH_Y6Ex2ja_KQrD-VhNLIsQT0l0xiRsuD4NxCSVzIO3oh4AR-qxkF088JN5s0b7vYNrPpc20UUPEMmP9UTsO9TnPKrxhEPZX0qoIvEdXQwWA__4p652c_mTlKFK9F-VOq0Af-rMFWJFT2H9Cstjh4J7RyLp70DgUD5IJYLysP3eEEvknyDAOnB3IbnpMF4vePgU0vTSD-QHvvd-GxNquZ1d-hGjAfcdG70mwq736OitfXAGrZBSH7tm4yOCxSllF5t1z0TIE-LuM1UKKvYFt1H8B8TI-mhgtXwx6FhLAytzRKYOQlO5A==&xkcb=SoBK6_M3hwflOvyN8p0JbzkdCdPP&p=2&camk=C3EPSzFlQw95L2GYPjH5nw==&vjs=3", + "tk": "1jtgbqmn921k6001" + }, + "newJob": false, + "normTitle": "Java Developer", + "numHires": 1096, + "openInterviewsInterviewsOnTheSpot": false, + "openInterviewsJob": false, + "openInterviewsOffersOnTheSpot": false, + "openInterviewsPhoneJob": false, + "organicApplyStartCount": 47202, + "overrideIndeedApplyText": true, + "packageTier": "NONE", + "preciseLocationModel": { + "obfuscateLocation": false, + "overrideJCMPreciseLocationModel": true + }, + "pubDate": 1778216400000, + "rankedBenefits": { + "GENERIC": [ + "WZ9TD" + ], + "OCCUPATION_TFIDF": [ + "WZ9TD" + ] + }, + "rankingScoresModel": { + "bid": 106902, + "bidPosition": 3, + "eApply": 0.014480283, + "eAttainability": 0, + "eQualified": 0 + }, + "recommendationReasonModel": { + "reason": null + }, + "redirectToThirdPartySite": false, + "remoteLocation": true, + "remoteWorkModel": { + "inlineText": true, + "text": "Remote", + "type": "REMOTE_ALWAYS" + }, + "resultBeforeExpansion": false, + "resumeMatch": false, + "salarySnippet": { + "currency": "USD", + "salaryTextFormatted": false, + "source": "EXTRACTION", + "text": "Up to $100 an hour" + }, + "saved": false, + "savedApplication": false, + "screenerQuestionsURL": "iqp://INDEXEDJOB/10128416810?sourceUrl=https%3A%2F%2Fapp.dataannotation.tech%2Fpartners%2Findeed%2Fscreener-questions%2FPROG_SA&country=US&language=en", + "searchUID": "1jtgbqmn921k6001", + "showCommutePromo": false, + "showEarlyApply": false, + "showJobType": false, + "showRelativeDate": true, + "showSponsoredLabel": false, + "showStrongerAppliedLabel": false, + "smartFillEnabled": false, + "smbD2iEnabled": false, + "snippet": "
    \n
  • Help train AI chatbots while gaining the flexibility of remote work and choosing your own schedule.
    • \n
  • We are looking for an existing Java Developer - AI Trainer\u2026
    • \n
", + "sourceId": 44932918, + "sponsored": true, + "taxoAttributes": [ + "Hourly pay" + ], + "taxoAttributesDisplayLimit": 3, + "taxoLogAttributes": [ + "8AD9K" + ], + "taxonomyAttributes": [ + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types" + }, + { + "attributes": [], + "label": "shifts" + }, + { + "attributes": [ + { + "label": "Remote", + "suid": "DSQF7" + } + ], + "label": "remote" + }, + { + "attributes": [ + { + "label": "Flexible schedule", + "suid": "WZ9TD" + } + ], + "label": "benefits" + }, + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types-cc" + }, + { + "attributes": [ + { + "label": "On demand", + "suid": "KPAXE" + } + ], + "label": "schedules" + } + ], + "thirdPartyApplyUrl": "http://www.indeed.com//applystart?jk=40da1d788bab69b0&from=jobsearch&mvj=0&jobsearchTk=1jtgbqmn921k6001&spon=1&adid=461480873&ad=-6NYlbfkN0CnM4TERr6XuVBW3VTRACvDXDx9S3Pbn6a0SwhkmoLTiLCFUpITpg4QVB7SR39MoP0BLR841cgbHYPJopI9fiQa67MfGt-TS3gF6m2xSij4aAarTk90jQV5vKgeHuaG-dJ4P0bDI4Zj_HA51wzDk11uApXr5YRiDtyBwiQNLf5d9Lyctn13orSbPQeRCVH0zqauP5hiYL4ZkfdYeGe3I1Wbq855uSspMHp95AmMxgY4r8gBmfjoH2ZckgGERlYYLnGOotBKSI7i1CVIHtUDpAaF8MlAFpdDsF7JJ3NyS5QmqxHMQDQvH-MTyCDpC3bADAnfw6-khs-18fT3MQkZxaiHCnFCeCieu5HX8FLJW0Vbld4azqHzvV6iSucVFTJi1AKqwERerK9DWIfC0zose7qxVHyZkCqyCO7MtebmgcFgz8G-TMEKd3KEBdjQHW4g989DJ5vUEw0r_96qW6vxxph2br8GDqh0gRdNttZbE8iRAL2dJQGEiGkx-SEOJbw4SoVYUwpjBejBIJ-rdG4cyC5ppwrlxPXTlnAcqq9Ie16k5dyg-_YEGJgAXpTkGiS_jUefF4WFy9TLA5mJmWV1UGK991Seolocw55nIqu8uewnGZrKlzn_URF3fqCCkPEu048MFp3V5B32llPFdw6ecDZM4ixH_Y6Ex2ja_KQrD-VhNLIsQT0l0xiRsuD4NxCSVzIO3oh4AR-qxkF088JN5s0b7vYNrPpc20UUPEMmP9UTsO9TnPKrxhEPZX0qoIvEdXQwWA__4p652c_mTlKFK9F-VOq0Af-rMFWJFT2H9Cstjh4J7RyLp70DgUD5IJYLysP3eEEvknyDAOnB3IbnpMF4vePgU0vTSD-QHvvd-GxNquZ1d-hGjAfcdG70mwq736OitfXAGrZBSH7tm4yOCxSllF5t1z0TIE-LuM1UKKvYFt1H8B8TI-mhgtXwx6FhLAytzRKYOQlO5A%3D%3D&xkcb=SoBK6_M3hwflOvyN8p0JbzkdCdPP&sjdu=Xbbwb0dbVlAPs0TIG7JumKleoDYenFeEkrm-BS2RFumAMJVRHnJnPXPQ1w4iVi_Bv12mcQcytWIaRGXVXpPFAJNHnyyAVakdybx_o10fpZKWxZ_XE6bUgpYqVY9LTx2dtS2-wQGVvKfiJXE4sfR0wlJijxkADwfaIzsQvfCpeWTbIoPN703ijAiPmVd-0iRuRqKLUPp4vg4qLycQPkInGwEJ8mAfOsifjoMlENy_ja0-fFKe71EIu7Z9bTd7syIkw1kTIG9N_3xV2F-5GZy3Y5T0JNFK-0qbOVkES4SHQ0SfH4NnnAa-f9nnHhIvvjmnptIrPbrObVWz49O6dy2-oLq1ZJUrsmKgOZzFKNCYYNCUbKfW-PqPLgNFoMJ1bqVan-wqq3U_828sw_TKiEbPLGBrX0Smwsv_nfNl645h88Ku6u2wUb1_sbeLBjNQYL-D2HF0NWgGwnwCdIgq-ovHBLz2azumxzGoE7sQwq5WgQU6Ejv27DnhAx3OksPVD9Q467ScvhtHyYiswIKxB_GqzPnKFzVfX6KlLlncqFzPtAdqUh6kbE--RtzZcE3CpSZy49vbhsBa3mK5jcKTo3kAW8OQHj80D0t3tmm_5Bz3HzmhEYtbc0mGitHUCyNLpy3-KmOMQ52bBemf2QzlsBwZM_s0P_i5BUDL-dg7xq4JOcYWtazSiL6AGCmfQ3-IZ66NzjFoAZecxXabAkK3Z3yS8YOicXGVy9B8QF5rlp7N10RoPP4JgHEDSdDTCZF8Vqkr81cMtwirsl6Ces3MDG_-lU8Qog1PeQs_BLn7VE7fnL8sFLH5stKSKIMHqtDXLFkv6ca-q2HsMP3KDd9l8Thw_GraXIfi1zP_-1dJJVesjBKRddONqBtekJ0dg4wz_0QGaCgpsJqAKTEWNu1jykb1WSIDsEyBoXQ6XBTbTgoVJg9W_pQ9pnl27ulNi4MRUUOuGkAhqGRX-suSwgxeatDCu6YGnLoNIz5_owj5ofazOCNHfLb8VF1mNhHIbmk41BvwrTMMyFcwLRwY5Zl6HRqlWTvnfRX_o71GiRqG-mo84veWnqey2tja4ynd5oGGEpJW9fAHiF_dYhtRE6_ZSH5JlkPuii5VUVatLa1wS1_UqHMTfm4yTZQ3k9qHVV5U6scDQUCqNBl8S_CEc8NZr2mHRzBoow79YUEPkNKI4KY03N1pG38fj16ZXK2GuKr1y3OpndPSDTO2ELdmaqpPn6H5V1YnWYL95pWEI_Zy8NrxXVDtYzPclQ0xpJHTWANEIb5vatND9npl3p-ADGXXrn291chTmlJv9lkd31Zosvlcvtn6BtL5lqZmS4Z29H6UBAhS-m4v2dwNroX8_PDn_lwGbwDY1d5uBB3EHNFQRlK-uGR-5UJ1eDDjSJ6uQi4hk6R-Z8H9h0Q5tm1cqEFUnMMiQpxZ6pdvi2b3fmxnjLYK0Ro&vjfrom=vjs&astse=da7fff3285780cc4&assa=589", + "tier": { + "matchedPreferences": { + "longMatchedPreferences": [], + "stringMatchedPreferences": [] + }, + "type": "DEFAULT" + }, + "title": "Java Developer - AI Trainer", + "translatedAttributes": [], + "translatedCmiJobTags": [], + "truncatedCompany": "DataAnnotation", + "urgentlyHiring": false, + "viewJobLink": "/viewjob?jk=40da1d788bab69b0&from=vjs&tk=1jtgbqmn921k6001&viewtype=embedded&advn=7831672186689475&adid=461480873&ad=-6NYlbfkN0CnM4TERr6XuVBW3VTRACvDXDx9S3Pbn6a0SwhkmoLTiLCFUpITpg4QVB7SR39MoP0BLR841cgbHYPJopI9fiQa67MfGt-TS3gF6m2xSij4aAarTk90jQV5vKgeHuaG-dJ4P0bDI4Zj_HA51wzDk11uApXr5YRiDtyBwiQNLf5d9Lyctn13orSbPQeRCVH0zqauP5hiYL4ZkfdYeGe3I1Wbq855uSspMHp95AmMxgY4r8gBmfjoH2ZckgGERlYYLnGOotBKSI7i1CVIHtUDpAaF8MlAFpdDsF7JJ3NyS5QmqxHMQDQvH-MTyCDpC3bADAnfw6-khs-18fT3MQkZxaiHCnFCeCieu5HX8FLJW0Vbld4azqHzvV6iSucVFTJi1AKqwERerK9DWIfC0zose7qxVHyZkCqyCO7MtebmgcFgz8G-TMEKd3KEBdjQHW4g989DJ5vUEw0r_96qW6vxxph2br8GDqh0gRdNttZbE8iRAL2dJQGEiGkx-SEOJbw4SoVYUwpjBejBIJ-rdG4cyC5ppwrlxPXTlnAcqq9Ie16k5dyg-_YEGJgAXpTkGiS_jUefF4WFy9TLA5mJmWV1UGK991Seolocw55nIqu8uewnGZrKlzn_URF3fqCCkPEu048MFp3V5B32llPFdw6ecDZM4ixH_Y6Ex2ja_KQrD-VhNLIsQT0l0xiRsuD4NxCSVzIO3oh4AR-qxkF088JN5s0b7vYNrPpc20UUPEMmP9UTsO9TnPKrxhEPZX0qoIvEdXQwWA__4p652c_mTlKFK9F-VOq0Af-rMFWJFT2H9Cstjh4J7RyLp70DgUD5IJYLysP3eEEvknyDAOnB3IbnpMF4vePgU0vTSD-QHvvd-GxNquZ1d-hGjAfcdG70mwq736OitfXAGrZBSH7tm4yOCxSllF5t1z0TIE-LuM1UKKvYFt1H8B8TI-mhgtXwx6FhLAytzRKYOQlO5A%3D%3D&xkcb=SoBK6_M3hwflOvyN8p0JbzkdCdPP&continueUrl=%2Fjobs%3Fq%3Dsoftware%2Bengineer%26l%3DRemote", + "vjFeaturedEmployerCandidate": false + } + ], + "pageNumber": 1, + "what": "software engineer", + "where": "Remote" + } + } +} \ No newline at end of file From 37963c64f1bd13c6e365e6e19dae38b6b0213bdf Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 09/56] test: indeed parsers --- .../platforms/indeed_jobs/test_parsers.py | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py new file mode 100644 index 000000000..7b9ca6298 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py @@ -0,0 +1,139 @@ +"""Offline parser tests: synthetic mapping plus a real captured blob.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from app.proprietary.platforms.indeed_jobs.parsers import ( + extract_jobcards_blob, + job_results, + parse_job, +) + +_FIXTURE_DIR = Path(__file__).parent / "fixtures" + + +# --- synthetic mapping (always runs) --------------------------------------- + + +def _raw_job() -> dict: + return { + "jobkey": "abc123", + "displayTitle": "Senior Data Analyst", + "title": "Senior Data Analyst (fallback)", + "company": "Acme Corp", + "truncatedCompany": "Acme", + "companyOverviewLink": "/cmp/Acme-Corp", + "companyRating": 4.1, + "companyReviewCount": 320, + "formattedLocation": "New York, NY", + "jobLocationCity": "New York", + "jobLocationState": "NY", + "jobLocationPostal": "10001", + "country": "US", + "remoteLocation": False, + "remoteWorkModel": {"type": "REMOTE_ALWAYS", "text": "Remote"}, + "jobTypes": [], + "salarySnippet": {"currency": "USD", "text": "$90,000 - $120,000 a year"}, + "extractedSalary": {"min": 90000, "max": 120000, "type": "YEARLY"}, + "snippet": "
  • 5+ years SQL & Python
", + "sponsored": True, + "newJob": False, + "urgentlyHiring": True, + "expired": False, + "indeedApplyEnabled": True, + "formattedRelativeTime": "3 days ago", + "pubDate": 1_774_242_000_000, + "createDate": 1_774_276_267_415, + "thirdPartyApplyUrl": "https://ats.example.com/apply/abc123", + "taxonomyAttributes": [ + {"label": "job-types", "attributes": [{"label": "Full-time", "suid": "x"}]}, + {"label": "remote", "attributes": [{"label": "Remote", "suid": "y"}]}, + { + "label": "benefits", + "attributes": [ + {"label": "Health insurance", "suid": "a"}, + {"label": "401(k)", "suid": "b"}, + ], + }, + ], + } + + +def test_parse_job_maps_core_fields(): + item = parse_job(_raw_job()) + assert item["jobKey"] == "abc123" + assert item["title"] == "Senior Data Analyst" # displayTitle wins over title + assert item["jobUrl"] == "https://www.indeed.com/viewjob?jk=abc123" + assert item["applyUrl"] == "https://ats.example.com/apply/abc123" + assert item["company"] == "Acme Corp" + assert item["companyUrl"] == "https://www.indeed.com/cmp/Acme-Corp" + assert item["companyReviewCount"] == 320 + assert item["city"] == "New York" + assert item["isRemote"] is True + assert item["remoteType"] == "REMOTE_ALWAYS" + assert item["jobTypes"] == ["Full-time"] + assert item["benefits"] == ["Health insurance", "401(k)"] + assert item["sponsored"] is True + assert item["urgentlyHiring"] is True + + +def test_parse_job_salary_and_snippet(): + item = parse_job(_raw_job()) + sal = item["salary"] + assert sal["salaryText"] == "$90,000 - $120,000 a year" + assert sal["salaryMin"] == 90000 + assert sal["salaryMax"] == 120000 + assert sal["currency"] == "USD" + assert sal["period"] == "year" + assert sal["isEstimated"] is False + # snippet HTML is stripped + entities decoded into plain text. + assert item["descriptionText"] == "5+ years SQL & Python" + assert item["descriptionHtml"] is None + + +def test_parse_job_dates_from_epoch_ms(): + item = parse_job(_raw_job()) + assert item["datePublished"] == "2026-03-23T05:00:00.000Z" + assert item["age"] == "3 days ago" + + +def test_parse_job_respects_base_url(): + item = parse_job(_raw_job(), base_url="https://uk.indeed.com") + assert item["jobUrl"] == "https://uk.indeed.com/viewjob?jk=abc123" + assert item["companyUrl"] == "https://uk.indeed.com/cmp/Acme-Corp" + + +def test_extract_blob_anchors_on_assignment_not_first_occurrence(): + # Decoy mention precedes the real assignment; the extractor must skip it. + html = ( + '' + '' + ) + blob = extract_jobcards_blob(html) + results = job_results(blob) + assert [r["jobkey"] for r in results] == ["k1", "k2"] + + +def test_extract_blob_missing_returns_none(): + assert extract_jobcards_blob("just a moment...") is None + assert job_results(None) == [] + + +# --- fixture-pinned (real captured blob) ----------------------------------- + + +def test_fixture_blob_parses_into_items(): + fixture = _FIXTURE_DIR / "sample_jobcards.json" + blob = json.loads(fixture.read_text()) + results = job_results(blob) + assert len(results) == 3 + for raw in results: + item = parse_job(raw) + assert isinstance(item["jobKey"], str) and item["jobKey"] + assert item["title"] + assert item["jobUrl"].startswith("https://www.indeed.com/viewjob?jk=") + assert "salaryText" in item["salary"] From 91e868fc26143b1307dec9cdb505d12ccb52abea Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 10/56] test: indeed url resolver --- .../indeed_jobs/test_url_resolver.py | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/test_url_resolver.py diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_url_resolver.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_url_resolver.py new file mode 100644 index 000000000..b995b3d0b --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_url_resolver.py @@ -0,0 +1,87 @@ +"""Offline tests for Indeed URL classification and search-URL building.""" + +from __future__ import annotations + +from urllib.parse import parse_qs, urlparse + +from app.proprietary.platforms.indeed_jobs.url_resolver import ( + build_search_url, + country_domain, + resolve_url, +) + + +def test_resolve_search_url(): + r = resolve_url( + "https://www.indeed.com/jobs?q=software+engineer&l=Remote&sort=date" + ) + assert r is not None + assert r.kind == "search" + assert r.value == "software engineer" + assert r.location == "Remote" + assert r.domain == "www.indeed.com" + assert r.params.get("sort") == "date" + + +def test_resolve_company_url(): + r = resolve_url("https://www.indeed.com/cmp/Google/jobs") + assert r is not None + assert r.kind == "company" + assert r.value == "Google" + + +def test_resolve_viewjob_url(): + r = resolve_url("https://uk.indeed.com/viewjob?jk=abc123&from=serp") + assert r is not None + assert r.kind == "job" + assert r.value == "abc123" + assert r.domain == "uk.indeed.com" + + +def test_resolve_country_subdomain_host(): + r = resolve_url("https://de.indeed.com/jobs?q=entwickler") + assert r is not None + assert r.kind == "search" + assert r.domain == "de.indeed.com" + + +def test_resolve_rejects_non_indeed(): + assert resolve_url("https://www.linkedin.com/jobs?q=dev") is None + assert resolve_url("https://notindeed.com.evil.com/jobs") is None + + +def test_country_domain_map(): + assert country_domain("us") == "www.indeed.com" + assert country_domain("gb") == "uk.indeed.com" + assert country_domain("de") == "de.indeed.com" + assert country_domain("") == "www.indeed.com" + + +def test_build_search_url_basic(): + url = build_search_url( + "data analyst", + country="us", + location="New York, NY", + sort="date", + start=20, + ) + parsed = urlparse(url) + qs = parse_qs(parsed.query) + assert parsed.netloc == "www.indeed.com" + assert parsed.path == "/jobs" + assert qs["q"] == ["data analyst"] + assert qs["l"] == ["New York, NY"] + assert qs["sort"] == ["date"] + assert qs["start"] == ["20"] + + +def test_build_search_url_remote_keyword_fallback_and_jobtype(): + url = build_search_url( + "developer", country="gb", remote="remote", job_type="fulltime", from_days=7 + ) + parsed = urlparse(url) + qs = parse_qs(parsed.query) + assert parsed.netloc == "uk.indeed.com" + assert qs["q"] == ["developer remote"] + assert qs["jt"] == ["fulltime"] + assert qs["fromage"] == ["7"] From a60aba112c52ae744d9d20204548bea65188d198 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 11/56] test: indeed fetch rotation resilience --- .../indeed_jobs/test_fetch_resilience.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py new file mode 100644 index 000000000..4eae65f84 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py @@ -0,0 +1,95 @@ +"""Offline tests for the rotate-on-block fetch loop (no network, fake session).""" + +from __future__ import annotations + +import pytest + +from app.proprietary.platforms.indeed_jobs.fetch import ( + IndeedAccessBlockedError, + IndeedSession, +) + +_OK_HTML = "jobs listing" +_BLOCK_HTML = "secure.indeed.com security check" + + +class _FakePage: + def __init__(self, html: str, url: str) -> None: + self.html_content = html + self.url = url + + +class _Controller: + """Shared state across sessions the factory hands out (survives rotation).""" + + def __init__(self, target_outcomes: list[str]) -> None: + self.target_outcomes = target_outcomes + self.sessions_started = 0 + self.home_fetches: dict[str, int] = {} + self.target_index = 0 + + def factory(self) -> _FakeSession: + return _FakeSession(self) + + +class _FakeSession: + def __init__(self, ctrl: _Controller) -> None: + self._ctrl = ctrl + + async def start(self) -> None: + self._ctrl.sessions_started += 1 + + async def close(self) -> None: + pass + + async def fetch(self, url: str, **_: object) -> _FakePage: + if url.endswith("/") and "/jobs" not in url: # warm-up hit + self._ctrl.home_fetches[url] = self._ctrl.home_fetches.get(url, 0) + 1 + return _FakePage("home", url) + outcome = self._ctrl.target_outcomes[self._ctrl.target_index] + self._ctrl.target_index += 1 + if outcome == "OK": + return _FakePage(_OK_HTML, url) + if outcome == "ERROR": + raise RuntimeError("boom") + return _FakePage(_BLOCK_HTML, "https://secure.indeed.com/auth") + + +_URL = "https://www.indeed.com/jobs?q=dev" + + +@pytest.mark.asyncio +async def test_rotates_past_a_block_then_succeeds(): + ctrl = _Controller(["BLOCK", "OK"]) + session = IndeedSession(ctrl.factory) + html = await session.fetch_html(_URL) + assert html == _OK_HTML + assert session.rotations == 1 + assert ctrl.sessions_started == 2 # initial + one rotation + + +@pytest.mark.asyncio +async def test_recovers_after_a_fetch_error(): + ctrl = _Controller(["ERROR", "OK"]) + session = IndeedSession(ctrl.factory) + assert await session.fetch_html(_URL) == _OK_HTML + assert session.rotations == 1 + + +@pytest.mark.asyncio +async def test_raises_after_exhausting_rotations(): + ctrl = _Controller(["BLOCK"] * 10) + session = IndeedSession(ctrl.factory) + with pytest.raises(IndeedAccessBlockedError): + await session.fetch_html(_URL) + assert session.rotations == 3 + + +@pytest.mark.asyncio +async def test_warms_domain_once_without_rotation(): + ctrl = _Controller(["OK", "OK"]) + session = IndeedSession(ctrl.factory) + await session.fetch_html(_URL) + await session.fetch_html(_URL + "&start=10") + assert ctrl.home_fetches["https://www.indeed.com/"] == 1 + await session.close() From 063faf4bd6f92e7851ad76a725d53e68eca3d586 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 12/56] test: indeed scraper orchestration --- .../platforms/indeed_jobs/test_scraper.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py new file mode 100644 index 000000000..c56d64f75 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py @@ -0,0 +1,105 @@ +"""Offline orchestration tests: pagination, dedupe, and caps via a fake session.""" + +from __future__ import annotations + +import json +from urllib.parse import parse_qs, urlparse + +import pytest + +from app.proprietary.platforms.indeed_jobs.schemas import IndeedScrapeInput +from app.proprietary.platforms.indeed_jobs.scraper import iter_indeed, scrape_indeed + + +def _page_html(job_keys: list[str]) -> str: + """Wrap job keys in the ``mosaic-provider-jobcards`` assignment shape.""" + results = [ + {"jobkey": k, "displayTitle": f"Job {k}", "company": "Acme"} for k in job_keys + ] + model = {"metaData": {"mosaicProviderJobCardsModel": {"results": results}}} + return ( + f'window.mosaic.providerData["mosaic-provider-jobcards"]={json.dumps(model)};' + ) + + +class _FakeSession: + """Returns per-``start`` pages and records the URLs requested.""" + + def __init__(self, pages: dict[int, list[str]]) -> None: + self._pages = pages + self.fetched: list[str] = [] + + async def fetch_html(self, url: str) -> str: + self.fetched.append(url) + start = int(parse_qs(urlparse(url).query).get("start", ["0"])[0]) + return _page_html(self._pages.get(start, [])) + + +async def _collect(input_model, session) -> list[dict]: + return [item async for item in iter_indeed(input_model, session)] + + +@pytest.mark.asyncio +async def test_paginates_and_dedupes_across_pages(): + session = _FakeSession({0: ["k1", "k2", "k3"], 10: ["k3", "k4"], 20: []}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2", "k3", "k4"] + assert all(i["scrapedAt"] for i in items) # stamped by the orchestrator + + +@pytest.mark.asyncio +async def test_stops_when_a_page_is_all_duplicates(): + session = _FakeSession({0: ["k1", "k2"], 10: ["k1", "k2"], 20: ["k9"]}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + assert 20 not in { + int(parse_qs(urlparse(u).query).get("start", ["0"])[0]) for u in session.fetched + } + + +@pytest.mark.asyncio +async def test_respects_max_items_per_query(): + session = _FakeSession({0: ["k1", "k2", "k3", "k4"]}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=2), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + + +@pytest.mark.asyncio +async def test_global_dedupe_across_queries(): + # Both queries hit page 0 (same fake pages) and return the same keys. + session = _FakeSession({0: ["k1", "k2"]}) + items = await _collect( + IndeedScrapeInput(queries=["dev", "engineer"], maxItemsPerQuery=100), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + + +@pytest.mark.asyncio +async def test_start_urls_skip_viewjob_and_scrape_search(): + session = _FakeSession({0: ["k1"]}) + input_model = IndeedScrapeInput( + startUrls=[ + {"url": "https://www.indeed.com/jobs?q=dev"}, + {"url": "https://www.indeed.com/viewjob?jk=abc"}, + ], + maxItemsPerQuery=100, + ) + items = await _collect(input_model, session) + assert [i["jobKey"] for i in items] == ["k1"] + + +@pytest.mark.asyncio +async def test_scrape_indeed_limit_with_injected_session(): + session = _FakeSession({0: ["k1", "k2", "k3"]}) + items = await scrape_indeed( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), + limit=2, + session=session, + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] From 05bfc6a10ef521204b83fc3a68346f7402ef5b88 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 13/56] feat: add INDEED_JOB billing unit --- surfsense_backend/app/capabilities/core/types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index 67b9175fc..6b3be70d0 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -30,6 +30,7 @@ class BillingUnit(StrEnum): TIKTOK_VIDEO = "tiktok_video" TIKTOK_USER = "tiktok_user" TIKTOK_COMMENT = "tiktok_comment" + INDEED_JOB = "indeed_job" class BillableInput(Protocol): From e25d09c25e33fab2f583ffce4e01fb19153bec7e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 14/56] feat: add indeed per-job scrape rate config --- surfsense_backend/app/config/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 92e448d9a..c006c8cc6 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -736,6 +736,11 @@ class Config: # Comments are the cheapest per-item TikTok data, matching the per-comment # market (and YouTube's comment meter). TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) + # Warmed-browser listings put Indeed on par with the other browser-driven + # scrapers (Reddit, Instagram) rather than the cheaper API-backed meters. + INDEED_SCRAPE_MICROS_PER_JOB = int( + os.getenv("INDEED_SCRAPE_MICROS_PER_JOB", "3500") + ) # Retry an empty listing draw on a fresh rotating IP. Set to 1 for a static # proxy, where every retry re-hits the same exit. TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3")) From d89ca141a4b914ff342ab68c58cd2cf6bd408df4 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 15/56] feat: meter indeed_job billing unit --- surfsense_backend/app/capabilities/core/billing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 67abf4164..bc031db3f 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -40,6 +40,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT", + BillingUnit.INDEED_JOB: "INDEED_SCRAPE_MICROS_PER_JOB", } @@ -61,6 +62,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.TIKTOK_VIDEO: "video", BillingUnit.TIKTOK_USER: "profile", BillingUnit.TIKTOK_COMMENT: "comment", + BillingUnit.INDEED_JOB: "job", } From e887c1cae718fa0b7ca4ca1a57c04a2d82bdf4ae Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 16/56] feat: indeed.scrape io schemas --- .../app/capabilities/indeed/scrape/schemas.py | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 surfsense_backend/app/capabilities/indeed/scrape/schemas.py diff --git a/surfsense_backend/app/capabilities/indeed/scrape/schemas.py b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py new file mode 100644 index 000000000..a02f6f13b --- /dev/null +++ b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py @@ -0,0 +1,112 @@ +"""``indeed.scrape`` I/O contracts. + +A lean, agent-friendly surface over ``IndeedScrapeInput`` +(``app/proprietary/platforms/indeed_jobs``). The executor maps this to the full +scraper input; the scraper's ``IndeedItem`` is reused verbatim as the output +element. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.indeed_jobs import IndeedItem +from app.proprietary.platforms.indeed_jobs.schemas import ( + IndeedJobType, + IndeedLevel, + IndeedRemote, + IndeedSort, +) + +MAX_INDEED_SOURCES = 20 +"""Per-call cap on urls + search_queries: bounds a synchronous request's fan-out.""" + +MAX_INDEED_ITEMS = 100 +"""Hard ceiling on jobs returned per call, regardless of the per-query caps.""" + + +class ScrapeInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_INDEED_SOURCES, + description=( + "Indeed URLs to scrape: a search page (/jobs?q=&l=) or a company " + "jobs page (/cmp//jobs). Provide these OR search_queries " + "(at least one source is required)." + ), + ) + search_queries: list[str] = Field( + default_factory=list, + max_length=MAX_INDEED_SOURCES, + description=( + "Job search terms; each returns up to max_items_per_query results, " + "shaped by country/location/job_type/etc." + ), + ) + country: str = Field( + default="us", + description="Country code selecting the Indeed domain, e.g. 'us', 'gb', 'de'.", + ) + location: str | None = Field( + default=None, + description="Where to search, e.g. 'Remote', 'New York, NY'.", + ) + radius: int | None = Field( + default=None, + description="Search radius in miles/km around location.", + ) + job_type: IndeedJobType | None = Field( + default=None, + description="Employment type filter: fulltime, parttime, contract, etc.", + ) + level: IndeedLevel | None = Field( + default=None, + description="Experience level filter: entry_level, mid_level, senior_level.", + ) + remote: IndeedRemote | None = Field( + default=None, + description="Work model filter: remote or hybrid.", + ) + from_days: int | None = Field( + default=None, + description="Only return jobs posted within the last N days.", + ) + sort: IndeedSort = Field( + default="relevance", + description="Result ordering: relevance or date.", + ) + max_items: int = Field( + default=25, + ge=1, + le=MAX_INDEED_ITEMS, + description="Max total jobs to return across all sources.", + ) + max_items_per_query: int = Field( + default=25, + ge=0, + description="Max jobs to pull per search/company target.", + ) + + @model_validator(mode="after") + def _require_a_source(self) -> ScrapeInput: + if not self.urls and not self.search_queries: + raise ValueError("Provide at least one of 'urls' or 'search_queries'.") + return self + + @property + def estimated_units(self) -> int: + """Worst-case billable jobs for the pre-flight gate: ``max_items`` is a + hard cross-source ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class ScrapeOutput(BaseModel): + items: list[IndeedItem] = Field( + default_factory=list, + description="One item per job posting, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned job = one billable unit.""" + return len(self.items) From b457599fd4577a65dc1fba61288574e1ba3c6204 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 17/56] feat: indeed.scrape executor --- .../capabilities/indeed/scrape/executor.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 surfsense_backend/app/capabilities/indeed/scrape/executor.py diff --git a/surfsense_backend/app/capabilities/indeed/scrape/executor.py b/surfsense_backend/app/capabilities/indeed/scrape/executor.py new file mode 100644 index 000000000..60df11972 --- /dev/null +++ b/surfsense_backend/app/capabilities/indeed/scrape/executor.py @@ -0,0 +1,55 @@ +"""``indeed.scrape`` executor: verb input → scraper → Indeed job items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.indeed.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.indeed_jobs import ( + IndeedAccessBlockedError, + IndeedScrapeInput, + scrape_indeed, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_indeed + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + actor_input = IndeedScrapeInput( + startUrls=[{"url": url} for url in payload.urls], + queries=payload.search_queries, + country=payload.country, + location=payload.location, + radius=payload.radius, + jobType=payload.job_type, + level=payload.level, + remote=payload.remote, + fromDays=payload.from_days, + sort=payload.sort, + maxItems=payload.max_items, + maxItemsPerQuery=payload.max_items_per_query, + ) + emit_progress( + "starting", "Resolving Indeed targets", total=payload.max_items, unit="job" + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except IndeedAccessBlockedError as exc: + # Anonymous-only scraper; a hard block can't be retried with creds. + raise ForbiddenError( + f"Indeed refused anonymous access: {exc}", + code="INDEED_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", f"Scraped {len(items)} job(s)", current=len(items), unit="job" + ) + return ScrapeOutput(items=items) + + return execute From 3f91a006cd98967a03df3b72ab706957d3c83a2b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 18/56] feat: register indeed.scrape capability --- .../capabilities/indeed/scrape/definition.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 surfsense_backend/app/capabilities/indeed/scrape/definition.py diff --git a/surfsense_backend/app/capabilities/indeed/scrape/definition.py b/surfsense_backend/app/capabilities/indeed/scrape/definition.py new file mode 100644 index 000000000..b831cd4a7 --- /dev/null +++ b/surfsense_backend/app/capabilities/indeed/scrape/definition.py @@ -0,0 +1,23 @@ +"""``indeed.scrape`` capability registration (billed per job; see config +``INDEED_SCRAPE_MICROS_PER_JOB``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.indeed.scrape.executor import build_scrape_executor +from app.capabilities.indeed.scrape.schemas import ScrapeInput, ScrapeOutput + +INDEED_SCRAPE = Capability( + name="indeed.scrape", + description=( + "Scrape public Indeed job postings, including title, company, location, " + "salary, and description. Use urls or search_queries." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.INDEED_JOB, + docs_url="/docs/connectors/native/indeed", +) + +register_capability(INDEED_SCRAPE) From 17159172ee28e5cfefaae84e2f9f7a2f3e340cff Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 19/56] feat: indeed.scrape verb package --- surfsense_backend/app/capabilities/indeed/scrape/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 surfsense_backend/app/capabilities/indeed/scrape/__init__.py diff --git a/surfsense_backend/app/capabilities/indeed/scrape/__init__.py b/surfsense_backend/app/capabilities/indeed/scrape/__init__.py new file mode 100644 index 000000000..6b7fa588e --- /dev/null +++ b/surfsense_backend/app/capabilities/indeed/scrape/__init__.py @@ -0,0 +1,3 @@ +"""``indeed.scrape`` verb: Indeed search / company URLs → job postings.""" + +from __future__ import annotations From 8c410d8682097801f524293bbc25b34feae6e014 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 20/56] feat: indeed capability namespace --- surfsense_backend/app/capabilities/indeed/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 surfsense_backend/app/capabilities/indeed/__init__.py diff --git a/surfsense_backend/app/capabilities/indeed/__init__.py b/surfsense_backend/app/capabilities/indeed/__init__.py new file mode 100644 index 000000000..5e053244f --- /dev/null +++ b/surfsense_backend/app/capabilities/indeed/__init__.py @@ -0,0 +1,5 @@ +"""``indeed.*`` namespace: platform-native Indeed data verbs.""" + +from __future__ import annotations + +from app.capabilities.indeed.scrape import definition as _scrape # noqa: F401 From ad49e9004e33a11b8b333dafb3d187e01c189e60 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 21/56] feat: register indeed namespace on routes --- surfsense_backend/app/routes/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 85450d4b7..b5186a72f 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -3,6 +3,7 @@ from fastapi import APIRouter, Depends # Import verb namespaces for their registration side effects before the door builds. import app.capabilities.google_maps import app.capabilities.google_search +import app.capabilities.indeed import app.capabilities.instagram import app.capabilities.reddit import app.capabilities.tiktok From 4171248f512b9b343913cbd27627a1f185280281 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:31 +0200 Subject: [PATCH 22/56] test: indeed capability test package --- surfsense_backend/tests/unit/capabilities/indeed/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 surfsense_backend/tests/unit/capabilities/indeed/__init__.py diff --git a/surfsense_backend/tests/unit/capabilities/indeed/__init__.py b/surfsense_backend/tests/unit/capabilities/indeed/__init__.py new file mode 100644 index 000000000..e69de29bb From 725cb8a82b153a154d27d10469e93b2f3ab35e9f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:31 +0200 Subject: [PATCH 23/56] test: indeed.scrape test package --- .../tests/unit/capabilities/indeed/scrape/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 surfsense_backend/tests/unit/capabilities/indeed/scrape/__init__.py diff --git a/surfsense_backend/tests/unit/capabilities/indeed/scrape/__init__.py b/surfsense_backend/tests/unit/capabilities/indeed/scrape/__init__.py new file mode 100644 index 000000000..e69de29bb From d029bc0cb806312610473c82e9cfa1392c04d49b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:31 +0200 Subject: [PATCH 24/56] test: indeed.scrape registration --- .../unit/capabilities/indeed/test_registry.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 surfsense_backend/tests/unit/capabilities/indeed/test_registry.py diff --git a/surfsense_backend/tests/unit/capabilities/indeed/test_registry.py b/surfsense_backend/tests/unit/capabilities/indeed/test_registry.py new file mode 100644 index 000000000..08bc8d7cf --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/indeed/test_registry.py @@ -0,0 +1,23 @@ +"""The indeed namespace registers its verb as one Capability the doors/agent read.""" + +from __future__ import annotations + +import pytest + +from app.capabilities import ( + indeed, # noqa: F401 — importing the namespace registers its verbs +) +from app.capabilities.core.store import get_capability +from app.capabilities.core.types import BillingUnit +from app.capabilities.indeed.scrape.schemas import ScrapeInput, ScrapeOutput + +pytestmark = pytest.mark.unit + + +def test_indeed_scrape_is_registered_and_billed_per_job(): + cap = get_capability("indeed.scrape") + + assert cap.name == "indeed.scrape" + assert cap.input_schema is ScrapeInput + assert cap.output_schema is ScrapeOutput + assert cap.billing_unit is BillingUnit.INDEED_JOB From 695a7d69e0c0500836917ae57b185dff43cb49c1 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:31 +0200 Subject: [PATCH 25/56] test: indeed.scrape executor mapping --- .../indeed/scrape/test_executor.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 surfsense_backend/tests/unit/capabilities/indeed/scrape/test_executor.py diff --git a/surfsense_backend/tests/unit/capabilities/indeed/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/indeed/scrape/test_executor.py new file mode 100644 index 000000000..da7c48364 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/indeed/scrape/test_executor.py @@ -0,0 +1,100 @@ +"""``indeed.scrape`` executor: verb input → actor input mapping → typed items. + +Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's +own payload→IndeedScrapeInput mapping and the dict→IndeedItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.indeed.scrape.executor import build_scrape_executor +from app.capabilities.indeed.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.indeed_jobs import ( + IndeedAccessBlockedError, + IndeedScrapeInput, +) + +pytestmark = pytest.mark.unit + + +class _FakeScraper: + """Records the actor input + limit it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[IndeedScrapeInput, int | None]] = [] + + async def __call__( + self, actor_input: IndeedScrapeInput, *, limit: int | None = None + ) -> list[dict]: + self.calls.append((actor_input, limit)) + return self._items + + +async def test_maps_urls_to_start_urls_and_wraps_items(): + scraper = _FakeScraper([{"jobKey": "abc", "title": "Data Analyst"}]) + execute = build_scrape_executor(scrape_fn=scraper) + + out = await execute(ScrapeInput(urls=["https://www.indeed.com/jobs?q=dev"])) + + assert isinstance(out, ScrapeOutput) + assert len(out.items) == 1 + assert out.items[0].jobKey == "abc" + assert out.items[0].title == "Data Analyst" + + (actor_input, _limit) = scraper.calls[0] + assert [u.url for u in actor_input.startUrls] == [ + "https://www.indeed.com/jobs?q=dev" + ] + assert actor_input.queries == [] + + +async def test_maps_search_params_and_passes_limit(): + scraper = _FakeScraper([]) + execute = build_scrape_executor(scrape_fn=scraper) + + await execute( + ScrapeInput( + search_queries=["data analyst"], + country="gb", + location="Remote", + job_type="fulltime", + level="entry_level", + remote="remote", + from_days=7, + sort="date", + max_items=40, + max_items_per_query=15, + ) + ) + + (actor_input, limit) = scraper.calls[0] + assert actor_input.queries == ["data analyst"] + assert actor_input.country == "gb" + assert actor_input.location == "Remote" + assert actor_input.jobType == "fulltime" + assert actor_input.level == "entry_level" + assert actor_input.remote == "remote" + assert actor_input.fromDays == 7 + assert actor_input.sort == "date" + assert actor_input.maxItems == 40 + assert actor_input.maxItemsPerQuery == 15 + # The outer collection limit is the caller's total-item cap. + assert limit == 40 + + +async def test_access_blocked_maps_to_forbidden(): + async def _blocked(actor_input: IndeedScrapeInput, *, limit: int | None = None): + raise IndeedAccessBlockedError("all IPs refused") + + execute = build_scrape_executor(scrape_fn=_blocked) + + with pytest.raises(ForbiddenError): + await execute(ScrapeInput(search_queries=["x"])) + + +def test_requires_a_source(): + with pytest.raises(ValueError, match="at least one"): + ScrapeInput() From dc945aa14dfd124e2077bbbb8be34d3925f699ac Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:50:52 +0200 Subject: [PATCH 26/56] test: assert reddit.scrape billing unit --- .../tests/unit/capabilities/reddit/test_registry.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/reddit/test_registry.py b/surfsense_backend/tests/unit/capabilities/reddit/test_registry.py index e0bca5f26..2ac839670 100644 --- a/surfsense_backend/tests/unit/capabilities/reddit/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/reddit/test_registry.py @@ -8,15 +8,16 @@ from app.capabilities import ( reddit, # noqa: F401 — importing the namespace registers its verbs ) from app.capabilities.core.store import get_capability +from app.capabilities.core.types import BillingUnit from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput pytestmark = pytest.mark.unit -def test_reddit_scrape_is_registered_and_free(): +def test_reddit_scrape_is_registered_and_billed_per_item(): cap = get_capability("reddit.scrape") assert cap.name == "reddit.scrape" assert cap.input_schema is ScrapeInput assert cap.output_schema is ScrapeOutput - assert cap.billing_unit is None + assert cap.billing_unit is BillingUnit.REDDIT_ITEM From ffc0675ac288281acb77efbdc097d70c80033bf0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:50:52 +0200 Subject: [PATCH 27/56] test: assert google_maps billing units --- .../tests/unit/capabilities/google_maps/test_registry.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py b/surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py index c6a007645..f9f774e8f 100644 --- a/surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py @@ -8,25 +8,26 @@ from app.capabilities import ( google_maps, # noqa: F401 — importing the namespace registers its verbs ) from app.capabilities.core.store import get_capability +from app.capabilities.core.types import BillingUnit from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput pytestmark = pytest.mark.unit -def test_google_maps_scrape_is_registered_and_free(): +def test_google_maps_scrape_is_registered_and_billed_per_place(): cap = get_capability("google_maps.scrape") assert cap.name == "google_maps.scrape" assert cap.input_schema is ScrapeInput assert cap.output_schema is ScrapeOutput - assert cap.billing_unit is None + assert cap.billing_unit is BillingUnit.GOOGLE_MAPS_PLACE -def test_google_maps_reviews_is_registered_and_free(): +def test_google_maps_reviews_is_registered_and_billed_per_review(): cap = get_capability("google_maps.reviews") assert cap.name == "google_maps.reviews" assert cap.input_schema is ReviewsInput assert cap.output_schema is ReviewsOutput - assert cap.billing_unit is None + assert cap.billing_unit is BillingUnit.GOOGLE_MAPS_REVIEW From 0089bb90012e2c356be498bd19be76e893b0e88e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:50:52 +0200 Subject: [PATCH 28/56] test: assert youtube billing units --- .../tests/unit/capabilities/youtube/test_registry.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/youtube/test_registry.py b/surfsense_backend/tests/unit/capabilities/youtube/test_registry.py index 756b65176..5de00891f 100644 --- a/surfsense_backend/tests/unit/capabilities/youtube/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/youtube/test_registry.py @@ -8,25 +8,26 @@ from app.capabilities import ( youtube, # noqa: F401 — importing the namespace registers its verbs ) from app.capabilities.core.store import get_capability +from app.capabilities.core.types import BillingUnit from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput pytestmark = pytest.mark.unit -def test_youtube_scrape_is_registered_and_free(): +def test_youtube_scrape_is_registered_and_billed_per_video(): cap = get_capability("youtube.scrape") assert cap.name == "youtube.scrape" assert cap.input_schema is ScrapeInput assert cap.output_schema is ScrapeOutput - assert cap.billing_unit is None + assert cap.billing_unit is BillingUnit.YOUTUBE_VIDEO -def test_youtube_comments_is_registered_and_free(): +def test_youtube_comments_is_registered_and_billed_per_comment(): cap = get_capability("youtube.comments") assert cap.name == "youtube.comments" assert cap.input_schema is CommentsInput assert cap.output_schema is CommentsOutput - assert cap.billing_unit is None + assert cap.billing_unit is BillingUnit.YOUTUBE_COMMENT From 6ee95e5b2c21986dc31d3016a8c515c7da749c69 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 29/56] feat: parse indeed /viewjob detail page --- .../platforms/indeed_jobs/parsers.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py index ab2b97cf9..b61b13e6a 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py @@ -22,6 +22,9 @@ _DEFAULT_BASE = "https://www.indeed.com" _JOBCARDS_ANCHOR = 'window.mosaic.providerData["mosaic-provider-jobcards"]=' +# A /viewjob page assigns the full posting to this global. +_INITIAL_DATA_ANCHOR = "window._initialData" + # Indeed's extractedSalary.type -> our SalaryPeriod. _SALARY_PERIODS = { "HOURLY": "hour", @@ -79,6 +82,26 @@ def extract_jobcards_blob(html: str) -> dict | None: return data if isinstance(data, dict) else None +def extract_initial_data(html: str) -> dict | None: + """Decode the ``window._initialData`` assignment on a /viewjob page.""" + import json + + idx = html.find(_INITIAL_DATA_ANCHOR) + if idx == -1: + return None + brace = html.find("{", idx + len(_INITIAL_DATA_ANCHOR)) + if brace == -1: + return None + blob = _brace_match(html, brace) + if not blob: + return None + try: + data = json.loads(blob) + except ValueError: + return None + return data if isinstance(data, dict) else None + + def job_results(blob: dict | None) -> list[dict[str, Any]]: """Return the raw job records from a decoded blob.""" if not isinstance(blob, dict): @@ -226,3 +249,57 @@ def parse_job(raw: dict[str, Any], *, base_url: str = _DEFAULT_BASE) -> dict[str "datePublished": _utc_from_ms(raw.get("pubDate")), "createdAt": _utc_from_ms(raw.get("createDate")), } + + +def _detail_salary(hdr: dict[str, Any]) -> dict[str, Any] | None: + """Salary from the detail header's flat ``salaryMin/Max/Currency/Type`` fields.""" + smin = hdr.get("salaryMin") + smax = hdr.get("salaryMax") + if smin is None and smax is None: + return None + return { + "salaryText": None, + "salaryMin": smin, + "salaryMax": smax, + "currency": hdr.get("salaryCurrency"), + "period": _SALARY_PERIODS.get(hdr.get("salaryType")), + "isEstimated": False, + } + + +def parse_job_detail(html: str, *, base_url: str = _DEFAULT_BASE) -> dict[str, Any]: + """Map a /viewjob page to enrichment fields (empty dict if not a job page). + + Returns only fields the detail page actually carries, so the caller can merge + it onto a listing item without clobbering known values with blanks. The full + description (``sanitizedJobDescription``) is the field listings never have. + """ + data = extract_initial_data(html) + if not isinstance(data, dict): + return {} + jim = (data.get("jobInfoWrapperModel") or {}).get("jobInfoModel") or {} + if not isinstance(jim, dict): + return {} + hdr = jim.get("jobInfoHeaderModel") + hdr = hdr if isinstance(hdr, dict) else {} + taxo = _taxonomy(hdr) + desc_html = jim.get("sanitizedJobDescription") + desc_html = desc_html if isinstance(desc_html, str) and desc_html else None + remote_model = hdr.get("remoteWorkModel") + out: dict[str, Any] = { + "descriptionHtml": desc_html, + "descriptionText": _clean_snippet(desc_html), + "title": hdr.get("jobTitle"), + "company": hdr.get("companyName"), + "companyUrl": _abs_url(hdr.get("companyOverviewLink"), base_url), + "formattedLocation": hdr.get("formattedLocation") or data.get("jobLocation"), + "remoteType": remote_model.get("type") + if isinstance(remote_model, dict) + else None, + "jobTypes": _job_types(hdr, taxo), + "benefits": taxo.get("benefits", []), + "salary": _detail_salary(hdr), + } + if _is_remote(hdr, taxo): + out["isRemote"] = True + return {k: v for k, v in out.items() if v not in (None, [], {})} From 51c7220321103580a615ba730b463400531d6995 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 30/56] feat: indeed job-detail and enrichment flow --- .../platforms/indeed_jobs/scraper.py | 65 +++++++++++++++---- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py index 4f66f2ebf..0c9fcfb16 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py @@ -17,7 +17,12 @@ from typing import Any from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse from .fetch import IndeedSession, now_iso, open_session -from .parsers import extract_jobcards_blob, job_results, parse_job +from .parsers import ( + extract_jobcards_blob, + job_results, + parse_job, + parse_job_detail, +) from .schemas import IndeedItem, IndeedScrapeInput from .url_resolver import build_search_url, resolve_url @@ -76,24 +81,25 @@ async def _paginate( return -def _targets(input_model: IndeedScrapeInput) -> list[tuple[str, str]]: - """Resolve inputs to ``(url, domain)`` search/company targets. +def _targets(input_model: IndeedScrapeInput) -> list[tuple[str, str, str]]: + """Resolve inputs to ``(kind, url, domain)`` targets. - ``startUrls`` take precedence over ``queries``. Job (``/viewjob``) URLs are - skipped here; detail enrichment is a separate flow. + ``startUrls`` take precedence over ``queries``. ``kind`` is ``search`` for + query-built and search/company URLs, or ``job`` for a ``/viewjob`` URL. """ if input_model.startUrls: - out: list[tuple[str, str]] = [] + out: list[tuple[str, str, str]] = [] for entry in input_model.startUrls: resolved = resolve_url(entry.url) - if resolved is None or resolved.kind == "job": - logger.warning("[indeed] skipping unsupported URL: %s", entry.url) + if resolved is None: + logger.warning("[indeed] skipping unrecognized URL: %s", entry.url) continue - out.append((resolved.url, resolved.domain)) + kind = "job" if resolved.kind == "job" else "search" + out.append((kind, resolved.url, resolved.domain)) return out domain = None - urls: list[tuple[str, str]] = [] + urls: list[tuple[str, str, str]] = [] for query in input_model.queries: url = build_search_url( query, @@ -107,16 +113,49 @@ def _targets(input_model: IndeedScrapeInput) -> list[tuple[str, str]]: sort=input_model.sort, ) domain = domain or urlparse(url).hostname or "www.indeed.com" - urls.append((url, domain)) + urls.append(("search", url, domain)) return urls +async def _enrich(session: IndeedSession, item: dict[str, Any], base_url: str) -> None: + """Merge a job's /viewjob detail (full description, etc.) onto ``item`` in place. + + Best-effort: a blocked or malformed detail page leaves the listing fields as-is + rather than failing the run. + """ + job_url = item.get("jobUrl") + if not isinstance(job_url, str): + return + try: + detail = parse_job_detail(await session.fetch_html(job_url), base_url=base_url) + except Exception as exc: + logger.warning("[indeed] detail fetch failed for %s: %s", job_url, exc) + return + item.update(detail) + + +async def _job_item( + session: IndeedSession, url: str, base_url: str +) -> dict[str, Any] | None: + """Scrape a single /viewjob URL into an item from its detail page alone.""" + detail = parse_job_detail(await session.fetch_html(url), base_url=base_url) + if not detail: + return None + return _emit({"jobUrl": url, "source": "indeed", **detail}) + + async def iter_indeed( input_model: IndeedScrapeInput, session: IndeedSession ) -> AsyncIterator[dict[str, Any]]: """Stream flat job items for every target, deduped by ``jobKey`` across all.""" global_seen: set[str] = set() - for url, domain in _targets(input_model): + for kind, url, domain in _targets(input_model): + base_url = f"https://{domain}" + if kind == "job": + item = await _job_item(session, url, base_url) + if item is not None: + yield item + continue async for item in _paginate( session, url, domain=domain, max_items=input_model.maxItemsPerQuery ): @@ -125,6 +164,8 @@ async def iter_indeed( if job_key in global_seen: continue global_seen.add(job_key) + if input_model.scrapeJobDetails: + await _enrich(session, item, base_url) yield item From 81f535e9d0d3fde7fc0c509e553e3fb9b0e0db98 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 31/56] feat: add scrape_job_details to indeed.scrape input --- .../app/capabilities/indeed/scrape/schemas.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/surfsense_backend/app/capabilities/indeed/scrape/schemas.py b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py index a02f6f13b..02e29100d 100644 --- a/surfsense_backend/app/capabilities/indeed/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py @@ -75,6 +75,13 @@ class ScrapeInput(BaseModel): default="relevance", description="Result ordering: relevance or date.", ) + scrape_job_details: bool = Field( + default=False, + description=( + "Fetch each job's detail page for the full description (slower: one " + "extra page load per job)." + ), + ) max_items: int = Field( default=25, ge=1, From ec258d8387190f75e8fd133e9596e47504d0767b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 32/56] feat: map scrape_job_details in indeed executor --- surfsense_backend/app/capabilities/indeed/scrape/executor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/capabilities/indeed/scrape/executor.py b/surfsense_backend/app/capabilities/indeed/scrape/executor.py index 60df11972..85983e0da 100644 --- a/surfsense_backend/app/capabilities/indeed/scrape/executor.py +++ b/surfsense_backend/app/capabilities/indeed/scrape/executor.py @@ -33,6 +33,7 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: remote=payload.remote, fromDays=payload.from_days, sort=payload.sort, + scrapeJobDetails=payload.scrape_job_details, maxItems=payload.max_items, maxItemsPerQuery=payload.max_items_per_query, ) From 0b73581cbe501c5f74b2f11370289ec0ebd0be76 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 33/56] test: indeed viewjob detail fixture --- .../unit/platforms/indeed_jobs/fixtures/sample_viewjob.html | 1 + 1 file changed, 1 insertion(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_viewjob.html diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_viewjob.html b/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_viewjob.html new file mode 100644 index 000000000..ad09bf38c --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_viewjob.html @@ -0,0 +1 @@ + \ No newline at end of file From 38c98ba0e425b59508fa9e0536b84332c2a4d548 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 34/56] test: indeed detail parser --- .../platforms/indeed_jobs/test_parsers.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py index 7b9ca6298..38a6544b6 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py @@ -9,6 +9,7 @@ from app.proprietary.platforms.indeed_jobs.parsers import ( extract_jobcards_blob, job_results, parse_job, + parse_job_detail, ) _FIXTURE_DIR = Path(__file__).parent / "fixtures" @@ -137,3 +138,39 @@ def test_fixture_blob_parses_into_items(): assert item["title"] assert item["jobUrl"].startswith("https://www.indeed.com/viewjob?jk=") assert "salaryText" in item["salary"] + + +# --- detail page (parse_job_detail) ---------------------------------------- + + +def test_parse_job_detail_extracts_description_and_fields(): + html = (_FIXTURE_DIR / "sample_viewjob.html").read_text() + detail = parse_job_detail(html) + assert detail["descriptionHtml"].startswith("
") + assert "Data Analyst" in detail["descriptionText"] + assert "
" not in detail["descriptionText"] # tags stripped + assert detail["title"] + assert detail["company"] + assert detail["formattedLocation"] + assert detail["jobTypes"] == ["Full-time"] + assert detail["benefits"] == ["401(k)", "Health insurance"] + assert detail["isRemote"] is True + assert detail["remoteType"] == "HYBRID" + sal = detail["salary"] + assert (sal["salaryMin"], sal["salaryMax"], sal["period"]) == (60000, 90000, "year") + + +def test_parse_job_detail_omits_blank_fields(): + # A page with a header but no salary/description must not emit those keys, + # so a merge won't clobber listing values with blanks. + html = ( + "' + ) + detail = parse_job_detail(html) + assert detail == {"title": "Analyst"} + + +def test_parse_job_detail_not_a_job_page_returns_empty(): + assert parse_job_detail("just a moment...") == {} From d2af2402b4a3453dfdaaa1f2ddbeae96b72ba8cc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 35/56] test: indeed detail and enrichment flow --- .../platforms/indeed_jobs/test_scraper.py | 45 +++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py index c56d64f75..e5ea23452 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py @@ -22,8 +22,21 @@ def _page_html(job_keys: list[str]) -> str: ) +def _detail_html(job_key: str) -> str: + """A minimal /viewjob page carrying a full description for ``job_key``.""" + data = { + "jobInfoWrapperModel": { + "jobInfoModel": { + "sanitizedJobDescription": f"
Full description for {job_key}
", + "jobInfoHeaderModel": {"jobTitle": f"Detailed {job_key}"}, + } + } + } + return f"" + + class _FakeSession: - """Returns per-``start`` pages and records the URLs requested.""" + """Returns per-``start`` search pages (or a /viewjob detail) and records URLs.""" def __init__(self, pages: dict[int, list[str]]) -> None: self._pages = pages @@ -31,7 +44,10 @@ class _FakeSession: async def fetch_html(self, url: str) -> str: self.fetched.append(url) - start = int(parse_qs(urlparse(url).query).get("start", ["0"])[0]) + query = parse_qs(urlparse(url).query) + if "/viewjob" in url: + return _detail_html(query.get("jk", [""])[0]) + start = int(query.get("start", ["0"])[0]) return _page_html(self._pages.get(start, [])) @@ -81,7 +97,7 @@ async def test_global_dedupe_across_queries(): @pytest.mark.asyncio -async def test_start_urls_skip_viewjob_and_scrape_search(): +async def test_start_urls_scrape_search_and_job_url_detail(): session = _FakeSession({0: ["k1"]}) input_model = IndeedScrapeInput( startUrls=[ @@ -91,7 +107,28 @@ async def test_start_urls_skip_viewjob_and_scrape_search(): maxItemsPerQuery=100, ) items = await _collect(input_model, session) - assert [i["jobKey"] for i in items] == ["k1"] + assert len(items) == 2 + search_item, job_item = items + assert search_item["jobKey"] == "k1" + # The /viewjob URL is scraped from its detail page alone. + assert job_item["jobUrl"].endswith("jk=abc") + assert job_item["title"] == "Detailed abc" + assert "Full description for abc" in job_item["descriptionText"] + + +@pytest.mark.asyncio +async def test_scrape_job_details_enriches_listing_items(): + session = _FakeSession({0: ["k1", "k2"]}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100, scrapeJobDetails=True), + session, + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + for it in items: + assert it["descriptionHtml"].startswith("
Full description for") + assert "Full description for" in it["descriptionText"] + # One extra /viewjob load per listing item. + assert sum("/viewjob" in u for u in session.fetched) == 2 @pytest.mark.asyncio From 2ade6bb90318c4cd04df9ec2a07a0e88e1d6afbf Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 36/56] feat: indeed subagent package --- .../chat/multi_agent_chat/subagents/builtins/indeed/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/__init__.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/__init__.py new file mode 100644 index 000000000..ddd8a9cd3 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/__init__.py @@ -0,0 +1 @@ +"""``indeed`` builtin subagent: structured Indeed job postings.""" From 7cbd8345f9cb82f4daf6972736512515ee72b61e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 37/56] feat: indeed subagent tools package --- .../multi_agent_chat/subagents/builtins/indeed/tools/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/__init__.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/__init__.py new file mode 100644 index 000000000..e69de29bb From ada83efedf06e6458a9db36b07ddda8638163225 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 38/56] feat: wire indeed.scrape into subagent tools --- .../subagents/builtins/indeed/tools/index.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/index.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/index.py new file mode 100644 index 000000000..523579471 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/index.py @@ -0,0 +1,27 @@ +"""``indeed`` sub-agent tools: the Indeed scrape capability verb.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.indeed.scrape.definition import INDEED_SCRAPE + +NAME = "indeed" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [INDEED_SCRAPE] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) From 723eb8c6cb17e096d14ae9c68f06bd599d412237 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 39/56] feat: indeed subagent routing description --- .../multi_agent_chat/subagents/builtins/indeed/description.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/description.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/description.md new file mode 100644 index 000000000..221c54fe9 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/description.md @@ -0,0 +1,2 @@ +Indeed jobs specialist: pulls structured job postings — title, company, location, salary (range, currency, period), job types, benefits, remote/hybrid flag, posting age, apply URL, and the full job description. Discovers jobs by search query (with country, location, radius, job type, experience level, remote/hybrid, and date-posted filters), or scrapes a known Indeed search, company jobs, or single job URL as-is. Optionally fetches each job's detail page for the full description. +Use whenever the task is to find or gather job listings from Indeed — openings for a role, hiring at a company, salaries for a title in a location, or remote roles in a field. Triggers include "find jobs for X", "who is hiring X", "data analyst jobs in Y", "remote X roles", and scraping a specific Indeed URL. Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), or other job boards. From e0cc816f673477e1ad5fc6a0825341626f33a47d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 40/56] feat: indeed subagent system prompt --- .../builtins/indeed/system_prompt.md | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md new file mode 100644 index 000000000..b034b700a --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md @@ -0,0 +1,66 @@ +You are the SurfSense Indeed sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from live Indeed job data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it. + + + +- `indeed_scrape` +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Finding jobs for a role: call `indeed_scrape` with `search_queries`; narrow with `location`, `country`, `job_type`, `level`, `remote`, `radius`, and `from_days`. +- Scraping a specific Indeed URL: pass a search, company jobs, or single job URL in `urls`. +- Full descriptions: set `scrape_job_details=true` to fetch each job's detail page (slower: one extra load per job). Leave it false when the listing snippet is enough. +- Controlling volume: use `max_items` for the total cap and `max_items_per_query` per search. +- Requested counts: `max_items` defaults to only 25 — when the task asks for N jobs, set `max_items` and `max_items_per_query` above N (with headroom for off-topic hits). A call that caps below the target can never satisfy it. +- Under-delivery: if the first call returns fewer on-topic results than requested, broaden it yourself — more query phrasings, wider `radius`, drop restrictive filters, larger `from_days` — before settling. Return `status=partial` only after the broadened attempt, never after a single narrow call. +- Batch multiple search terms into one call rather than many single-term calls. + +- Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, salary/rank changes). + + + +- Use only tools in ``. +- Report only results present in the tool output. Never invent titles, companies, salaries, locations, or description text. + + + +- Do not read arbitrary web pages — that belongs to the web crawling specialist. +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Google results belong to the Google Search specialist; other job boards are out of scope. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable query or URL — return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the scope you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: one entry per distinct job or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). +- `evidence.sources`: one Indeed job URL per finding when applicable, same cap as findings. List each URL once. + + From e193867e7724f372b4f87a8aa7577e77a2f3cbae Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 41/56] feat: indeed subagent builder --- .../subagents/builtins/indeed/agent.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/agent.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/agent.py new file mode 100644 index 000000000..6da835dbd --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/agent.py @@ -0,0 +1,43 @@ +"""``indeed`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Pulls structured job postings from Indeed search and company pages." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) From 238d2767c5e4392a0cd4e51424e7e3bfe4b957d8 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 42/56] feat: register indeed subagent in registry --- .../app/agents/chat/multi_agent_chat/subagents/registry.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index de37edfb9..bc368facd 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -21,6 +21,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.google_maps.agent impor from app.agents.chat.multi_agent_chat.subagents.builtins.google_search.agent import ( build_subagent as build_google_search_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.indeed.agent import ( + build_subagent as build_indeed_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.instagram.agent import ( build_subagent as build_instagram_subagent, ) @@ -85,6 +88,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "google_drive": build_google_drive_subagent, "google_maps": build_google_maps_subagent, "google_search": build_google_search_subagent, + "indeed": build_indeed_subagent, "instagram": build_instagram_subagent, "knowledge_base": build_knowledge_base_subagent, "mcp_discovery": build_mcp_discovery_subagent, From 3a9b0405c3588d18d7738821d229572a19bd8a01 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 43/56] feat: add indeed to required-connector map --- surfsense_backend/app/agents/chat/multi_agent_chat/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index 6bbd808fb..f0066484a 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -35,6 +35,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "youtube": frozenset(), "google_maps": frozenset(), "google_search": frozenset(), + "indeed": frozenset(), "reddit": frozenset(), "instagram": frozenset(), "tiktok": frozenset(), From 9e379602434344ef1513f4fe339d9ebdd4bb4319 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 44/56] test: pin indeed in subagent roster --- .../unit/agents/multi_agent_chat/test_subagent_composition.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index ee889cd17..cf5982780 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -32,6 +32,7 @@ _EXPECTED_SUBAGENTS = frozenset( "google_drive", "google_maps", "google_search", + "indeed", "instagram", "knowledge_base", "mcp_discovery", From 457c073c44a5d2b129abe0026198f88d4122d43e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:54:49 +0200 Subject: [PATCH 45/56] feat: add indeed MCP scraper tool --- .../features/scrapers/platforms/indeed.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 surfsense_mcp/mcp_server/features/scrapers/platforms/indeed.py diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/indeed.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/indeed.py new file mode 100644 index 000000000..6ea0007f9 --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/indeed.py @@ -0,0 +1,135 @@ +"""Indeed scraper tool.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +IndeedSort = Literal["relevance", "date"] +IndeedJobType = Literal[ + "fulltime", + "parttime", + "contract", + "internship", + "temporary", + "permanent", + "seasonal", + "freelance", +] +IndeedLevel = Literal["entry_level", "mid_level", "senior_level"] +IndeedRemote = Literal["remote", "hybrid"] + + +def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None: + """Register the Indeed tool.""" + + @mcp.tool( + name="surfsense_indeed_scrape", + title="Search or scrape Indeed jobs", + annotations=SCRAPE, + structured_output=False, + ) + async def indeed_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="Indeed URLs: a search page " + "('https://www.indeed.com/jobs?q=data+analyst'), a company jobs " + "page ('/cmp//jobs'), or a single job ('/viewjob?jk=...'). " + "Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Job search terms, e.g. ['data analyst', 'ml engineer']. " + "Provide search_queries OR urls." + ), + ] = None, + country: Annotated[ + str, + Field(description="Country code selecting the Indeed domain, e.g. 'us', 'gb'."), + ] = "us", + location: Annotated[ + str | None, + Field(description="Where to search, e.g. 'Remote', 'New York, NY'."), + ] = None, + radius: Annotated[ + int | None, + Field(description="Search radius in miles/km around location."), + ] = None, + job_type: Annotated[ + IndeedJobType | None, + Field(description="Employment type filter."), + ] = None, + level: Annotated[ + IndeedLevel | None, + Field(description="Experience level filter."), + ] = None, + remote: Annotated[ + IndeedRemote | None, + Field(description="Work model filter: remote or hybrid."), + ] = None, + from_days: Annotated[ + int | None, + Field(description="Only return jobs posted within the last N days."), + ] = None, + sort: Annotated[ + IndeedSort, Field(description="Result ordering: relevance or date.") + ] = "relevance", + scrape_job_details: Annotated[ + bool, + Field( + description="True fetches each job's detail page for the full " + "description (slower); False returns the listing snippet only." + ), + ] = False, + max_items: Annotated[ + int, Field(ge=1, description="Maximum jobs to return in total.") + ] = 25, + max_items_per_query: Annotated[ + int, Field(ge=0, description="Max jobs per search/company target.") + ] = 25, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape public Indeed job postings. + + Use this for ANY Indeed job research — openings for a role, who is hiring + at a company, salaries for a title in a location, or remote roles — + instead of a generic web search. Returns jobs with title, company, + location, salary, job types, and description; set scrape_job_details for + the full description per job. + Example: search_queries=['data analyst'], location='Remote', max_items=30. + """ + return await run_scraper( + client, + context, + platform="indeed", + verb="scrape", + payload={ + "urls": urls, + "search_queries": search_queries, + "country": country, + "location": location, + "radius": radius, + "job_type": job_type, + "level": level, + "remote": remote, + "from_days": from_days, + "sort": sort, + "scrape_job_details": scrape_job_details, + "max_items": max_items, + "max_items_per_query": max_items_per_query, + }, + workspace=workspace, + response_format=response_format, + ) From b89fe1a4f08f5069108ca4e065f64c051f3b596b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:54:49 +0200 Subject: [PATCH 46/56] feat: register indeed MCP tool --- surfsense_mcp/mcp_server/features/scrapers/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index 9aabbe0e5..e4823ca5c 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -1,7 +1,7 @@ """Scraper tools: one MCP surface per SurfSense platform capability. -Web crawl, Google Search, Reddit, YouTube, and Google Maps each get a tool that -maps a natural-language request to the workspace's scraper. Two run-history tools +Web crawl, Google Search, Reddit, YouTube, Google Maps, and Indeed each get a +tool that maps a natural-language request to the workspace's scraper. Two run-history tools list and fetch past runs, so a large result truncated inline can be retrieved in full later. Each platform lives in its own module under platforms/. """ @@ -16,6 +16,7 @@ from . import run_history from .platforms import ( google_maps, google_search, + indeed, instagram, reddit, tiktok, @@ -31,6 +32,7 @@ _REGISTRARS = ( instagram, tiktok, google_maps, + indeed, run_history, ) From e88ff8cef7a22e96aba153eea88b4f8886838f83 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:54:49 +0200 Subject: [PATCH 47/56] test: expect indeed MCP tool in selfcheck --- surfsense_mcp/mcp_server/selfcheck.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index bb61151bc..b9f21c638 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -31,6 +31,7 @@ EXPECTED_TOOLS = { "surfsense_google_maps_reviews", "surfsense_instagram_scrape", "surfsense_instagram_details", + "surfsense_indeed_scrape", "surfsense_list_scraper_runs", "surfsense_get_scraper_run", # knowledge-base management From 65c757fbc60d8e8d77ce24158a59922e762aff64 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:54:49 +0200 Subject: [PATCH 48/56] feat: mention indeed in MCP server instructions --- surfsense_mcp/mcp_server/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/surfsense_mcp/mcp_server/server.py b/surfsense_mcp/mcp_server/server.py index f801aeb15..ef5f15c8a 100644 --- a/surfsense_mcp/mcp_server/server.py +++ b/surfsense_mcp/mcp_server/server.py @@ -38,7 +38,8 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: "task involves Reddit (posts, comments, finding subreddits or " "communities), YouTube (videos, transcripts, comments), Instagram " "(posts, reels, profile details), TikTok (videos by hashtag, " - "search, or URL), Google Maps (places, reviews), Google Search " + "search, or URL), Google Maps (places, reviews), Indeed (job " + "postings by role, company, or location), Google Search " "results, or reading " "specific web pages. Scraper results are persisted as runs; if an " "inline result is truncated, fetch it in full with " From 5b0982f80b19a317117d5500724d2b27fc63a45f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:57:15 +0200 Subject: [PATCH 49/56] docs: add indeed native connector page and listings --- .../content/docs/connectors/index.mdx | 2 +- .../content/docs/connectors/native/indeed.mdx | 51 +++++++++++++++++++ .../content/docs/connectors/native/index.mdx | 7 ++- .../content/docs/connectors/native/meta.json | 1 + .../content/docs/how-to/mcp-server.mdx | 8 +-- 5 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 surfsense_web/content/docs/connectors/native/indeed.mdx diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx index f16cf79b9..5bf7d33c1 100644 --- a/surfsense_web/content/docs/connectors/index.mdx +++ b/surfsense_web/content/docs/connectors/index.mdx @@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li } title="Native Connectors" - description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" + description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, and Web Crawl — usable in chat, the API Playground, or via REST" href="/docs/connectors/native" /> /jobs`), or a single job (`/viewjob?jk=`) (max 20 sources per call, combined with queries) | +| `search_queries` | — | Job search terms, e.g. `["data analyst"]` | +| `country` | `us` | Country code selecting the Indeed domain, e.g. `us`, `gb`, `de` | +| `location` | — | Where to search, e.g. `Remote`, `New York, NY` | +| `radius` | — | Search radius in miles/km around `location` | +| `job_type` | — | `fulltime`, `parttime`, `contract`, `internship`, `temporary`, `permanent`, `seasonal`, or `freelance` | +| `level` | — | `entry_level`, `mid_level`, or `senior_level` | +| `remote` | — | `remote` or `hybrid` | +| `from_days` | — | Only return jobs posted within the last N days | +| `sort` | `relevance` | `relevance` or `date` | +| `scrape_job_details` | `false` | Fetch each job's detail page for the full description (slower: one extra page load per job) | +| `max_items` | `25` | Max total jobs returned across all sources (hard cap 100) | +| `max_items_per_query` | `25` | Max jobs per search/company target | + +## Example + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/indeed/scrape" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "search_queries": ["data analyst"], + "location": "Remote", + "remote": "remote", + "sort": "date", + "max_items": 30 + }' +``` + +The response is `{ "items": [...] }` — one item per job posting. Billing is per returned job. + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Indeed → Scrape** in your workspace. diff --git a/surfsense_web/content/docs/connectors/native/index.mdx b/surfsense_web/content/docs/connectors/native/index.mdx index 12e1a7be3..fff814b7f 100644 --- a/surfsense_web/content/docs/connectors/native/index.mdx +++ b/surfsense_web/content/docs/connectors/native/index.mdx @@ -1,6 +1,6 @@ --- title: Native Connectors -description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the web +description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, and the web --- import { Card, Cards } from 'fumadocs-ui/components/card'; @@ -38,6 +38,11 @@ Native connectors are SurfSense's own scraper APIs — built into the platform, description="Structured SERPs: organic results, people-also-ask, AI overviews" href="/docs/connectors/native/google-search" /> + @@ -257,14 +257,14 @@ For self-host (stdio), all settings are environment variables passed by the clie - **401 errors** — the API key is wrong or expired; create a new one. - **403 errors** — API access is disabled for the workspace; toggle **API key access** on under **API Playground → API Keys**. - **"Could not reach SurfSense"** — the backend isn't running or `SURFSENSE_BASE_URL` is wrong. -- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 24 tools register without needing a backend. +- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 25 tools register without needing a backend. ## Tools reference | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_tiktok_comments`, `surfsense_tiktok_user_search`, `surfsense_tiktok_trending`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_tiktok_comments`, `surfsense_tiktok_user_search`, `surfsense_tiktok_trending`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_indeed_scrape`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. From e19c876fbc8e984e6cd18348c127b6d8c803c98b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 19:03:51 +0200 Subject: [PATCH 50/56] feat: wire indeed connector into env, readmes, playground and marketing --- README.es.md | 5 +- README.hi.md | 5 +- README.md | 5 +- README.pt-BR.md | 5 +- README.zh-CN.md | 5 +- docker/.env.example | 1 + surfsense_backend/.env.example | 1 + surfsense_web/lib/auth-utils.ts | 1 + .../lib/connectors-marketing/google-maps.tsx | 1 + .../connectors-marketing/google-search.tsx | 1 + .../lib/connectors-marketing/indeed.tsx | 327 ++++++++++++++++++ .../lib/connectors-marketing/index.ts | 2 + .../lib/connectors-marketing/instagram.tsx | 1 + .../lib/connectors-marketing/reddit.tsx | 1 + .../lib/connectors-marketing/tiktok.tsx | 1 + .../lib/connectors-marketing/web-crawl.tsx | 1 + .../lib/connectors-marketing/youtube.tsx | 1 + surfsense_web/lib/playground/catalog.ts | 7 + .../lib/playground/platform-icons.tsx | 1 + surfsense_web/public/connectors/indeed.svg | 5 + 20 files changed, 367 insertions(+), 10 deletions(-) create mode 100644 surfsense_web/lib/connectors-marketing/indeed.tsx create mode 100644 surfsense_web/public/connectors/indeed.svg diff --git a/README.es.md b/README.es.md index 08231b5ab..3d38b8334 100644 --- a/README.es.md +++ b/README.es.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM para investigación de inteligencia competitiva -SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. +SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. > [!NOTE] > **📢 Una nota para nuestros usuarios de la alternativa a NotebookLM** @@ -107,6 +107,7 @@ Las automatizaciones ejecutan turnos completos de agente según un horario o en | **TikTok** | Videos, comentarios, hashtags y perfiles sin aprobación de la Research API | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Lugares, calificaciones y reseñas para investigar competidores locales y prospectos | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | SERPs en vivo para seguimiento de posiciones y monitoreo de mercado | [Google Search API](https://www.surfsense.com/google-search) | +| **Indeed** | Ofertas de empleo públicas con salarios y descripciones completas, por búsqueda o empresa | [Indeed Scraper API](https://www.surfsense.com/indeed) | | **Web Crawl** (rastreo web) | Cualquier página de la web abierta como contenido limpio y estructurado | [Web Crawling API](https://www.surfsense.com/web-crawl) | | **Conectores MCP externos** | Conecta cualquier servidor MCP a tus agentes, con OAuth de un clic para Notion, Slack, Jira y más | [External MCP Connectors](https://www.surfsense.com/external-mcp-connectors) | @@ -246,7 +247,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | Característica | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Datos de mercado en vivo para agentes** | No | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search y rastreo web vía API REST y MCP | +| **Datos de mercado en vivo para agentes** | No | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed y rastreo web vía API REST y MCP | | **Servidor MCP** | No | Cada conector expuesto como herramienta nativa de agente, más servidores MCP propios con aplicaciones OAuth de un clic | | **Fuentes por Notebook** | 50 (gratis) a 600 (Ultra, $249.99/mes) | Ilimitadas | | **Número de Notebooks** | 100 (gratis) a 500 (niveles de pago) | Ilimitado | diff --git a/README.hi.md b/README.hi.md index eb65ff489..2a26e2146 100644 --- a/README.hi.md +++ b/README.hi.md @@ -22,7 +22,7 @@ # SurfSense: कॉम्पिटिटिव इंटेलिजेंस रिसर्च के लिए NotebookLM -SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। +SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। > [!NOTE] > **📢 हमारे NotebookLM-विकल्प उपयोगकर्ताओं के लिए एक सूचना** @@ -107,6 +107,7 @@ SurfSense **AI एजेंट्स के लिए ओपन सोर्स | **TikTok** | Research API अप्रूवल के बिना वीडियो, कमेंट, हैशटैग और प्रोफ़ाइल | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | स्थानीय प्रतिस्पर्धी और लीड रिसर्च के लिए स्थान, रेटिंग और रिव्यू | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | रैंक ट्रैकिंग और मार्केट मॉनिटरिंग के लिए लाइव SERP | [Google Search API](https://www.surfsense.com/google-search) | +| **Indeed** | सार्वजनिक नौकरी लिस्टिंग, सैलरी और पूरे विवरण के साथ, सर्च या कंपनी के अनुसार | [Indeed Scraper API](https://www.surfsense.com/indeed) | | **Web Crawl** | ओपन वेब का कोई भी पेज साफ़-सुथरे, स्ट्रक्चर्ड कंटेंट के रूप में | [Web Crawling API](https://www.surfsense.com/web-crawl) | | **External MCP Connectors** | कोई भी MCP सर्वर अपने एजेंट्स से जोड़ें, Notion, Slack, Jira और अन्य के लिए वन-क्लिक OAuth के साथ | [External MCP Connectors](https://www.surfsense.com/external-mcp-connectors) | @@ -246,7 +247,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | फ़ीचर | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **एजेंट्स के लिए लाइव मार्केट डेटा** | नहीं | REST API और MCP के ज़रिए Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search और वेब क्रॉल कनेक्टर | +| **एजेंट्स के लिए लाइव मार्केट डेटा** | नहीं | REST API और MCP के ज़रिए Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed और वेब क्रॉल कनेक्टर | | **MCP सर्वर** | नहीं | हर कनेक्टर नेटिव एजेंट टूल के रूप में उपलब्ध, साथ ही वन-क्लिक OAuth ऐप्स के साथ अपने MCP सर्वर लाने की सुविधा | | **प्रति नोटबुक स्रोत** | 50 (Free) से 600 (Ultra, $249.99/माह) | असीमित | | **नोटबुक की संख्या** | 100 (Free) से 500 (सशुल्क टियर) | असीमित | diff --git a/README.md b/README.md index ba289ada4..56af65dd1 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM for Competitive Intelligence Research -SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. +SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. > [!NOTE] > **📢 A note for our NotebookLM-alternative users** @@ -107,6 +107,7 @@ Automations run full agent turns on a schedule or in response to events, then wr | **TikTok** | Videos, comments, hashtags, and profiles without Research API approval | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Places, ratings, and reviews for local competitor and lead research | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | Live SERPs for rank tracking and market monitoring | [Google Search API](https://www.surfsense.com/google-search) | +| **Indeed** | Public job postings with salaries and full descriptions, by search or company | [Indeed Scraper API](https://www.surfsense.com/indeed) | | **Web Crawl** | Any page on the open web as clean, structured content | [Web Crawling API](https://www.surfsense.com/web-crawl) | | **External MCP Connectors** | Bring any MCP server to your agents, with one-click OAuth for Notion, Slack, Jira, and more | [External MCP Connectors](https://www.surfsense.com/external-mcp-connectors) | @@ -245,7 +246,7 @@ Still comparing us as a NotebookLM alternative? Here is the honest breakdown. | Feature | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Live market data for agents** | No | Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and web crawl connectors via REST API and MCP | +| **Live market data for agents** | No | Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, and web crawl connectors via REST API and MCP | | **MCP server** | No | Every connector exposed as a native agent tool, plus bring-your-own MCP servers with one-click OAuth apps | | **Sources per Notebook** | 50 (Free) to 600 (Ultra, $249.99/mo) | Unlimited | | **Number of Notebooks** | 100 (Free) to 500 (paid tiers) | Unlimited | diff --git a/README.pt-BR.md b/README.pt-BR.md index 327e3373c..496398d19 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM para Pesquisa de Inteligência Competitiva -O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. +O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. > [!NOTE] > **📢 Um recado para nossos usuários que buscavam uma alternativa ao NotebookLM** @@ -107,6 +107,7 @@ As automações executam turnos completos de agente de forma agendada ou em resp | **TikTok** | Vídeos, comentários, hashtags e perfis sem aprovação da Research API | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Estabelecimentos, avaliações e reviews para pesquisa local de concorrentes e leads | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | SERPs ao vivo para acompanhamento de rankings e monitoramento de mercado | [Google Search API](https://www.surfsense.com/google-search) | +| **Indeed** | Vagas públicas com salários e descrições completas, por busca ou empresa | [Indeed Scraper API](https://www.surfsense.com/indeed) | | **Web Crawl** (rastreamento web) | Qualquer página da web aberta como conteúdo limpo e estruturado | [Web Crawling API](https://www.surfsense.com/web-crawl) | | **Conectores MCP externos** | Traga qualquer servidor MCP para seus agentes, com OAuth em um clique para Notion, Slack, Jira e outros | [External MCP Connectors](https://www.surfsense.com/external-mcp-connectors) | @@ -246,7 +247,7 @@ Ainda nos comparando como alternativa ao NotebookLM? Aqui está o comparativo ho | Recurso | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Dados de mercado ao vivo para agentes** | Não | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search e rastreamento web via API REST e MCP | +| **Dados de mercado ao vivo para agentes** | Não | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed e rastreamento web via API REST e MCP | | **Servidor MCP** | Não | Cada conector exposto como ferramenta nativa de agente, além de servidores MCP próprios com apps OAuth em um clique | | **Fontes por Notebook** | 50 (gratuito) a 600 (Ultra, US$ 249,99/mês) | Ilimitadas | | **Número de Notebooks** | 100 (gratuito) a 500 (planos pagos) | Ilimitado | diff --git a/README.zh-CN.md b/README.zh-CN.md index 53a95968b..c75e71a81 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -22,7 +22,7 @@ # SurfSense：面向竞争情报研究的 NotebookLM -SurfSense 是**面向 AI 智能体的开源竞争情报平台**，就像 NotebookLM，但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**，利用来自 **Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search 和开放网络**的实时数据，监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警，内置的知识库则让每一条发现都可搜索、可引用。 +SurfSense 是**面向 AI 智能体的开源竞争情报平台**，就像 NotebookLM，但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**，利用来自 **Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search、Indeed 和开放网络**的实时数据，监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警，内置的知识库则让每一条发现都可搜索、可引用。 > [!NOTE] > **📢 致我们的 NotebookLM 替代品用户** @@ -107,6 +107,7 @@ SurfSense 是**面向 AI 智能体的开源竞争情报平台**，就像 Noteboo | **TikTok** | 视频、评论、话题标签和主页，无需 Research API 审批 | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | 地点、评分和评论，用于本地竞争对手和潜在客户调研 | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | 实时搜索结果页，用于排名追踪和市场监控 | [Google Search API](https://www.surfsense.com/google-search) | +| **Indeed** | 公开职位信息，含薪资与完整职位描述，按搜索或公司抓取 | [Indeed Scraper API](https://www.surfsense.com/indeed) | | **Web Crawl** | 把开放网络上的任意页面转为干净、结构化的内容 | [Web Crawling API](https://www.surfsense.com/web-crawl) | | **外部 MCP 连接器** | 将任意 MCP 服务器接入你的智能体，Notion、Slack、Jira 等支持一键 OAuth | [External MCP Connectors](https://www.surfsense.com/external-mcp-connectors) | @@ -246,7 +247,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | 功能 | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **面向智能体的实时市场数据** | 无 | 通过 REST API 和 MCP 提供 Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search 和网页爬取连接器 | +| **面向智能体的实时市场数据** | 无 | 通过 REST API 和 MCP 提供 Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search、Indeed 和网页爬取连接器 | | **MCP 服务器** | 无 | 每个连接器都作为原生智能体工具暴露，还可自带 MCP 服务器并使用一键 OAuth 应用 | | **每个笔记本的来源数** | 50 个（免费版）至 600 个（Ultra 版，249.99 美元/月） | 无限制 | | **笔记本数量** | 100 个（免费版）至 500 个（付费档位） | 无限制 | diff --git a/docker/.env.example b/docker/.env.example index 2aa0806a8..7f4510320 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -457,6 +457,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 # TIKTOK_MICROS_PER_COMMENT=1500 +# INDEED_SCRAPE_MICROS_PER_JOB=3500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 3d2355460..44a3af872 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -297,6 +297,7 @@ MICROS_PER_PAGE=1000 # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 # TIKTOK_MICROS_PER_COMMENT=1500 +# INDEED_SCRAPE_MICROS_PER_JOB=3500 # Browser-listing retries when a feed is empty (profile feed is withheld from # flagged IPs; each retry draws a fresh rotating exit IP). Set to 1 for a static IP. # TIKTOK_LISTING_MAX_ATTEMPTS=3 diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts index 7d9320fb0..837c3eb44 100644 --- a/surfsense_web/lib/auth-utils.ts +++ b/surfsense_web/lib/auth-utils.ts @@ -44,6 +44,7 @@ const PUBLIC_ROUTE_PREFIXES = [ "/youtube", "/google-maps", "/google-search", + "/indeed", "/web-crawl", ]; diff --git a/surfsense_web/lib/connectors-marketing/google-maps.tsx b/surfsense_web/lib/connectors-marketing/google-maps.tsx index 03b1d247b..b09301ca6 100644 --- a/surfsense_web/lib/connectors-marketing/google-maps.tsx +++ b/surfsense_web/lib/connectors-marketing/google-maps.tsx @@ -306,6 +306,7 @@ export const googleMaps: ConnectorPageContent = { { label: "Instagram API", href: "/instagram" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, { label: "Read the docs", href: "/docs" }, ], diff --git a/surfsense_web/lib/connectors-marketing/google-search.tsx b/surfsense_web/lib/connectors-marketing/google-search.tsx index 7f436f9b0..a7b69c2ad 100644 --- a/surfsense_web/lib/connectors-marketing/google-search.tsx +++ b/surfsense_web/lib/connectors-marketing/google-search.tsx @@ -261,6 +261,7 @@ export const googleSearch: ConnectorPageContent = { { label: "Google Maps API", href: "/google-maps" }, { label: "Reddit API", href: "/reddit" }, { label: "Instagram API", href: "/instagram" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, { label: "Read the docs", href: "/docs" }, ], diff --git a/surfsense_web/lib/connectors-marketing/indeed.tsx b/surfsense_web/lib/connectors-marketing/indeed.tsx new file mode 100644 index 000000000..ce717f3cc --- /dev/null +++ b/surfsense_web/lib/connectors-marketing/indeed.tsx @@ -0,0 +1,327 @@ +import { IconBriefcase } from "@tabler/icons-react"; +import type { ConnectorPageContent } from "./types"; + +export const indeed: ConnectorPageContent = { + slug: "indeed", + name: "Indeed", + icon: IconBriefcase, + + metaTitle: "Indeed Scraper API for Jobs and Hiring Data | SurfSense", + metaDescription: + "Scrape public Indeed job postings with the SurfSense Indeed Scraper API: titles, companies, salaries, and full descriptions by search or company. No Indeed API. Start free.", + keywords: [ + "indeed scraper", + "indeed scraper api", + "indeed api", + "indeed jobs api", + "scrape indeed", + "indeed job scraper", + "job posting scraper", + "salary data api", + "hiring data api", + "indeed mcp", + "labor market data", + "recruiting data tool", + ], + + h1: "Indeed Scraper API for Job Postings and Hiring Data", + heroLede: + "The SurfSense Indeed API extracts public job postings, salaries, companies, and full descriptions by search query, company page, or job URL, without Indeed's official API. Give your AI agents a live feed of who is hiring, for what, at what pay, so you track the labor market as it moves.", + + transcript: { + prompt: "Find remote data analyst roles posted this week and what they pay", + toolCall: + 'indeed.scrape({ search_queries: ["data analyst"], location: "Remote",\n remote: "remote", from_days: 7, sort: "date", max_items: 30 })', + rows: [ + { + primary: "Senior Data Analyst · Acme Corp", + secondary: "Remote (US) · $120k–$145k/year · posted 2 days ago", + tag: "salary listed", + }, + { + primary: "Data Analyst, Growth · Globex", + secondary: "Remote · $95k–$110k/year · Indeed Apply", + tag: "buying signal", + }, + { + primary: "Marketing Data Analyst · Initech", + secondary: "Remote (US) · estimated $88k–$102k · posted today", + tag: "new today", + }, + ], + resultSummary: "30 jobs · 22 with salary · surfaced in 3.4s", + }, + + extractIntro: + "Every call returns structured job items. Point the API at a search query, an Indeed search or company page, or a single job URL, and set scrape_job_details for the full description per job.", + extractFields: [ + { + label: "Job", + description: "Title, job key, listing URL, apply URL, and whether Indeed Apply is enabled.", + }, + { + label: "Company", + description: "Company name, profile URL, star rating, and review count where available.", + }, + { + label: "Location", + description: "Formatted location, city, state, postal code, country, and remote or hybrid flags.", + }, + { + label: "Salary", + description: + "Pay text, min and max bounds, currency, period, and whether the figure is an Indeed estimate.", + }, + { + label: "Description", + description: + "Listing snippet by default; the full text and HTML description with scrape_job_details.", + }, + { + label: "Signals", + description: + "Job types, benefits, sponsored, urgently hiring, new, and expired flags, plus post age.", + }, + ], + + useCasesHeading: "What teams do with the Indeed API", + useCases: [ + { + title: "Competitor hiring intelligence", + description: + "Track what your competitors are hiring for and where. A spike in sales or ML roles is a roadmap signal months before it ships. Feed the stream to an agent that flags the moves that matter.", + }, + { + title: "Salary and compensation benchmarking", + description: + "Pull real posted salaries for a title in a location and benchmark your own bands against the live market, instead of a survey that is a year stale.", + }, + { + title: "Labor market and sector research", + description: + "Measure hiring demand for a role, skill, or sector over time. Turn thousands of postings into a demand index your analysts and clients can act on.", + }, + { + title: "Recruiting and lead sourcing", + description: + "Find companies actively hiring for a role and reach them while the need is hot. Job postings are a public, timely buying signal for staffing and B2B sales.", + }, + ], + + comparison: { + heading: "An Indeed API alternative built for agents", + intro: + "Indeed retired its public Publisher jobs API and gates data behind partner programs. If you cannot get access or need clean structured jobs now, here is how SurfSense compares.", + columnLabel: "DIY Indeed scraping", + rows: [ + { + feature: "Access", + official: "Publisher API retired; partner-gated and approval-only", + surfsense: "One API key; scrape public postings without an approval process", + }, + { + feature: "Anti-bot", + official: "You fight Cloudflare, fingerprinting, and CAPTCHAs yourself", + surfsense: "Warmed, rotated sessions managed for you; no proxy plumbing", + }, + { + feature: "Pricing", + official: "Proxy, browser, and maintenance costs you own", + surfsense: "Pay per job returned, with a free tier to start", + }, + { + feature: "Descriptions", + official: "Extra page fetch and parsing you build and maintain", + surfsense: "Full description per job with one scrape_job_details flag", + }, + { + feature: "Agent-ready", + official: "No; you build the harness yourself", + surfsense: "MCP server exposes indeed.scrape as a native tool", + }, + ], + }, + + api: { + platform: "indeed", + verb: "scrape", + mcpTool: "indeed.scrape", + requestBody: { + search_queries: ["data analyst"], + location: "Remote", + remote: "remote", + from_days: 7, + sort: "date", + max_items: 30, + }, + }, + + schema: { + requestNote: + "Provide at least one source: urls or search_queries. Up to 20 sources per call.", + request: [ + { + name: "urls", + type: "string[]", + defaultValue: "[]", + description: + "Indeed URLs: a search page (/jobs?q=&l=), a company jobs page (/cmp//jobs), or a single job (/viewjob?jk=...). Max 20.", + }, + { + name: "search_queries", + type: "string[]", + defaultValue: "[]", + description: + "Job search terms. Each returns up to max_items_per_query results, shaped by the filters below. Max 20.", + }, + { + name: "country", + type: "string", + defaultValue: '"us"', + description: "Country code selecting the Indeed domain, e.g. 'us', 'gb', 'de'.", + }, + { + name: "location", + type: "string", + description: "Where to search, e.g. 'Remote', 'New York, NY'.", + }, + { + name: "radius", + type: "integer", + description: "Search radius in miles or km around location.", + }, + { + name: "job_type", + type: "string", + description: "Employment type: fulltime, parttime, contract, internship, and more.", + }, + { + name: "level", + type: "string", + description: "Experience level: entry_level, mid_level, or senior_level.", + }, + { + name: "remote", + type: "string", + description: "Work model filter: remote or hybrid.", + }, + { + name: "from_days", + type: "integer", + description: "Only return jobs posted within the last N days.", + }, + { + name: "sort", + type: "string", + defaultValue: '"relevance"', + description: "Result ordering: relevance or date.", + }, + { + name: "scrape_job_details", + type: "boolean", + defaultValue: "false", + description: + "Fetch each job's detail page for the full description. Slower: one extra page load per job.", + }, + { + name: "max_items", + type: "integer", + defaultValue: "25", + description: "Max total jobs to return across all sources. 1 to 100.", + }, + { + name: "max_items_per_query", + type: "integer", + defaultValue: "25", + description: "Max jobs to pull per search or company target.", + }, + ], + responseNote: + "The response is { items: [...] } with one flat item per job. Fields Indeed omits are null. One returned job is one billable unit.", + response: [ + { + name: "jobKey / jobUrl / applyUrl", + type: "string", + description: "Indeed job key, listing URL, and third-party apply URL.", + }, + { + name: "title", + type: "string", + description: "The job title as posted.", + }, + { + name: "company / companyUrl", + type: "string", + description: "Company name and its Indeed profile URL.", + }, + { + name: "companyRating / companyReviewCount", + type: "number / integer", + description: "Employer star rating and number of reviews, where Indeed shows them.", + }, + { + name: "formattedLocation / isRemote / remoteType", + type: "string / boolean", + description: "Location string plus remote and hybrid flags.", + }, + { + name: "salary", + type: "object", + description: + "salaryText, salaryMin, salaryMax, currency, period, and isEstimated when the pay is an Indeed estimate.", + }, + { + name: "jobTypes / benefits", + type: "string[]", + description: "Employment types and listed benefits parsed from the posting.", + }, + { + name: "descriptionText / descriptionHtml", + type: "string", + description: "Snippet by default; the full description when scrape_job_details is set.", + }, + { + name: "sponsored / urgentlyHiring / isNew / expired", + type: "boolean", + description: "Listing flags for ranking and filtering.", + }, + { + name: "age / datePublished / scrapedAt", + type: "string", + description: "Relative post age, ISO publish date, and when the job was scraped.", + }, + ], + }, + + faq: [ + { + question: "Is scraping Indeed legal?", + answer: + "SurfSense reads only public Indeed job postings, the same listings any logged-out visitor can see. It never logs in and cannot access private or applicant data. As always, review Indeed's terms and your own compliance needs before you run at scale.", + }, + { + question: "Does Indeed have an official jobs API?", + answer: + "Indeed retired its public Publisher jobs API and now gates job data behind partner and approval programs. SurfSense is an independent alternative: you call one API, or add the MCP server to your agent, and get structured public postings back.", + }, + { + question: "Can I get the full job description?", + answer: + "Yes. By default each job returns the listing snippet, which is fast. Set scrape_job_details to true and SurfSense fetches each job's detail page for the full description text and HTML, at the cost of one extra page load per job.", + }, + { + question: "What are the rate limits?", + answer: + "Each call returns up to 100 jobs across all sources, with up to 20 URLs or search queries per request. SurfSense manages the anti-bot request budget for you, so you scale reads without running proxies or a headless browser yourself.", + }, + ], + + related: [ + { label: "Reddit API", href: "/reddit" }, + { label: "YouTube API", href: "/youtube" }, + { label: "Google Maps API", href: "/google-maps" }, + { label: "SERP API", href: "/google-search" }, + { label: "Web Crawl API", href: "/web-crawl" }, + { label: "SurfSense MCP Server", href: "/mcp-server" }, + ], +}; diff --git a/surfsense_web/lib/connectors-marketing/index.ts b/surfsense_web/lib/connectors-marketing/index.ts index 826a1f261..be3948398 100644 --- a/surfsense_web/lib/connectors-marketing/index.ts +++ b/surfsense_web/lib/connectors-marketing/index.ts @@ -1,5 +1,6 @@ import { googleMaps } from "./google-maps"; import { googleSearch } from "./google-search"; +import { indeed } from "./indeed"; import { instagram } from "./instagram"; import { reddit } from "./reddit"; import { tiktok } from "./tiktok"; @@ -17,6 +18,7 @@ const CONNECTOR_LIST: ConnectorPageContent[] = [ tiktok, googleMaps, googleSearch, + indeed, webCrawl, ]; diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx index 5422a0c90..999be0de8 100644 --- a/surfsense_web/lib/connectors-marketing/instagram.tsx +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -288,6 +288,7 @@ export const instagram: ConnectorPageContent = { { label: "Reddit API", href: "/reddit" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, ], }; diff --git a/surfsense_web/lib/connectors-marketing/reddit.tsx b/surfsense_web/lib/connectors-marketing/reddit.tsx index 2a930afe9..c76c9984f 100644 --- a/surfsense_web/lib/connectors-marketing/reddit.tsx +++ b/surfsense_web/lib/connectors-marketing/reddit.tsx @@ -315,6 +315,7 @@ export const reddit: ConnectorPageContent = { { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, ], }; diff --git a/surfsense_web/lib/connectors-marketing/tiktok.tsx b/surfsense_web/lib/connectors-marketing/tiktok.tsx index 6063fd1ae..9a765ef0e 100644 --- a/surfsense_web/lib/connectors-marketing/tiktok.tsx +++ b/surfsense_web/lib/connectors-marketing/tiktok.tsx @@ -284,6 +284,7 @@ export const tiktok: ConnectorPageContent = { { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, ], }; diff --git a/surfsense_web/lib/connectors-marketing/web-crawl.tsx b/surfsense_web/lib/connectors-marketing/web-crawl.tsx index 1cb4d5dac..1684a61e2 100644 --- a/surfsense_web/lib/connectors-marketing/web-crawl.tsx +++ b/surfsense_web/lib/connectors-marketing/web-crawl.tsx @@ -283,6 +283,7 @@ export const webCrawl: ConnectorPageContent = { { label: "Google Maps API", href: "/google-maps" }, { label: "Reddit API", href: "/reddit" }, { label: "Instagram API", href: "/instagram" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, { label: "Read the docs", href: "/docs" }, ], diff --git a/surfsense_web/lib/connectors-marketing/youtube.tsx b/surfsense_web/lib/connectors-marketing/youtube.tsx index fceaee940..431c78357 100644 --- a/surfsense_web/lib/connectors-marketing/youtube.tsx +++ b/surfsense_web/lib/connectors-marketing/youtube.tsx @@ -275,6 +275,7 @@ export const youtube: ConnectorPageContent = { { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, ], }; diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index 456e52008..a8825e52a 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -2,6 +2,7 @@ import type { ComponentType } from "react"; import { GoogleMapsIcon, GoogleSearchIcon, + IndeedIcon, InstagramIcon, RedditIcon, TikTokIcon, @@ -89,6 +90,12 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ icon: GoogleSearchIcon, verbs: [{ name: "google_search.scrape", verb: "scrape", label: "Scrape" }], }, + { + id: "indeed", + label: "Indeed", + icon: IndeedIcon, + verbs: [{ name: "indeed.scrape", verb: "scrape", label: "Scrape" }], + }, { id: "web", label: "Web", diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx index c1f61978b..5bae99d1b 100644 --- a/surfsense_web/lib/playground/platform-icons.tsx +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -28,4 +28,5 @@ export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram") export const TikTokIcon = brandIcon("/connectors/tiktok.svg", "TikTok"); export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps"); export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search"); +export const IndeedIcon = brandIcon("/connectors/indeed.svg", "Indeed"); export const WebIcon = brandIcon("/connectors/web.svg", "Web"); diff --git a/surfsense_web/public/connectors/indeed.svg b/surfsense_web/public/connectors/indeed.svg new file mode 100644 index 000000000..39d792bd6 --- /dev/null +++ b/surfsense_web/public/connectors/indeed.svg @@ -0,0 +1,5 @@ + + + + + From 2a3d750cbeec21a5336e396fabbe37835f202c4d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 19:08:50 +0200 Subject: [PATCH 51/56] fix: use official indeed brand mark for connector icon --- surfsense_web/public/connectors/indeed.svg | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/surfsense_web/public/connectors/indeed.svg b/surfsense_web/public/connectors/indeed.svg index 39d792bd6..9265a1934 100644 --- a/surfsense_web/public/connectors/indeed.svg +++ b/surfsense_web/public/connectors/indeed.svg @@ -1,5 +1 @@ - - - - - +Indeed From b53f00231d24b653bd969c63684c015b25e244d2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 19:57:19 +0200 Subject: [PATCH 52/56] fix: parse indeed /viewjob detail from window._rootProps --- .../platforms/indeed_jobs/parsers.py | 32 ++++++++++++++++--- .../platforms/indeed_jobs/test_parsers.py | 20 ++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py index b61b13e6a..87824e8c7 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py @@ -22,7 +22,10 @@ _DEFAULT_BASE = "https://www.indeed.com" _JOBCARDS_ANCHOR = 'window.mosaic.providerData["mosaic-provider-jobcards"]=' -# A /viewjob page assigns the full posting to this global. +# A /viewjob page carries the posting model in ``window._rootProps`` (JSON, +# under ``preloadedVJData``); older pages inlined it as ``window._initialData``. +_ROOT_PROPS_ANCHOR = "window._rootProps" +_ROOT_PROPS_KEY = "preloadedVJData" _INITIAL_DATA_ANCHOR = "window._initialData" # Indeed's extractedSalary.type -> our SalaryPeriod. @@ -82,14 +85,14 @@ def extract_jobcards_blob(html: str) -> dict | None: return data if isinstance(data, dict) else None -def extract_initial_data(html: str) -> dict | None: - """Decode the ``window._initialData`` assignment on a /viewjob page.""" +def _decode_assignment(html: str, anchor: str) -> dict | None: + """Decode the balanced JSON object assigned after ``anchor``, or ``None``.""" import json - idx = html.find(_INITIAL_DATA_ANCHOR) + idx = html.find(anchor) if idx == -1: return None - brace = html.find("{", idx + len(_INITIAL_DATA_ANCHOR)) + brace = html.find("{", idx + len(anchor)) if brace == -1: return None blob = _brace_match(html, brace) @@ -102,6 +105,25 @@ def extract_initial_data(html: str) -> dict | None: return data if isinstance(data, dict) else None +def extract_initial_data(html: str) -> dict | None: + """Return a /viewjob posting model rooted at ``jobInfoWrapperModel``. + + Prefers ``window._rootProps`` (JSON) unwrapped at ``preloadedVJData``; falls + back to a legacy inline ``window._initialData`` blob. ``window._initialData`` + is now a JS object literal that references other globals, so it is not JSON + and is skipped when the JSON parse fails. + """ + root = _decode_assignment(html, _ROOT_PROPS_ANCHOR) + if isinstance(root, dict): + vj = root.get(_ROOT_PROPS_KEY) + if isinstance(vj, dict) and vj.get("jobInfoWrapperModel"): + return vj + legacy = _decode_assignment(html, _INITIAL_DATA_ANCHOR) + if isinstance(legacy, dict) and legacy.get("jobInfoWrapperModel"): + return legacy + return None + + def job_results(blob: dict | None) -> list[dict[str, Any]]: """Return the raw job records from a decoded blob.""" if not isinstance(blob, dict): diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py index 38a6544b6..43fed6948 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py @@ -160,6 +160,26 @@ def test_parse_job_detail_extracts_description_and_fields(): assert (sal["salaryMin"], sal["salaryMax"], sal["period"]) == (60000, 90000, "year") +def test_parse_job_detail_reads_rootprops_shape(): + # Live /viewjob assigns window._rootProps (JSON) with the model under + # preloadedVJData; window._initialData is now a non-JSON JS literal that + # references other globals and must be skipped, not misparsed. + html = ( + "" + '' + ) + detail = parse_job_detail(html) + assert detail["title"] == "Data Analyst" + assert detail["company"] == "Acme" + assert detail["descriptionHtml"] == "

Full JD text

" + assert detail["descriptionText"] == "Full JD text" + + def test_parse_job_detail_omits_blank_fields(): # A page with a header but no salary/description must not emit those keys, # so a merge won't clobber listing values with blanks. From b171002d15de8e4fd57d0ba1fe9b2bc0a920c358 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 19:57:19 +0200 Subject: [PATCH 53/56] test: add live indeed scraper e2e harness --- .../scripts/e2e_indeed_scraper.py | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 surfsense_backend/scripts/e2e_indeed_scraper.py diff --git a/surfsense_backend/scripts/e2e_indeed_scraper.py b/surfsense_backend/scripts/e2e_indeed_scraper.py new file mode 100644 index 000000000..24143af4f --- /dev/null +++ b/surfsense_backend/scripts/e2e_indeed_scraper.py @@ -0,0 +1,149 @@ +"""Manual functional e2e for the Indeed scraper (app/proprietary/platforms/indeed_jobs). + +Run from the backend directory: + cd surfsense_backend + uv run python scripts/e2e_indeed_scraper.py + # or: .venv/bin/python scripts/e2e_indeed_scraper.py + +This is NOT a pytest test (it needs live network + a residential/custom proxy). +All steps share one warmed browser session: + + Step 0 — go/no-go probe: open a session, fetch a live search page, and assert + its embedded job-cards blob parses into results. If this fails the whole + approach is blocked on this IP/proxy — later steps are skipped. + Step 1 — search query -> job items; keep one discovered /viewjob URL. + Step 2 — scrape a search URL via startUrls. + Step 3 — scrape the discovered /viewjob URL and assert a full description. + Step 4 — scrape_job_details enrichment: a query with detail pages fetched. +""" + +import asyncio +import sys +from pathlib import Path + +from dotenv import load_dotenv + +# --- bootstrap: load .env and put the backend root on sys.path before app.* --- +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +from app.proprietary.platforms.indeed_jobs import ( # noqa: E402 + IndeedScrapeInput, + scrape_indeed, +) +from app.proprietary.platforms.indeed_jobs.fetch import open_session # noqa: E402 +from app.proprietary.platforms.indeed_jobs.parsers import ( # noqa: E402 + extract_jobcards_blob, + job_results, +) +from app.proprietary.platforms.indeed_jobs.url_resolver import ( # noqa: E402 + build_search_url, +) + +_QUERY = "data analyst" +_LOCATION = "Remote" + + +def _hr(title: str) -> None: + print(f"\n{'=' * 70}\n{title}\n{'=' * 70}") + + +def _check(label: str, ok: bool, detail: str = "") -> bool: + print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}") + return ok + + +async def step0_probe(sess, state: dict) -> bool: + _hr("STEP 0 — go/no-go: warmed session + parseable job cards") + url = build_search_url(_QUERY, location=_LOCATION) + html = await sess.fetch_html(url) + raws = job_results(extract_jobcards_blob(html)) + print(f" {url} -> job_cards={len(raws)}") + return _check("search page parsed job cards", len(raws) > 0, f"{len(raws)} cards") + + +async def step1_search(sess, state: dict) -> bool: + _hr("STEP 1 — search query -> items") + items = await scrape_indeed( + IndeedScrapeInput(queries=[_QUERY], location=_LOCATION, maxItems=5), + limit=5, + session=sess, + ) + for it in items[:5]: + print(f" - {it.get('jobKey')} | {it.get('title')} @ {it.get('company')}") + state["job_url"] = next( + (it["jobUrl"] for it in items if it.get("jobUrl")), None + ) + return _check("search returned jobs", len(items) > 0, f"{len(items)} jobs") + + +async def step2_search_url(sess, state: dict) -> bool: + _hr("STEP 2 — scrape a search URL (startUrls)") + url = build_search_url(_QUERY, location=_LOCATION) + items = await scrape_indeed( + IndeedScrapeInput(startUrls=[{"url": url}], maxItems=5), + limit=5, + session=sess, + ) + return _check("search URL returned jobs", len(items) > 0, f"{len(items)} jobs") + + +async def step3_viewjob(sess, state: dict) -> bool: + _hr("STEP 3 — scrape a discovered /viewjob URL (full description)") + url = state.get("job_url") + if not url: + return _check("had a discovered job URL", False) + items = await scrape_indeed( + IndeedScrapeInput(startUrls=[{"url": url}], maxItems=1), + limit=1, + session=sess, + ) + desc = items[0].get("descriptionText") if items else None + print(f" url={url}\n description_chars={len(desc or '')}") + return _check("viewjob returned a description", bool(desc), url) + + +async def step4_enrich(sess, state: dict) -> bool: + _hr("STEP 4 — search with scrape_job_details enrichment") + items = await scrape_indeed( + IndeedScrapeInput( + queries=[_QUERY], + location=_LOCATION, + maxItems=3, + scrapeJobDetails=True, + ), + limit=3, + session=sess, + ) + # descriptionHtml is set only by the detail page; listings never carry it, + # so it proves enrichment actually merged the /viewjob model. + enriched = [i for i in items if i.get("descriptionHtml")] + return _check( + "enriched items carry full descriptionHtml", + bool(enriched), + f"{len(enriched)}/{len(items)} enriched", + ) + + +async def main() -> int: + state: dict = {} + async with open_session() as sess: + results = [await step0_probe(sess, state)] + if not results[-1]: + print("\nprobe failed — Indeed is blocking this IP/proxy. Aborting.") + return 1 + results.append(await step1_search(sess, state)) + results.append(await step2_search_url(sess, state)) + results.append(await step3_viewjob(sess, state)) + results.append(await step4_enrich(sess, state)) + _hr("SUMMARY") + print(f" {sum(results)}/{len(results)} steps passed") + return 0 if all(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) From 6180b969129ac4b878dabb0fca22554c3746954e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 20:24:22 +0200 Subject: [PATCH 54/56] fix: fail fast on gated indeed pagination, keep collected pages --- .../platforms/indeed_jobs/fetch.py | 12 +++++--- .../platforms/indeed_jobs/scraper.py | 27 ++++++++++++++--- .../indeed_jobs/test_fetch_resilience.py | 10 +++++++ .../platforms/indeed_jobs/test_scraper.py | 29 +++++++++++++++++-- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py index 9f913a579..c3be9c2a8 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py @@ -140,14 +140,18 @@ class IndeedSession: await self._timed_fetch(f"https://{domain}/", google_search=True) self._warmed.add(domain) - async def fetch_html(self, url: str) -> str: + async def fetch_html(self, url: str, *, max_rotations: int | None = None) -> str: """Return a search/company/job page's HTML through the warmed session. Rotates the IP and re-warms on a security-wall bounce or timeout; raises - :class:`IndeedAccessBlockedError` once every rotation is exhausted. + :class:`IndeedAccessBlockedError` once rotations are exhausted. ``max_rotations`` + overrides the default budget: pass ``0`` to fail fast on a systematically + gated page (e.g. anonymous pagination) instead of burning rotations on a + block no fresh IP will clear. """ if self._session is None: await self.start() + budget = _MAX_ROTATIONS if max_rotations is None else max_rotations domain = urlparse(url).hostname or "www.indeed.com" attempt = 0 while True: @@ -163,9 +167,9 @@ class IndeedSession: except Exception as e: logger.warning("[indeed] fetch failed on %s: %s", url, e) - if attempt >= _MAX_ROTATIONS: + if attempt >= budget: raise IndeedAccessBlockedError( - f"Indeed refused {url} on {attempt} rotated IPs" + f"Indeed refused {url} after {attempt + 1} attempt(s)" ) attempt += 1 await self._rotate() diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py index 0c9fcfb16..01371d281 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py @@ -2,7 +2,9 @@ :func:`iter_indeed` streams flat job items from one warmed browser session: ``startUrls`` (search/company pages) or ``queries`` are paged by the ``start`` -offset, deduped by ``jobKey``, and stopped on the first page that adds nothing. +offset, deduped by ``jobKey``, and stopped on the first page that adds nothing +or that Indeed gates (anonymous pagination is bounced past page 1, so a gated +page ends that target and keeps the pages already yielded). :func:`scrape_indeed` collects the stream under a caller ``limit``. Targets run sequentially on a single session — a browser per target would cost @@ -16,7 +18,7 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse -from .fetch import IndeedSession, now_iso, open_session +from .fetch import IndeedAccessBlockedError, IndeedSession, now_iso, open_session from .parsers import ( extract_jobcards_blob, job_results, @@ -60,7 +62,21 @@ async def _paginate( seen: set[str] = set() emitted = 0 for page in range(_MAX_PAGES): - html = await session.fetch_html(_with_start(first_url, page * _PAGE_STEP)) + try: + # Rotate to establish first-page access, but fail fast on deeper + # pages: Indeed gates anonymous pagination to secure.indeed.com, a + # block no fresh IP clears — retrying it only burns the time budget. + # ponytail: deeper pages are dropped rather than retried; lift the + # cap with an async job if full-depth pagination is ever needed. + html = await session.fetch_html( + _with_start(first_url, page * _PAGE_STEP), + max_rotations=None if page == 0 else 0, + ) + except IndeedAccessBlockedError: + if page == 0: + raise # never got in on this target; surface the block + logger.info("[indeed] pagination gated at start=%d; stopping", page * _PAGE_STEP) + return # keep the pages already yielded raws = job_results(extract_jobcards_blob(html)) if not raws: return @@ -127,7 +143,10 @@ async def _enrich(session: IndeedSession, item: dict[str, Any], base_url: str) - if not isinstance(job_url, str): return try: - detail = parse_job_detail(await session.fetch_html(job_url), base_url=base_url) + # Fail fast: enrichment is best-effort, so a gated detail page must not + # rotate IPs and eat the run's time budget for one job's description. + html = await session.fetch_html(job_url, max_rotations=0) + detail = parse_job_detail(html, base_url=base_url) except Exception as exc: logger.warning("[indeed] detail fetch failed for %s: %s", job_url, exc) return diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py index 4eae65f84..258f5154d 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py @@ -85,6 +85,16 @@ async def test_raises_after_exhausting_rotations(): assert session.rotations == 3 +@pytest.mark.asyncio +async def test_max_rotations_zero_fails_fast(): + # A gated page (pagination) must raise on the first block without rotating. + ctrl = _Controller(["BLOCK", "OK"]) + session = IndeedSession(ctrl.factory) + with pytest.raises(IndeedAccessBlockedError): + await session.fetch_html(_URL, max_rotations=0) + assert session.rotations == 0 + + @pytest.mark.asyncio async def test_warms_domain_once_without_rotation(): ctrl = _Controller(["OK", "OK"]) diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py index e5ea23452..acf52296b 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py @@ -7,6 +7,7 @@ from urllib.parse import parse_qs, urlparse import pytest +from app.proprietary.platforms.indeed_jobs.fetch import IndeedAccessBlockedError from app.proprietary.platforms.indeed_jobs.schemas import IndeedScrapeInput from app.proprietary.platforms.indeed_jobs.scraper import iter_indeed, scrape_indeed @@ -38,16 +39,21 @@ def _detail_html(job_key: str) -> str: class _FakeSession: """Returns per-``start`` search pages (or a /viewjob detail) and records URLs.""" - def __init__(self, pages: dict[int, list[str]]) -> None: + def __init__( + self, pages: dict[int, list[str]], blocked_starts: set[int] | None = None + ) -> None: self._pages = pages + self._blocked = blocked_starts or set() self.fetched: list[str] = [] - async def fetch_html(self, url: str) -> str: + async def fetch_html(self, url: str, *, max_rotations: int | None = None) -> str: self.fetched.append(url) query = parse_qs(urlparse(url).query) if "/viewjob" in url: return _detail_html(query.get("jk", [""])[0]) start = int(query.get("start", ["0"])[0]) + if start in self._blocked: + raise IndeedAccessBlockedError(f"gated at start={start}") return _page_html(self._pages.get(start, [])) @@ -77,6 +83,25 @@ async def test_stops_when_a_page_is_all_duplicates(): } +@pytest.mark.asyncio +async def test_pagination_gate_keeps_earlier_pages(): + # Indeed gates anonymous pagination: page 0 serves, start=10 is blocked. + # The already-yielded page-0 items must survive; paging just stops. + session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}, blocked_starts={10}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + + +@pytest.mark.asyncio +async def test_first_page_block_propagates(): + # A block on the very first page yielded nothing, so it surfaces as an error. + session = _FakeSession({}, blocked_starts={0}) + with pytest.raises(IndeedAccessBlockedError): + await _collect(IndeedScrapeInput(queries=["dev"]), session) + + @pytest.mark.asyncio async def test_respects_max_items_per_query(): session = _FakeSession({0: ["k1", "k2", "k3", "k4"]}) From c485fe4c43fc900ab75b16e49efc2dca5aa7fd3f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 20:54:37 +0200 Subject: [PATCH 55/56] fix: scrape indeed first page only, steer subagent to one query --- .../builtins/indeed/system_prompt.md | 7 +- .../platforms/indeed_jobs/scraper.py | 86 ++++++------------- .../platforms/indeed_jobs/test_scraper.py | 30 ++----- 3 files changed, 36 insertions(+), 87 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md index b034b700a..36eb855d2 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md @@ -14,10 +14,9 @@ Answer the delegated question from live Indeed job data gathered with your verb, - Finding jobs for a role: call `indeed_scrape` with `search_queries`; narrow with `location`, `country`, `job_type`, `level`, `remote`, `radius`, and `from_days`. - Scraping a specific Indeed URL: pass a search, company jobs, or single job URL in `urls`. - Full descriptions: set `scrape_job_details=true` to fetch each job's detail page (slower: one extra load per job). Leave it false when the listing snippet is enough. -- Controlling volume: use `max_items` for the total cap and `max_items_per_query` per search. -- Requested counts: `max_items` defaults to only 25 — when the task asks for N jobs, set `max_items` and `max_items_per_query` above N (with headroom for off-topic hits). A call that caps below the target can never satisfy it. -- Under-delivery: if the first call returns fewer on-topic results than requested, broaden it yourself — more query phrasings, wider `radius`, drop restrictive filters, larger `from_days` — before settling. Return `status=partial` only after the broadened attempt, never after a single narrow call. -- Batch multiple search terms into one call rather than many single-term calls. +- Cost model: Indeed is latency-bound. A cold session spends minutes solving Cloudflare, and each query returns only its first page (~15 jobs) — anonymous pagination is gated, so `max_items` above ~15/query buys nothing. Every extra phrasing adds wait, not depth. +- Default to ONE focused query. Only add phrasings when the role genuinely needs variety, and then put at most 2–3 in a SINGLE call's `search_queries` (they reuse one warmed session); never make separate `indeed_scrape` calls, which each pay the cold-start cost. +- Prefer returning the first page's on-topic hits promptly over exhaustive coverage. If a single call under-delivers against a large requested N, return `status=partial` noting the ~one-page-per-query ceiling — do not chase it with more calls. - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, salary/rank changes). diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py index 01371d281..68b70870a 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py @@ -1,14 +1,8 @@ """Orchestrator for the Indeed scraper. -:func:`iter_indeed` streams flat job items from one warmed browser session: -``startUrls`` (search/company pages) or ``queries`` are paged by the ``start`` -offset, deduped by ``jobKey``, and stopped on the first page that adds nothing -or that Indeed gates (anonymous pagination is bounced past page 1, so a gated -page ends that target and keeps the pages already yielded). -:func:`scrape_indeed` collects the stream under a caller ``limit``. - -Targets run sequentially on a single session — a browser per target would cost -far more than the sequential paging it would save. +:func:`iter_indeed` streams deduped job items from one warmed session; each +search/company target contributes its first page. :func:`scrape_indeed` collects +the stream under a caller ``limit``. Targets run sequentially to reuse the session. """ from __future__ import annotations @@ -16,9 +10,9 @@ from __future__ import annotations import logging from collections.abc import AsyncIterator from typing import Any -from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse +from urllib.parse import urlparse -from .fetch import IndeedAccessBlockedError, IndeedSession, now_iso, open_session +from .fetch import IndeedSession, now_iso, open_session from .parsers import ( extract_jobcards_blob, job_results, @@ -32,68 +26,36 @@ logger = logging.getLogger(__name__) __all__ = ["iter_indeed", "scrape_indeed"] -# Indeed's search pagination increments the ``start`` offset by 10. -_PAGE_STEP = 10 -# Indeed stops serving useful results past ~1000; cap pages well before that. -_MAX_PAGES = 10 - def _emit(partial: dict[str, Any]) -> dict[str, Any]: """Stamp ``scrapedAt`` and normalize through the output model.""" return IndeedItem(**{**partial, "scrapedAt": now_iso()}).to_output() -def _with_start(url: str, start: int) -> str: - """Return ``url`` with its ``start`` query param set (removed when 0).""" - parsed = urlparse(url) - params = [(k, v) for k, v in parse_qsl(parsed.query) if k != "start"] - if start: - params.append(("start", str(start))) - return urlunparse(parsed._replace(query=urlencode(params))) - - -async def _paginate( - session: IndeedSession, first_url: str, *, domain: str, max_items: int +async def _search_items( + session: IndeedSession, url: str, *, domain: str, max_items: int ) -> AsyncIterator[dict[str, Any]]: - """Yield items across pages of one search/company URL, deduped by ``jobKey``.""" + """Yield deduped job cards from one search/company page. + + ponytail: caps a query at its first page (~15 jobs) — anonymous Indeed gates + ``start>=10``; deeper depth needs an authenticated session or Indeed's API. + """ if max_items <= 0: return base_url = f"https://{domain}" + html = await session.fetch_html(url) seen: set[str] = set() emitted = 0 - for page in range(_MAX_PAGES): - try: - # Rotate to establish first-page access, but fail fast on deeper - # pages: Indeed gates anonymous pagination to secure.indeed.com, a - # block no fresh IP clears — retrying it only burns the time budget. - # ponytail: deeper pages are dropped rather than retried; lift the - # cap with an async job if full-depth pagination is ever needed. - html = await session.fetch_html( - _with_start(first_url, page * _PAGE_STEP), - max_rotations=None if page == 0 else 0, - ) - except IndeedAccessBlockedError: - if page == 0: - raise # never got in on this target; surface the block - logger.info("[indeed] pagination gated at start=%d; stopping", page * _PAGE_STEP) - return # keep the pages already yielded - raws = job_results(extract_jobcards_blob(html)) - if not raws: - return - added = 0 - for raw in raws: - item = parse_job(raw, base_url=base_url) - job_key = item.get("jobKey") - if isinstance(job_key, str): - if job_key in seen: - continue - seen.add(job_key) - yield _emit(item) - emitted += 1 - added += 1 - if emitted >= max_items: - return - if added == 0: # a whole page of duplicates: end of useful results + for raw in job_results(extract_jobcards_blob(html)): + item = parse_job(raw, base_url=base_url) + job_key = item.get("jobKey") + if isinstance(job_key, str): + if job_key in seen: + continue + seen.add(job_key) + yield _emit(item) + emitted += 1 + if emitted >= max_items: return @@ -175,7 +137,7 @@ async def iter_indeed( if item is not None: yield item continue - async for item in _paginate( + async for item in _search_items( session, url, domain=domain, max_items=input_model.maxItemsPerQuery ): job_key = item.get("jobKey") diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py index acf52296b..1d815ce98 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py @@ -62,41 +62,29 @@ async def _collect(input_model, session) -> list[dict]: @pytest.mark.asyncio -async def test_paginates_and_dedupes_across_pages(): - session = _FakeSession({0: ["k1", "k2", "k3"], 10: ["k3", "k4"], 20: []}) +async def test_dedupes_within_page(): + session = _FakeSession({0: ["k1", "k2", "k2", "k3"]}) items = await _collect( IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session ) - assert [i["jobKey"] for i in items] == ["k1", "k2", "k3", "k4"] + assert [i["jobKey"] for i in items] == ["k1", "k2", "k3"] assert all(i["scrapedAt"] for i in items) # stamped by the orchestrator @pytest.mark.asyncio -async def test_stops_when_a_page_is_all_duplicates(): - session = _FakeSession({0: ["k1", "k2"], 10: ["k1", "k2"], 20: ["k9"]}) +async def test_does_not_fetch_deeper_pages(): + # First page only; ``start>=10`` must never be requested. + session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}) items = await _collect( IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session ) assert [i["jobKey"] for i in items] == ["k1", "k2"] - assert 20 not in { - int(parse_qs(urlparse(u).query).get("start", ["0"])[0]) for u in session.fetched - } + assert all("start=" not in u for u in session.fetched) @pytest.mark.asyncio -async def test_pagination_gate_keeps_earlier_pages(): - # Indeed gates anonymous pagination: page 0 serves, start=10 is blocked. - # The already-yielded page-0 items must survive; paging just stops. - session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}, blocked_starts={10}) - items = await _collect( - IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session - ) - assert [i["jobKey"] for i in items] == ["k1", "k2"] - - -@pytest.mark.asyncio -async def test_first_page_block_propagates(): - # A block on the very first page yielded nothing, so it surfaces as an error. +async def test_page_block_propagates(): + # Nothing yielded before the block, so it surfaces as an error. session = _FakeSession({}, blocked_starts={0}) with pytest.raises(IndeedAccessBlockedError): await _collect(IndeedScrapeInput(queries=["dev"]), session) From 5cc05d90f0de4e4c093f0917381d726817f7dc4e Mon Sep 17 00:00:00 2001 From: Sanjay Santhanam <51058514+Sanjays2402@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:46:31 -0700 Subject: [PATCH 56/56] fix: avoid blocking audio writes TTS responses were written synchronously from the async presentation pipeline, blocking concurrent requests. Offload byte writes to a worker thread and add a regression test that keeps the event loop responsive. --- .../app/agents/video_presentation/nodes.py | 4 +-- surfsense_backend/app/utils/file_io.py | 7 +++++ .../agents/video_presentation/test_file_io.py | 29 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/app/utils/file_io.py create mode 100644 surfsense_backend/tests/unit/agents/video_presentation/test_file_io.py diff --git a/surfsense_backend/app/agents/video_presentation/nodes.py b/surfsense_backend/app/agents/video_presentation/nodes.py index dca89059f..e52d1f4ce 100644 --- a/surfsense_backend/app/agents/video_presentation/nodes.py +++ b/surfsense_backend/app/agents/video_presentation/nodes.py @@ -17,6 +17,7 @@ from app.config import config as app_config from app.services.kokoro_tts_service import get_kokoro_tts_service from app.services.llm_service import get_agent_llm from app.utils.content_utils import extract_text_content, strip_markdown_fences +from app.utils.file_io import write_bytes from .configuration import Configuration from .prompts import ( @@ -137,8 +138,7 @@ async def create_slide_audio(state: State, config: RunnableConfig) -> dict[str, kwargs["api_base"] = app_config.TTS_SERVICE_API_BASE response = await aspeech(**kwargs) - with open(chunk_path, "wb") as f: - f.write(response.content) + await write_bytes(chunk_path, response.content) return chunk_path diff --git a/surfsense_backend/app/utils/file_io.py b/surfsense_backend/app/utils/file_io.py new file mode 100644 index 000000000..2e5700955 --- /dev/null +++ b/surfsense_backend/app/utils/file_io.py @@ -0,0 +1,7 @@ +import asyncio +from pathlib import Path + + +async def write_bytes(path: str, content: bytes) -> None: + """Write bytes without blocking the event loop.""" + await asyncio.to_thread(Path(path).write_bytes, content) diff --git a/surfsense_backend/tests/unit/agents/video_presentation/test_file_io.py b/surfsense_backend/tests/unit/agents/video_presentation/test_file_io.py new file mode 100644 index 000000000..c85cef19c --- /dev/null +++ b/surfsense_backend/tests/unit/agents/video_presentation/test_file_io.py @@ -0,0 +1,29 @@ +import asyncio +import threading + +import pytest + +from app.utils.file_io import write_bytes + +pytestmark = pytest.mark.unit + + +@pytest.mark.asyncio +async def test_write_bytes_does_not_block_event_loop(monkeypatch, tmp_path): + write_started = threading.Event() + release_write = threading.Event() + + def blocking_write(_path, _content): + write_started.set() + release_write.wait(timeout=1) + return 5 + + monkeypatch.setattr("pathlib.Path.write_bytes", blocking_write) + + task = asyncio.create_task(write_bytes(str(tmp_path / "audio.mp3"), b"audio")) + while not write_started.is_set(): + await asyncio.sleep(0) + + assert not task.done() + release_write.set() + await task