From c600a2920bb521847d1fd50e8034d23a8fe179a8 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 5 Jul 2026 03:51:16 -0700 Subject: [PATCH] feat(crawler): harden web crawler and agent tooling for real-world research tasks Crawler engine: escalate thin JS-shell pages past static fetch, repair currency-lossy extractions, emit categorized link records with anchor provenance, and decode percent-encoded mailto:/tel: contacts; site crawls reuse the connector ladder via Scrapling's spider engine with URL pattern filters. Agent layer: read_run gains char_offset paging, search_run gains match excerpts, new export_run turns stored runs into CSV workspace docs; reddit search fair-shares the item budget across queries and dedupes cross-query hits. Subagent prompts and routing teach crawl-after-search, full-run coverage before summarizing, and executing own-tool next steps instead of returning partial. --- .../system_prompt/prompts/routing.md | 31 ++ .../main_agent/tools/scrape_webpage.py | 27 +- .../builtins/google_maps/system_prompt.md | 2 + .../builtins/google_search/system_prompt.md | 2 + .../builtins/reddit/system_prompt.md | 9 +- .../builtins/web_crawler/system_prompt.md | 7 +- .../builtins/youtube/system_prompt.md | 3 + .../subagents/shared/run_reader.py | 313 +++++++++++++++++- .../shared/snippets/output_contract_base.md | 1 + .../subagents/shared/snippets/run_reader.md | 1 + .../app/capabilities/web/crawl/definition.py | 16 +- .../app/capabilities/web/crawl/executor.py | 57 +++- .../app/capabilities/web/crawl/schemas.py | 104 +++++- .../proprietary/platforms/reddit/scraper.py | 26 +- .../app/proprietary/web_crawler/README.md | 124 +++++++ .../app/proprietary/web_crawler/connector.py | 220 +++++++++++- .../proprietary/web_crawler/site_crawler.py | 220 +++++++++--- .../app/proprietary/web_crawler/url_policy.py | 160 +++++++-- surfsense_backend/app/utils/crawl/__init__.py | 4 + surfsense_backend/app/utils/crawl/contacts.py | 229 +++++++++++++ .../unit/capabilities/test_run_truncation.py | 135 +++++++- .../capabilities/web/crawl/test_executor.py | 49 +++ .../platforms/reddit/test_search_budget.py | 69 ++++ .../proprietary/web_crawler/test_connector.py | 134 ++++++++ .../web_crawler/test_url_policy.py | 76 +++-- .../tests/unit/utils/crawl/test_contacts.py | 103 ++++++ surfsense_backend/uv.lock | 163 +++++++-- 27 files changed, 2111 insertions(+), 174 deletions(-) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/run_reader.md create mode 100644 surfsense_backend/app/proprietary/web_crawler/README.md create mode 100644 surfsense_backend/app/utils/crawl/contacts.py create mode 100644 surfsense_backend/tests/unit/platforms/reddit/test_search_budget.py create mode 100644 surfsense_backend/tests/unit/utils/crawl/test_contacts.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md index 99296ecc8..2dd079be5 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md @@ -14,6 +14,25 @@ simulate one with the other. connectors, feature behavior) — point the user to the documentation: https://www.surfsense.com/docs. There is no docs-search tool; give the link. +**Search discovers — the crawler reads.** Search results (snippets, AI +overviews, a specialist's summary of a SERP) are pointers, not sources. +When the answer lives on a page — a team roster, a portfolio or directory +listing, a pricing table, docs — fetch the page before answering: +- One or a few known URLs → `scrape_webpage` directly. +- A site section or many pages (a whole team + portfolio, every pricing + page of a list of companies, a paginated directory) → + `task(web_crawler, …)` with the seed URLs. +Never answer with "you can find it at " for public facts your tools +can retrieve — retrieve them, then answer with the facts and cite the page. +Large results are fine: extract and return them, don't ask permission for +bounded fan-out (≤20 sites) the user already requested. + +**Full datasets become files, not chat.** When the user wants a complete +large dataset (an entire roster, portfolio, or directory — or asks for a +CSV/file), do not paste or summarize hundreds of rows: instruct the +web_crawler specialist to crawl and then save the data with its +`export_run` CSV tool, and relay the saved workspace path and row count. + **You have NO filesystem tools.** Any read, write, edit, move, rename, or search inside the user's workspace goes through `task(knowledge_base, …)` — never via `write_file`, `ls`, or any direct file operation. @@ -75,6 +94,18 @@ user: "What's the current USD/INR rate?" rate and return the rate with its source URL.") + +user: "Get the a16z team and their portfolio companies." +→ Search only *locates* a16z.com/team/ and their investment list — the + answer is the CONTENT of those pages. Crawl them and return the extracted + people and companies, never just the links: + task(web_crawler, "Crawl https://a16z.com/team/ and + https://a16z.com/investment-list/ and return (1) the full team roster + with each person's name and role/department, and (2) the complete + portfolio company list. Use the pages' link records if the markdown + is sparse.") + + user: "Find my Q2 roadmap and summarise the milestones." → task(knowledge_base, "Locate the Q2 roadmap document under /documents diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py index 2c1173bfc..5f2eaa913 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py @@ -244,15 +244,23 @@ def create_scrape_webpage_tool(): Scrape and extract the main content from a webpage. Use this tool when the user wants you to read, summarize, or answer - questions about a specific webpage's content. This tool actually - fetches and reads the full page content. For YouTube video URLs it - fetches the transcript directly instead of crawling the page. + questions about a specific webpage's content, or to pull a site's + contact details (emails, phone numbers, social profiles) for lead or + competitive-intelligence work. This tool actually fetches and reads the + full page content; JS-rendered pages are loaded in a real browser and + auto-scrolled, so lazy-loaded listings (directories, infinite-scroll + feeds) are captured too. For YouTube video URLs it fetches the + transcript directly instead of crawling the page. + + For a single page this returns its content plus any contacts found. To + sweep a whole site (e.g. hunt a contact/privacy page for an email), use + the web.crawl capability with maxCrawlDepth > 0 instead. Common triggers: - "Read this article and summarize it" - "What does this page say about X?" - "Summarize this blog post for me" - - "Tell me the key points from this article" + - "Find the contact email / socials for this company" - "What's in this webpage?" Args: @@ -271,6 +279,11 @@ def create_scrape_webpage_tool(): - domain: The domain name - word_count: Approximate word count - was_truncated: Whether content was truncated + - contacts: {emails, phones, socials} harvested from the page + - links: every link on the page as {url, text, rel, kind} where + kind is internal/external/social/email/tel. Use the anchor text + to tie a link to an entity (e.g. which person a LinkedIn URL + belongs to) and to pick the next page to scrape. - error: Error message (if scraping failed) """ scrape_id = generate_scrape_id(url) @@ -335,6 +348,12 @@ def create_scrape_webpage_tool(): "crawler_type": result.get("crawler_type", "unknown"), "author": metadata.get("author"), "date": metadata.get("date"), + # Contact/social signals from raw HTML (footer/legal boilerplate + # the markdown omits) — surfaced for lead/CI discovery. + "contacts": result.get("contacts"), + # Per-anchor inventory (url/text/rel/kind): anchor text is the + # raw material for tying targets to entities. + "links": result.get("link_records"), } except Exception as e: diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_maps/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_maps/system_prompt.md index d598e8fe8..e7e8e05ab 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_maps/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_maps/system_prompt.md @@ -8,6 +8,7 @@ Answer the delegated question from live Google Maps data gathered with your verb - `google_maps_scrape` - `google_maps_reviews` +- `read_run` / `search_run` (free readers for stored scrape output) @@ -16,6 +17,7 @@ Answer the delegated question from live Google Maps data gathered with your verb - Need richer detail (opening hours, popular times, extra contact info): set `include_details=true`. - Reviews / sentiment on specific places: call `google_maps_reviews` with the place `urls` or `place_ids`. - Batch multiple queries, URLs, or place IDs into one call rather than many single-item calls. + - Comparison requests: pull the current values, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/system_prompt.md index b5935878a..8626322a3 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/google_search/system_prompt.md @@ -7,6 +7,7 @@ Answer the delegated question from live Google Search data gathered with your ve - `google_search_scrape` +- `read_run` / `search_run` (free readers for stored scrape output) @@ -15,6 +16,7 @@ Answer the delegated question from live Google Search data gathered with your ve - Scraping a specific results page: pass the full Google Search URL in `queries`. - Need more results: raise `max_pages_per_query` to page beyond the first page. - Batch multiple search terms into one call rather than many single-term calls. + - Handing URLs off for crawling: return the organic result URLs so the supervisor can route them to the web crawling specialist. - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, moved up/down). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/reddit/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/reddit/system_prompt.md index 84a65d147..d1992582b 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/reddit/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/reddit/system_prompt.md @@ -7,6 +7,7 @@ Answer the delegated question from live Reddit data gathered with your verb, com - `reddit_scrape` +- `read_run` / `search_run` (free readers for stored scrape output) @@ -15,7 +16,11 @@ Answer the delegated question from live Reddit data gathered with your verb, com - Scraping a specific post, subreddit, or user: pass its Reddit URL in `urls`. - Reading comment sentiment: keep `skip_comments` false and raise `max_comments`; set `skip_comments` true when you only need posts (faster). - Controlling volume: use `max_items` for the total cap, `max_posts` per target, `max_comments` per post. +- Requested counts: `max_items` defaults to only 10 — when the task asks for N posts, set `max_items` and `max_posts` above N (with headroom for off-topic hits) and set `skip_comments=true` unless comments are needed. A call that caps below the target can never satisfy it. +- Topical discovery ("posts asking for X"): use broad unquoted queries and several phrasings (e.g. "X alternative", "alternative to X", "app like X") with `sort=relevance`; quoted exact phrases and `sort=new` are precision tools that miss most matches. +- Under-delivery: if the first call returns fewer on-topic results than requested, broaden it yourself — more phrasings, `sort=relevance`, wider or no time window — before settling. Return `status=partial` only after the broadened attempt, never after a single narrow call. - Batch multiple search terms into one call rather than many single-term calls. + - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, score/rank changes). @@ -57,6 +62,6 @@ Return **only** one JSON object (no markdown/prose): } Route-specific rules: -- `evidence.findings`: max 10 entries, each a single sentence stating one distinct post, comment, or delta. Do not paste raw payloads. -- `evidence.sources`: max 10 Reddit URLs, one per finding when applicable. List each URL once. +- `evidence.findings`: one entry per distinct post, comment, or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). +- `evidence.sources`: one Reddit URL per finding when applicable, same cap as findings. List each URL once. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/web_crawler/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/web_crawler/system_prompt.md index 2983f3274..59450bb05 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/web_crawler/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/web_crawler/system_prompt.md @@ -7,6 +7,8 @@ Answer the delegated question from live web evidence gathered with `web_crawl`, - `web_crawl` +- `read_run` / `search_run` (free readers for stored crawl output) +- `export_run` (save a stored run's rows as a CSV file in the workspace) @@ -14,6 +16,9 @@ Answer the delegated question from live web evidence gathered with `web_crawl`, - Whole site / "pages under X": set `maxCrawlDepth` to 1+ to follow links, and cap the run with `maxCrawlPages`. The crawl stays on the start URL's site. - Batch known URLs into one `web_crawl` call (pass them all in `startUrls`) rather than many single-URL calls. - Keep depth and page caps as small as the task allows — each fetched page is billable. + +- Rosters and listings: when a page's markdown is truncated or sparse, the item's `links` records (url, anchor text, context) usually carry the full list — read them from the stored run before re-crawling. +- Full-dataset requests ("the complete roster/list", "as a CSV/file"): never re-type hundreds of rows. Crawl, then `export_run(ref, path, rows='links', include_pattern=...)` — the rows are copied in code, byte-exact. Verify with the returned row count + preview, and report the saved path. - Comparison requests: crawl the current values, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new). @@ -24,7 +29,7 @@ Answer the delegated question from live web evidence gathered with `web_crawl`, -- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Do not generate deliverables (reports, podcasts, videos, images) or perform connector mutations; return findings for the supervisor to act on. Saving crawled data as a CSV via `export_run` is in scope. - YouTube URLs belong to the youtube specialist, not here. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/youtube/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/youtube/system_prompt.md index 9b5ebc472..f93b4f49a 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/youtube/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/youtube/system_prompt.md @@ -8,6 +8,7 @@ Answer the delegated question from live YouTube data gathered with your verbs, c - `youtube_scrape` - `youtube_comments` +- `read_run` / `search_run` (free readers for stored scrape output) @@ -15,6 +16,8 @@ Answer the delegated question from live YouTube data gathered with your verbs, c - Finding videos on a topic: call `youtube_scrape` with `search_queries`. - Comments / sentiment on specific videos: call `youtube_comments` with the video `urls`. - Batch multiple URLs (or queries) into one call rather than many single-item calls. + +- Multi-video comment analysis: a batched comments result lists videos in order, so a truncated preview usually shows only the first video(s). Before summarizing, page the stored run (or `search_run` by video id) until you have read real comments for EVERY video in the batch — never infer one video's sentiment from another's, and never report a video as "limited data" while its comments sit unread in the run. - Comparison requests: pull the current values, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py index 3a37d4979..7ef03a68c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py @@ -1,17 +1,24 @@ -"""``read_run`` / ``search_run``: page and grep a stored run or spill by line. +"""``read_run`` / ``search_run`` / ``export_run``: work with a stored run or spill. Scraper capability outputs and evicted context spills are stored full in Postgres (``runs`` / ``tool_output_spills``); the model only ever sees a capped preview plus -a reference like ``run_`` or ``spill_``. These two tools let the agent -retrieve the rest on demand — line-based paging and pattern search — without ever -loading the whole payload into context. Every lookup is scoped to the caller's -workspace (the trust boundary). +a reference like ``run_`` or ``spill_``. The read tools retrieve the +rest on demand — line-based paging and pattern search — without ever loading the +whole payload into context. ``export_run`` goes one step further for bulk +datasets: it converts the stored items (or their nested link records) to CSV +**in code** and saves the file as a workspace document, so hundreds of rows never +flow through the model at all. Every lookup is scoped to the caller's workspace +(the trust boundary). """ from __future__ import annotations +import csv +import io +import json +import logging import re -from typing import Annotated +from typing import Annotated, Any from uuid import UUID from langchain_core.tools import BaseTool, StructuredTool @@ -20,10 +27,19 @@ from sqlalchemy import select from app.capabilities.core.runs import RUN_OUTPUT_CHAR_CAP from app.db import Run, ToolOutputSpill, shielded_async_session +logger = logging.getLogger(__name__) + _MAX_LIMIT = 100 _MAX_PATTERN_LEN = 200 """ReDoS guard: reject/simplify absurdly long model-supplied patterns.""" +_EXPORT_MAX_ROWS = 20_000 +"""ponytail: hard row cap so a 200-page crawl can't produce a CSV whose +embedding pass stalls the turn. Raise alongside a background-embedding path.""" + +_ITEM_DEFAULT_FIELDS = ["url", "status", "error"] +_LINK_DEFAULT_FIELDS = ["page", "url", "text", "context", "kind"] + def _parse_ref(ref: str) -> tuple[str, UUID] | None: """Split ``run_`` / ``spill_`` into ``(kind, uuid)``; ``None`` if malformed.""" @@ -79,12 +95,118 @@ def _cap(body: str) -> str: return ( body[:RUN_OUTPUT_CHAR_CAP] + f"\n\n...[response truncated at {RUN_OUTPUT_CHAR_CAP} chars; " - "narrow with a larger offset or use search_run]..." + "narrow the pattern or lower max_matches]..." ) +def _rows_from_body(body: str, rows: str) -> list[dict[str, Any]]: + """Deterministically flatten stored JSONL into export rows. + + ``rows="items"`` → one row per stored item. ``rows="links"`` → explode each + item's ``links`` records, prefixing every row with the page it came from. + Non-JSON lines (plain-text spills) are skipped. + """ + out: list[dict[str, Any]] = [] + for line in body.split("\n"): + try: + item = json.loads(line) + except (json.JSONDecodeError, ValueError): + continue + if not isinstance(item, dict): + continue + if rows == "items": + out.append(item) + continue + page = str(item.get("url") or "") + for link in item.get("links") or []: + if isinstance(link, dict): + out.append({"page": page, **link}) + return out + + +def _cell(value: Any) -> str: + """Render one CSV cell: scalars as-is, nested structures as compact JSON.""" + if value is None: + return "" + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False, default=str) + return str(value) + + +def _rows_to_csv(records: list[dict[str, Any]], fields: list[str]) -> tuple[str, int]: + """Serialize deduplicated rows to CSV text; returns ``(csv_text, row_count)``.""" + buf = io.StringIO() + writer = csv.writer(buf, lineterminator="\n") + writer.writerow(fields) + seen: set[tuple[str, ...]] = set() + count = 0 + for record in records: + row = tuple(_cell(record.get(field)) for field in fields) + if row in seen: + continue + seen.add(row) + writer.writerow(row) + count += 1 + if count >= _EXPORT_MAX_ROWS: + break + return buf.getvalue(), count + + +async def _save_export_document( + *, virtual_path: str, content: str, workspace_id: int +) -> tuple[int, str] | str: + """Persist the CSV as a workspace document; ``(doc_id, path)`` or an error string. + + Uses the same canonical create path as end-of-turn KB persistence (folder + hierarchy + Document + chunks + embeddings), committed immediately — an + export is deterministic, so there is nothing to stage. + """ + # Deferred import: kb_persistence lives in the main-agent package, which + # transitively imports this module — same cycle-avoidance as the tool builder. + from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence.middleware import ( + _create_document, + ) + from app.agents.chat.runtime.path_resolver import DOCUMENTS_ROOT + from app.db import async_session_maker + + path = virtual_path.strip() + if not path.startswith("/"): + path = "/" + path + if not path.startswith(DOCUMENTS_ROOT + "/"): + path = DOCUMENTS_ROOT + path + + try: + async with async_session_maker() as session: + doc = await _create_document( + session, + virtual_path=path, + content=content, + workspace_id=workspace_id, + created_by_id=None, + ) + await session.commit() + doc_id = doc.id + except ValueError as exc: + return f"Error: {exc}. Pick a different path." + except Exception: + logger.exception("export_run: document create failed for %s", path) + return "Error: could not save the export document (storage failure)." + + # Best-effort UI refresh; the document row is already committed. + try: + from langchain_core.callbacks import adispatch_custom_event + + await adispatch_custom_event( + "document_created", + {"id": doc_id, "title": path.rsplit("/", 1)[-1], "virtualPath": path}, + ) + except Exception: + logger.debug("export_run: document_created dispatch failed", exc_info=True) + return doc_id, path + + def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]: - """Build the ``read_run`` / ``search_run`` tools bound to one workspace.""" + """Build the ``read_run`` / ``search_run`` / ``export_run`` tools for one workspace.""" async def _read_run( ref: Annotated[ @@ -92,6 +214,12 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]: ], offset: Annotated[int, "0-based line (item) index to start from."] = 0, limit: Annotated[int, "Max lines (items) to return (default 20)."] = 20, + char_offset: Annotated[ + int, + "0-based character index within the selected lines to start from. " + "Use this to page through a single item bigger than one response " + "(the truncation note tells you the next char_offset).", + ] = 0, ) -> str: loaded = await _load_body(ref, workspace_id) if isinstance(loaded, str): @@ -105,10 +233,30 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]: return ( f"No lines at offset {start} (total {len(lines)} lines in {ref})." ) + window_body = "\n".join(window) + start_char = max(0, char_offset) + if start_char >= len(window_body) > 0: + return ( + f"No content at char_offset {start_char} " + f"(this window is {len(window_body)} chars)." + ) + remaining = window_body[start_char:] + shown = remaining[:RUN_OUTPUT_CHAR_CAP] header = ( - f"Showing lines {start}-{start + len(window) - 1} of {len(lines)} in {ref}:\n" + f"Showing lines {start}-{start + len(window) - 1} of {len(lines)} in {ref}" + + (f", from char {start_char} of this window" if start_char else "") + + ":\n" ) - return _cap(header + "\n".join(window)) + if len(remaining) > len(shown): + left = len(remaining) - len(shown) + return ( + header + + shown + + f"\n\n...[truncated; {left} chars remain in this window — " + f"continue with char_offset={start_char + len(shown)}, or use " + "search_run]..." + ) + return header + shown async def _search_run( ref: Annotated[str, "The run reference to search, e.g. 'run_'."], @@ -128,10 +276,11 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]: matches: list[str] = [] total = 0 for idx, line in enumerate(body.split("\n")): - if matcher(line): + span = matcher(line) + if span is not None: total += 1 if len(matches) < limit: - matches.append(f"[{idx}] {line}") + matches.append(f"[{idx}] {_excerpt(line, span)}") if not matches: return f"No lines in {ref} matched {pattern!r}." header = ( @@ -140,14 +289,89 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]: ) return _cap(header + "\n".join(matches)) + async def _export_run( + ref: Annotated[ + str, "The run reference to export, e.g. 'run_' or 'spill_'." + ], + path: Annotated[ + str, + "Destination file path in the workspace, e.g. " + "'/documents/exports/a16z-team.csv'.", + ], + rows: Annotated[ + str, + "'links' = one CSV row per link record on each crawled page " + "(columns like page, url, text, context, kind — use for rosters, " + "directories, listings). 'items' = one row per stored result item.", + ] = "links", + fields: Annotated[ + list[str] | None, + "Columns to include, in order. Defaults: links -> " + "page,url,text,context,kind; items -> url,status,error.", + ] = None, + include_pattern: Annotated[ + str | None, + "Only keep rows matching this substring/regex (tested against the " + "row's combined values), e.g. '/author/' for team-profile links.", + ] = None, + exclude_pattern: Annotated[ + str | None, "Drop rows matching this substring/regex." + ] = None, + ) -> str: + loaded = await _load_body(ref, workspace_id) + if isinstance(loaded, str): + return loaded + body, _kind = loaded + if rows not in ("items", "links"): + return "Error: rows must be 'items' or 'links'." + + records = _rows_from_body(body, rows) + if include_pattern: + inc = _build_matcher(include_pattern.strip()) + records = [r for r in records if inc(" ".join(map(_cell, r.values()))) is not None] + if exclude_pattern: + exc = _build_matcher(exclude_pattern.strip()) + records = [r for r in records if exc(" ".join(map(_cell, r.values()))) is None] + if not records: + return ( + f"Error: no rows to export from {ref} " + f"(rows={rows}, include={include_pattern!r}, exclude={exclude_pattern!r}). " + "Loosen the filters or check the run with search_run." + ) + + columns = [f for f in (fields or []) if f] or ( + _LINK_DEFAULT_FIELDS if rows == "links" else _ITEM_DEFAULT_FIELDS + ) + csv_text, row_count = _rows_to_csv(records, columns) + + saved = await _save_export_document( + virtual_path=path, content=csv_text, workspace_id=workspace_id + ) + if isinstance(saved, str): + return saved + doc_id, final_path = saved + + preview_lines = csv_text.split("\n")[:4] + truncated_note = ( + f" (capped at {_EXPORT_MAX_ROWS} rows)" if row_count >= _EXPORT_MAX_ROWS else "" + ) + return ( + f"Exported {row_count} rows{truncated_note} to {final_path} " + f"(document id {doc_id}, {len(csv_text)} chars).\n" + f"Columns: {', '.join(columns)}\n" + "First lines:\n" + "\n".join(preview_lines) + ) + return [ StructuredTool.from_function( name="read_run", description=( "Read a stored scraper run or spilled tool output by line, in pages. " "Use the reference from a truncated tool result (e.g. 'run_'). " - "Each line is one result item (JSON). Page with offset/limit; prefer " - "search_run when hunting for something specific." + "Each line is one result item (JSON). Page with offset/limit; when a " + "single item is bigger than one response, keep offset fixed and page " + "inside it with char_offset. Prefer search_run when hunting for " + "something specific." ), coroutine=_read_run, ), @@ -161,15 +385,68 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]: ), coroutine=_search_run, ), + StructuredTool.from_function( + name="export_run", + description=( + "Export a stored run's structured data to a CSV file saved in the " + "user's workspace — deterministically, in code, without the rows " + "passing through you. Use for full-dataset requests (a complete " + "team roster, portfolio list, directory): crawl first, then export " + "the run instead of re-typing hundreds of rows. rows='links' " + "explodes each page's link records (filter with include_pattern, " + "e.g. a profile-URL fragment); rows='items' exports one row per " + "result item. Identical rows are deduplicated — on multi-page " + "crawls, omit 'page' from fields so the same link found on many " + "pages collapses to one row. Returns the saved path, row count, " + "and a preview." + ), + coroutine=_export_run, + ), ] +_EXCERPT_RADIUS = 300 +"""Chars shown on each side of a match when the line itself is huge.""" + + +def _excerpt(line: str, match_start: int) -> str: + """Return the line whole, or a window around the match for oversized lines. + + A crawled page is one JSON line that can run to hundreds of kB; returning it + verbatim would blow the response cap after one match. The noted char offset + plugs straight into ``read_run(..., char_offset=)`` for wider context. + """ + if len(line) <= _EXCERPT_RADIUS * 2: + return line + start = max(0, match_start - _EXCERPT_RADIUS) + end = min(len(line), match_start + _EXCERPT_RADIUS) + prefix = "..." if start > 0 else "" + suffix = "..." if end < len(line) else "" + return ( + f"(match at char {match_start} of {len(line)}) " + f"{prefix}{line[start:end]}{suffix}" + ) + + def _build_matcher(pattern: str): - """Compile a line matcher; fall back to substring on bad/oversized regex (ReDoS guard).""" + """Compile a line matcher returning the match start index, or ``None``. + + Falls back to substring on bad/oversized regex (ReDoS guard). + """ + + def _substring(line: str) -> int | None: + idx = line.lower().find(pattern.lower()) + return idx if idx >= 0 else None + if len(pattern) > _MAX_PATTERN_LEN: - return lambda line: pattern in line + return _substring try: compiled = re.compile(pattern, re.IGNORECASE) except re.error: - return lambda line: pattern.lower() in line.lower() - return lambda line: compiled.search(line) is not None + return _substring + + def _regex(line: str) -> int | None: + m = compiled.search(line) + return m.start() if m else None + + return _regex diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md index 100daae75..35fd48814 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md @@ -1,6 +1,7 @@ Rules (universal): - `status=success` -> `next_step=null`, `missing_fields=null`. - `status=partial|blocked|error` -> `next_step` must be non-null. +- `next_step` is only for actions you cannot take yourself. If the step is a call to one of your own tools (paging a stored run with `read_run`/`search_run`, re-running with adjusted parameters), execute it now and report the improved result instead of returning `partial`. - `status=blocked` due to missing required inputs -> `missing_fields` must be non-null. - `assumptions`: any inferences you made about the user's intent; `null` when no inferences were needed. - The `evidence` object's fields are documented in your route-specific `` above; never invent fields the tool did not return. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/run_reader.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/run_reader.md new file mode 100644 index 000000000..0cae9ec9e --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/run_reader.md @@ -0,0 +1 @@ +- Truncated results: a large tool result is stored in full and shown as a preview ending with a `run_` reference. Never re-run the tool to see more — page the stored run with `read_run(ref, offset, limit)` (each line is one result item as JSON) or grep it with `search_run(ref, pattern)`. If one item is itself bigger than a response, keep `offset` on that line and continue inside it with `char_offset` (the truncation note gives the next value). diff --git a/surfsense_backend/app/capabilities/web/crawl/definition.py b/surfsense_backend/app/capabilities/web/crawl/definition.py index de3cef0ca..cd3db2306 100644 --- a/surfsense_backend/app/capabilities/web/crawl/definition.py +++ b/surfsense_backend/app/capabilities/web/crawl/definition.py @@ -13,8 +13,20 @@ WEB_CRAWL = Capability( "startUrls. Set maxCrawlDepth=0 to fetch just those URLs, or higher to " "also follow the links on each page (depth 1 = the start pages plus the " "pages they link to, and so on) — staying on the same site and stopping " - "at maxCrawlPages. Returns one item per fetched page with clean markdown " - "content, metadata (title, description), and crawl provenance." + "at maxCrawlPages. On a deeper crawl, narrow which links are followed with " + "includeUrlPatterns / excludeUrlPatterns (regexes). Returns one item per " + "fetched page with clean markdown content, metadata (title, description), " + "crawl provenance, every link with its anchor text and kind " + "(internal/external/social/email/tel — use the text/context to tie a " + "profile URL to a person or company), and contact signals (emails, phone " + "numbers, social profiles). The site-wide contacts summary deduplicates " + "them with provenance: siteWide=true marks footer/header values (the " + "company's own contacts) vs page-local finds (e.g. team members' " + "profiles). Useful for lead generation and competitive intelligence; " + "contact details often live on about/contact/privacy pages, so crawl " + "with maxCrawlDepth >= 1 to surface them. JS-rendered pages are loaded " + "in a real browser and auto-scrolled, so lazy-loaded listings " + "(directories, infinite-scroll feeds) are captured too." ), input_schema=CrawlInput, output_schema=CrawlOutput, diff --git a/surfsense_backend/app/capabilities/web/crawl/executor.py b/surfsense_backend/app/capabilities/web/crawl/executor.py index 45f5b3004..cb000c6ef 100644 --- a/surfsense_backend/app/capabilities/web/crawl/executor.py +++ b/surfsense_backend/app/capabilities/web/crawl/executor.py @@ -10,10 +10,14 @@ from __future__ import annotations from app.capabilities.core import Executor from app.capabilities.web.crawl.schemas import ( + ContactRef, + Contacts, CrawlInput, CrawlItem, CrawlMeta, CrawlOutput, + Link, + SiteContacts, ) from app.proprietary.web_crawler import ( CrawlOutcomeStatus, @@ -39,9 +43,13 @@ def build_crawl_executor(engine: WebCrawlerConnector | None = None) -> Executor: payload.startUrls, max_crawl_depth=payload.maxCrawlDepth, max_crawl_pages=payload.maxCrawlPages, + include_patterns=payload.includeUrlPatterns, + exclude_patterns=payload.excludeUrlPatterns, ) + items = [_to_item(page, payload.maxLength) for page in pages] return CrawlOutput( - items=[_to_item(page, payload.maxLength) for page in pages], + items=items, + contacts=_aggregate_contacts(items), captcha_attempts=sum(page.captcha_attempts for page in pages), captcha_solved=sum(1 for page in pages if page.captcha_solved), ) @@ -51,6 +59,7 @@ def build_crawl_executor(engine: WebCrawlerConnector | None = None) -> Executor: def _to_item(page: CrawlPage, max_length: int) -> CrawlItem: content = page.content[:max_length] if page.content is not None else None + contacts = Contacts(**page.contacts) if page.contacts else None return CrawlItem( url=page.url, status=_STATUS_LABEL[page.status], @@ -61,5 +70,51 @@ def _to_item(page: CrawlPage, max_length: int) -> CrawlItem: ), markdown=content, metadata=page.metadata, + contacts=contacts, + links=[Link(**record) for record in page.links or []], error=page.error, ) + + +# Pages listed per contact value; the full list lives in the per-page items. +_MAX_REF_PAGES = 5 + + +def _aggregate_contacts(items: list[CrawlItem]) -> SiteContacts: + """Union each page's contacts with provenance (which pages, site-wide or not). + + ``siteWide`` marks values found on the majority of successfully parsed + pages: on a multi-page crawl that's header/footer boilerplate — the + company's own contacts — as opposed to page-local finds like one person's + LinkedIn on the team page. ponytail: a single-page crawl can't tell the + two apart, so everything is siteWide there; only page structure (footer + detection) could do better. + """ + pages_with_contacts = sum(1 for item in items if item.contacts is not None) + threshold = max(2, pages_with_contacts / 2) + + def refs(values_by_page: dict[str, list[str]]) -> list[ContactRef]: + return [ + ContactRef( + value=value, + pages=pages[:_MAX_REF_PAGES], + pageCount=len(pages), + siteWide=pages_with_contacts == 1 or len(pages) >= threshold, + ) + for value, pages in values_by_page.items() + ] + + emails: dict[str, list[str]] = {} + phones: dict[str, list[str]] = {} + socials: dict[str, list[str]] = {} + for item in items: + if item.contacts is None: + continue + for bucket, values in ( + (emails, item.contacts.emails), + (phones, item.contacts.phones), + (socials, item.contacts.socials), + ): + for value in values: + bucket.setdefault(value, []).append(item.url) + return SiteContacts(emails=refs(emails), phones=refs(phones), socials=refs(socials)) diff --git a/surfsense_backend/app/capabilities/web/crawl/schemas.py b/surfsense_backend/app/capabilities/web/crawl/schemas.py index 6b7b627eb..6894bd8e9 100644 --- a/surfsense_backend/app/capabilities/web/crawl/schemas.py +++ b/surfsense_backend/app/capabilities/web/crawl/schemas.py @@ -7,8 +7,9 @@ bounded by ``maxCrawlPages`` and kept on the seed's site. Fields are trimmed to what the proprietary engine honors today. Knobs the engine handles automatically (crawler type, proxy, dynamic-render waits) are -intentionally omitted, as are features we haven't built (URL globs, output -formats, click actions, PII handling). +intentionally omitted, as are features we haven't built (output formats, click +actions, PII handling). Link following can be narrowed with include/exclude URL +regex patterns. """ from __future__ import annotations @@ -60,6 +61,23 @@ class CrawlInput(BaseModel): ge=1, description="Maximum characters of cleaned markdown kept per page (truncates beyond).", ) + includeUrlPatterns: list[str] = Field( + default_factory=list, + max_length=25, + description=( + "Regex patterns a discovered link must match to be followed " + "(when maxCrawlDepth > 0). Empty = follow every same-site link. " + "Ignored for the start URLs, which are always fetched." + ), + ) + excludeUrlPatterns: list[str] = Field( + default_factory=list, + max_length=25, + description=( + "Regex patterns that exclude a discovered link from being followed. " + "Takes precedence over includeUrlPatterns." + ), + ) @property def estimated_units(self) -> int: @@ -80,6 +98,64 @@ class CrawlMeta(BaseModel): ) +class Link(BaseModel): + url: str = Field(description="Absolute link target (or address for email/tel).") + text: str = Field( + default="", + description="Anchor text — the label the page gives this link (e.g. a person's name on a LinkedIn link).", + ) + context: str = Field( + default="", + description=( + "For unlabeled social/email/tel links: surrounding text (e.g. the " + "person card an icon link sits in). Empty when text says it all." + ), + ) + rel: str = Field(default="", description="The anchor's rel attribute, if any.") + kind: Literal["internal", "external", "social", "email", "tel"] = Field( + description=( + "internal = same site; external = other site; social = known " + "profile host (LinkedIn, X, GitHub, ...); email/tel = mailto:/tel: targets." + ), + ) + + +class Contacts(BaseModel): + emails: list[str] = Field( + default_factory=list, description="Email addresses found on the page." + ) + phones: list[str] = Field( + default_factory=list, description="Phone numbers (from tel: links)." + ) + socials: list[str] = Field( + default_factory=list, + description="Social/profile URLs (LinkedIn, X, GitHub, Instagram, etc.).", + ) + + +class ContactRef(BaseModel): + """One site-wide contact value plus where it was found.""" + + value: str = Field(description="The email address, phone number, or profile URL.") + pages: list[str] = Field( + description="First few page URLs this value was found on (crawl order)." + ) + pageCount: int = Field(description="Total number of pages it appeared on.") + siteWide: bool = Field( + description=( + "True when found on most fetched pages — i.e. header/footer " + "boilerplate, so it belongs to the site itself (the company). " + "False = page-local, e.g. one person's profile on a team page." + ), + ) + + +class SiteContacts(BaseModel): + emails: list[ContactRef] = Field(default_factory=list) + phones: list[ContactRef] = Field(default_factory=list) + socials: list[ContactRef] = Field(default_factory=list) + + class CrawlItem(BaseModel): url: str = Field(description="The requested URL for this page.") status: Literal["success", "empty", "failed"] = Field( @@ -94,6 +170,21 @@ class CrawlItem(BaseModel): metadata: dict[str, str] | None = Field( default=None, description="Page metadata such as title and description." ) + contacts: Contacts | None = Field( + default=None, + description=( + "Contact/social signals harvested from the page's raw HTML " + "(footer/legal boilerplate that the markdown omits)." + ), + ) + links: list[Link] = Field( + default_factory=list, + description=( + "Every link on the page with its anchor text and kind. The anchor " + "text ties targets to entities (e.g. which person a LinkedIn URL " + "belongs to) — use it instead of guessing from the URL." + ), + ) error: str | None = Field( default=None, description="Failure reason when status is not success." ) @@ -104,6 +195,15 @@ class CrawlOutput(BaseModel): default_factory=list, description="One item per fetched page, in crawl (BFS) order.", ) + contacts: SiteContacts = Field( + default_factory=SiteContacts, + description=( + "Deduplicated union of every page's contact signals with provenance: " + "each value lists the pages it was found on, and siteWide separates " + "footer/header boilerplate (the company's own contacts) from " + "page-local finds (e.g. individual people on a team page)." + ), + ) # Billing-only telemetry; excluded from the wire shape (mirrors web.scrape). captcha_attempts: int = Field(default=0, exclude=True) captcha_solved: int = Field(default=0, exclude=True) diff --git a/surfsense_backend/app/proprietary/platforms/reddit/scraper.py b/surfsense_backend/app/proprietary/platforms/reddit/scraper.py index 6fe97877a..e2384f706 100644 --- a/surfsense_backend/app/proprietary/platforms/reddit/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/reddit/scraper.py @@ -304,8 +304,14 @@ async def _search_flow( *, input_model: RedditScrapeInput, subreddit: str | None = None, + max_items: int | None = None, ) -> AsyncIterator[dict[str, Any]]: - """Global search, or in-subreddit when ``subreddit`` is set. De-dupes by id.""" + """Global search, or in-subreddit when ``subreddit`` is set. De-dupes by id. + + ``max_items`` overrides ``input_model.maxItems`` as this one query's cap — + used by :func:`iter_reddit` to fair-share the global budget across + concurrent searches. + """ params: dict[str, Any] = {"q": query, "sort": input_model.sort} if input_model.time: params["t"] = input_model.time @@ -320,7 +326,7 @@ async def _search_flow( path, params, frozenset({"t3"}), - max_items=input_model.maxItems, + max_items=input_model.maxItems if max_items is None else max_items, include_nsfw=input_model.includeNSFW, date_limit=input_model.postDateLimit, ): @@ -402,15 +408,31 @@ async def iter_reddit( yield item return + # Fair-share the item budget across queries: with a shared cap, the + # first-finishing (often broadest/noisiest) search would fill the whole + # collector limit and starve the precise queries. + # ponytail: ceil-split leaves slack unredistributed when a query + # under-fills its share; a work-stealing budget would fix that. + n = len(input_model.searches) + per_query = -(-input_model.maxItems // n) if n else 0 jobs = [ _search_flow( query, input_model=input_model, subreddit=input_model.searchCommunityName, + max_items=per_query, ) for query in input_model.searches ] + # Cross-query de-dup: each flow only de-dups within itself, but the same + # hot post matches several phrasings and would eat the collector budget. + seen_ids: set[str] = set() async for item in fan_out(jobs): + item_id = item.get("id") + if isinstance(item_id, str): + if item_id in seen_ids: + continue + seen_ids.add(item_id) yield item diff --git a/surfsense_backend/app/proprietary/web_crawler/README.md b/surfsense_backend/app/proprietary/web_crawler/README.md new file mode 100644 index 000000000..029c9e147 --- /dev/null +++ b/surfsense_backend/app/proprietary/web_crawler/README.md @@ -0,0 +1,124 @@ +# Web Crawler Engine + +Proprietary crawling engine (licensed separately from the Apache-2.0 project +root — see `app/proprietary/LICENSE`). Single framework (Scrapling) for +fetching, Trafilatura for HTML → markdown extraction. Callers import only from +`__init__.py`: `WebCrawlerConnector` / `crawl_url` for one page, `crawl_site` +for depth-bounded multi-page crawls, both returning the same outcome contract. + +## Module map + +| Module | Role | +|---|---| +| `connector.py` | Single-URL crawl: tiered fetch ladder, extraction, escalation heuristics | +| `site_crawler.py` | Multi-page crawl: Scrapling `CrawlerEngine` frontier over the connector | +| `url_policy.py` | Link record extraction and categorization (nav/social/contact/document) | +| `captcha.py` | Captcha detection, token harvesting, and injection page-actions | +| `stealth.py` | Stealth/anti-bot configuration for the StealthyFetcher tier | +| `testbench/` | Live-site regression bench (own README) | + +Contact extraction (`extract_contacts`) lives in `app/utils/crawl/contacts.py` +because non-proprietary callers use it too. + +## The fetch ladder + +Every crawl walks the same escalation ladder until one tier produces usable +content; callers see only the resulting `CrawlOutcome`, never the tier: + +1. **AsyncFetcher** — static HTTP, TLS-impersonated, cheap. Handles most pages. +2. **DynamicFetcher** — full browser (thread), for JS-rendered content. +3. **StealthyFetcher** — patchright Chromium with anti-bot + Cloudflare + solving and captcha handling, the expensive last resort. + +Success alone does not stop the ladder — two content-quality heuristics can +force escalation or re-extraction: + +### Thin-page (JS-shell) escalation + +A static fetch can "succeed" on an SPA that server-renders only a hero +paragraph and hydrates everything else client-side (a16z.com/team ships 4.2 MB +of HTML that extracts to 597 chars). A result is tagged `thin_static` and +escalated to the browser tier when **both** hold: + +- raw HTML ≥ 1 MB (`_JS_SHELL_MIN_HTML_BYTES`), and +- extracted content < 2.5 KB (`_JS_SHELL_MAX_CONTENT_CHARS`). + +Calibrated on live pages: true shells shipped ≥ 3.4 MB with < 0.05 % text; +every healthy page was under ~650 KB. Semi-shells (~150 KB, e.g. +ycombinator.com/people) intentionally stay on static — their server-rendered +link records still carry the roster. Upgrade path: hydration-marker sniffing +instead of size thresholds. + +### Lossy-extraction repair (currency-guarded) + +Trafilatura sometimes drops structured content (pricing cards, tables). We +can't detect every loss, but currency amounts are a cheap, high-precision +tripwire: if the raw HTML's visible text contains a currency amount +(`_CURRENCY_AMOUNT_RE`) and the extracted markdown doesn't, re-extract with +`favor_recall=True`; if the amount is still missing, fall back to a sanitized +`markdownify` of the whole ``. + +## Link records and contacts + +`url_policy.extract_link_records` returns categorized links with anchor-text +provenance — these records, not the markdown, are the primary source for +roster/directory answers (names survive in link records even when extraction +drops them). `extract_contacts` harvests emails, phones, and social profiles +country-agnostically (global social-host list, `unquote()` applied to +percent-encoded `mailto:`/`tel:` hrefs — both here and in `url_policy`). + +## Multi-page crawls + +`crawl_site` uses Scrapling's spider engine for the traversal only (frontier, +dedupe, same-site scope, `includeUrlPatterns`/`excludeUrlPatterns` regex +filtering); every fetch still goes through `crawl_url`, so the ladder, proxy +rotation, and captcha handling are reused unchanged. Each `CrawlPage` carries +provenance (depth, referrer). + +## Agent tooling layer (outside this package) + +- The main chat agent has `scrape_webpage`; the `web_crawler` subagent has the + `web.crawl` capability (single URL or site mode). +- Tool outputs over the 40k-char cap (`RUN_OUTPUT_CHAR_CAP` in + `app/capabilities/core/runs.py`) are stored as JSONL runs; agents page them + with `read_run` (line paging + `char_offset` for giant single items), grep + them with `search_run` (returns excerpts around matches), and export them + deterministically with `export_run` (JSONL → CSV → workspace document, with + filtering and dedupe). Prompts live in + `app/agents/chat/multi_agent_chat/subagents/`. + +## Session learnings (agent E2E hardening, Jul 2026) + +Natural-language tasks run end-to-end through the multi-agent chat surfaced +these; each fix has a matching unit test: + +1. **Search discovers — the crawler reads.** The agent initially summarized + from SERP snippets instead of crawling the pages it found. Routing guidance + (`main_agent/system_prompt/prompts/routing.md`) now tells it to crawl every + URL whose full content would improve the answer, executing bounded fan-out + without asking permission. +2. **Success alone is not enough** — content-quality tripwires (thin-page, + currency-loss) must gate the ladder, because a "successful" fetch can carry + an empty shell or a lossy extraction. Tests: + `tests/unit/proprietary/web_crawler/test_connector.py`. +3. **Full datasets become files, not chat.** LLMs are bad data pipes: + transcribing a 486-row roster through the model loses rows and burns + tokens. `export_run` converts the stored run to CSV in code and saves it to + the workspace KB. Tests: `tests/unit/capabilities/test_run_truncation.py`. +4. **Truncation needs an escape hatch the model will actually use.** Large + items defeated line-based paging until `read_run` grew `char_offset` and + `search_run` grew match excerpts; subagent prompts explicitly list the + readers and forbid re-running tools to "see more". +5. **Shared budgets starve precise queries.** In the Reddit scraper, one noisy + search consumed the whole `maxItems` cap before precise phrasings returned; + the fix fair-shares the budget across concurrent searches and de-dupes + across them (`tests/unit/platforms/reddit/test_search_budget.py`). The same + failure shape applies to any multi-query fan-out with a shared collector cap. +6. **Subagents must not hand back work they can do.** The universal output + contract (`subagents/shared/snippets/output_contract_base.md`) now requires: + if `next_step` is a call to one of the subagent's own tools (paging a run, + re-running with better parameters), execute it instead of returning + `partial`. +7. **Sizing caps to the ask.** When a task requests N items, tool caps + (`max_items`, findings limits in output contracts) must be set above N or + the task is unwinnable by construction; prompts now say so. diff --git a/surfsense_backend/app/proprietary/web_crawler/connector.py b/surfsense_backend/app/proprietary/web_crawler/connector.py index e393ff5b4..ba26638e7 100644 --- a/surfsense_backend/app/proprietary/web_crawler/connector.py +++ b/surfsense_backend/app/proprietary/web_crawler/connector.py @@ -22,6 +22,7 @@ which tier produced it. import asyncio import logging +import re import time from collections.abc import Awaitable, Callable from dataclasses import dataclass @@ -30,6 +31,8 @@ from typing import Any import trafilatura import validators +from lxml import html as lxml_html +from markdownify import markdownify from scrapling.engines.toolbelt import is_proxy_error from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher @@ -38,9 +41,9 @@ from app.proprietary.web_crawler.stealth import ( build_stealthy_kwargs, get_stealth_config, ) -from app.proprietary.web_crawler.url_policy import extract_links +from app.proprietary.web_crawler.url_policy import extract_link_records from app.utils.captcha import captcha_enabled, get_captcha_config -from app.utils.crawl import BlockType, classify_block +from app.utils.crawl import BlockType, classify_block, extract_contacts from app.utils.proxy import get_proxy_url, is_pool_backed logger = logging.getLogger(__name__) @@ -48,6 +51,114 @@ logger = logging.getLogger(__name__) # Prefix for performance/timing log lines so they are easy to grep/filter. _PERF = "[webcrawler][perf]" +# Thin-page (JS-shell) escalation: a static fetch can "succeed" on an SPA that +# server-renders only a hero paragraph and hydrates the real content client-side +# (a16z.com/team: 4.2MB of HTML -> 597 chars extracted), so success alone must +# not stop the ladder. Calibrated on live pages (probe_thin_calibration): true +# shells shipped >=3.4MB with <0.05% text, while every healthy page was under +# ~650KB — so require BOTH a huge document and near-empty extraction. +# ponytail: ~150KB semi-shells (ycombinator.com/people) stay on static; their +# server-rendered link records still carry the content. Upgrade path: DOM +# hydration-marker sniffing instead of size thresholds. +_JS_SHELL_MIN_HTML_BYTES = 1_000_000 +_JS_SHELL_MAX_CONTENT_CHARS = 2_500 + + +def looks_like_js_shell(html_len: int, content_len: int) -> bool: + """True when a static fetch smells like an unhydrated SPA shell.""" + return ( + html_len >= _JS_SHELL_MIN_HTML_BYTES + and content_len < _JS_SHELL_MAX_CONTENT_CHARS + ) + + +# Lossy-extraction repair: trafilatura's main-content detection drops div-grid +# pricing cards / stat tables as "boilerplate" (seen live: duplicati.com/pricing +# kept 15% of visible text, goauthentik.io/pricing 0 of 5 currency figures while +# every price sat in the static DOM). Currency amounts are the one token class +# that is (a) trivially detectable, (b) never navigation chrome, and (c) the +# payload of exactly the pages agents ask for (pricing/plans). So: if the raw +# DOM shows a currency amount that the markdown lost, re-extract with +# favor_recall; if still lost, fall back to sanitized markdownify of the whole +# body (bounded — callers truncate via maxLength anyway). +# Covers $ € £ ¥ ₹ ₩ ₪ ₫ ₴ ₦ ₱ ฿ plus ISO codes like "USD 49"/"49 EUR" so the +# trigger is country-agnostic, and amounts-before-symbol ("49€", French/German). +_CURRENCY_AMOUNT_RE = re.compile( + r"[$€£¥₹₩₪₫₴₦₱฿]\s?\d" + r"|\d\s?[$€£¥₹₩₪₫₴₦₱฿]" + r"|\b(USD|EUR|GBP|JPY|CNY|INR|BRL|MXN|CAD|AUD|CHF|KRW|SEK|NOK|DKK|PLN)\s?\d" + r"|\d\s?(USD|EUR|GBP|JPY|CNY|INR|BRL|MXN|CAD|AUD|CHF|KRW|SEK|NOK|DKK|PLN)\b", + re.IGNORECASE, +) + +_STRIP_XPATH = "//script | //style | //noscript | //template | //svg | //iframe | //head" + + +def _visible_text(raw_html: str) -> str: + """Text of the DOM minus script/style — what a reader actually sees.""" + root = lxml_html.fromstring(raw_html) + for bad in root.xpath(_STRIP_XPATH): + bad.getparent().remove(bad) + return " ".join(" ".join(root.itertext()).split()) + + +def dropped_currency_amounts(raw_html: str, markdown: str) -> bool: + """True when the visible DOM has currency figures but the markdown has none.""" + if _CURRENCY_AMOUNT_RE.search(markdown): + return False + try: + return bool(_CURRENCY_AMOUNT_RE.search(_visible_text(raw_html))) + except Exception: + return False + + +def markdown_of_whole_body(raw_html: str) -> str | None: + """Sanitized markdownify of the full DOM — recall 100%, precision be damned. + + Last resort when main-content extraction provably dropped the payload: + nav/footer noise is acceptable, silently missing prices is not. + """ + try: + root = lxml_html.fromstring(raw_html) + for bad in root.xpath(_STRIP_XPATH): + bad.getparent().remove(bad) + md = markdownify(lxml_html.tostring(root, encoding="unicode")) + md = re.sub(r"\n{3,}", "\n\n", md).strip() + return md or None + except Exception: + return None + + +# Auto-scroll bounds for the browser tiers. JS directories/feeds lazy-load on +# scroll, so the initial render misses most items (e.g. YC's batch directory +# shows 40 of 100+ companies). The round cap keeps endless feeds (social +# timelines) from holding a billable fetch hostage; static-height pages exit +# after one no-growth check, costing a single settle wait. +_SCROLL_MAX_ROUNDS = 8 +_SCROLL_SETTLE_MS = 700 + + +def scroll_to_bottom(page: Any) -> Any: + """``page_action`` that scrolls until the document height stops growing. + + ponytail: jumps straight to the bottom each round, which is enough for + sentinel-based infinite scroll (Algolia et al.); lazy loaders keyed to + intersection of mid-page elements would need viewport-sized steps. Errors + mid-scroll keep whatever is already rendered instead of failing the fetch. + """ + try: + last_height = 0 + for _ in range(_SCROLL_MAX_ROUNDS): + height = page.evaluate("document.body.scrollHeight") + if not height or height <= last_height: + break + last_height = height + page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + page.wait_for_timeout(_SCROLL_SETTLE_MS) + except Exception as exc: + logger.debug("[webcrawler] auto-scroll aborted: %s", exc) + return page + class CrawlOutcomeStatus(StrEnum): """Deterministic per-URL crawl result, single-sourcing the billable signal.""" @@ -132,6 +243,9 @@ class WebCrawlerConnector: # (distinguishes EMPTY from FAILED, where every tier raised/was # unavailable). reached_without_content = False + # Static result tagged as a JS shell: escalate to the browser tiers + # for the hydrated page, but keep it as a last-resort fallback. + thin_static_result: dict[str, Any] | None = None # --- 1. Scrapling AsyncFetcher (fast static HTTP) --- tier_start = time.perf_counter() @@ -141,7 +255,16 @@ class WebCrawlerConnector: "scrapling-static", lambda: self._crawl_with_async_fetcher(url, block_state), ) - if result: + if result and result.pop("thin_static", False): + thin_static_result = result + errors.append( + "Scrapling static: JS-shell page (huge HTML, near-empty " + "extraction); escalating to browser" + ) + self._log_tier_outcome( + "scrapling-static", url, tier_start, "thin_shell" + ) + elif result: self._log_tier_outcome( "scrapling-static", url, tier_start, "success" ) @@ -152,9 +275,12 @@ class WebCrawlerConnector: tier="scrapling-static", block_type=block_state["block_type"], ) - reached_without_content = True - errors.append("Scrapling static: empty extraction") - self._log_tier_outcome("scrapling-static", url, tier_start, "empty") + else: + reached_without_content = True + errors.append("Scrapling static: empty extraction") + self._log_tier_outcome( + "scrapling-static", url, tier_start, "empty" + ) except Exception as exc: errors.append(f"Scrapling static: {exc!s}") self._log_tier_outcome( @@ -235,6 +361,19 @@ class WebCrawlerConnector: "scrapling-stealthy", url, tier_start, "error", exc ) + # Browser tiers all failed/empty: the thin static extraction is + # still real (partial) content — better than reporting nothing. + if thin_static_result is not None: + self._log_total(url, "scrapling-static-thin", total_start) + return CrawlOutcome( + status=CrawlOutcomeStatus.SUCCESS, + result=thin_static_result, + tier="scrapling-static", + captcha_attempts=captcha_state["attempts"], + captcha_solved=captcha_state["solved"], + block_type=block_state["block_type"], + ) + self._log_total(url, "none", total_start) if reached_without_content: return CrawlOutcome( @@ -375,7 +514,7 @@ class WebCrawlerConnector: ) return None - return self._build_result( + result = self._build_result( page.html_content, url, "scrapling-static", @@ -384,6 +523,14 @@ class WebCrawlerConnector: status=status, block_state=block_state, ) + if result and looks_like_js_shell( + len(page.html_content or ""), len(result.get("content") or "") + ): + # Tag rather than drop: crawl_url escalates to the browser tiers but + # keeps this as a fallback if they all fail (e.g. no subprocess + # support on Windows dev loops). + result["thin_static"] = True + return result async def _crawl_with_dynamic( self, url: str, block_state: dict[str, Any] | None = None @@ -407,6 +554,7 @@ class WebCrawlerConnector: network_idle=True, timeout=30000, proxy=get_proxy_url(), + page_action=scroll_to_bottom, ) fetch_ms = (time.perf_counter() - fetch_start) * 1000 return self._build_result( @@ -456,13 +604,20 @@ class WebCrawlerConnector: proxy = get_proxy_url() # Build the captcha page_action only when solving is enabled (and not - # process-latched). ``None`` => stealth tier behaves exactly as before. - page_action = None + # process-latched); auto-scroll always runs after it so lazy-loaded + # content behind a bot wall is captured too (captcha first: scrolling a + # challenge interstitial is pointless). + captcha_action = None if captcha_state is not None and captcha_enabled(): - page_action = build_captcha_page_action( + captcha_action = build_captcha_page_action( captcha_state, proxy, get_captcha_config() ) + def page_action(page: Any) -> Any: + if captcha_action is not None: + page = captcha_action(page) + return scroll_to_bottom(page) + # ``solve_cloudflare=True`` runs the full Turnstile/Interstitial challenge # loop; scoped to this last-resort tier only (it spins up the browser). # Scrapling runs solve_cloudflare BEFORE page_action, so Cloudflare is @@ -479,8 +634,7 @@ class WebCrawlerConnector: # Keys never collide with the core kwargs above; defaults preserve # today's behavior and add no crawl-speed regression. fetch_kwargs.update(build_stealthy_kwargs(get_stealth_config())) - if page_action is not None: - fetch_kwargs["page_action"] = page_action + fetch_kwargs["page_action"] = page_action page = StealthyFetcher.fetch(url, **fetch_kwargs) fetch_ms = (time.perf_counter() - fetch_start) * 1000 return self._build_result( @@ -555,6 +709,35 @@ class WebCrawlerConnector: except Exception: extracted_content = None + # Repair chain for provably lossy extraction: trafilatura sometimes + # classifies pricing cards / stat grids as boilerplate. If the DOM shows + # currency amounts the markdown lost, retry with favor_recall, then fall + # back to sanitized whole-body markdown. Guarded by the currency check, + # so ordinary pages never pay for a second extraction pass. + if extracted_content and dropped_currency_amounts(raw_html, extracted_content): + try: + recall = trafilatura.extract( + raw_html, + output_format="markdown", + include_comments=False, + include_tables=True, + include_images=True, + include_links=True, + favor_recall=True, + ) + except Exception: + recall = None + if recall and _CURRENCY_AMOUNT_RE.search(recall): + extracted_content = recall + else: + whole = markdown_of_whole_body(raw_html) + if whole and _CURRENCY_AMOUNT_RE.search(whole): + extracted_content = whole + logger.info( + f"{_PERF} event=lossy_repair url={url} recovered=" + f"{bool(_CURRENCY_AMOUNT_RE.search(extracted_content))}" + ) + extract_ms = (time.perf_counter() - extract_start) * 1000 if not extracted_content and not allow_raw_fallback: @@ -594,13 +777,24 @@ class WebCrawlerConnector: "extracted" if extracted_content else "raw_fallback", ) + # One DOM parse feeds both views: the rich per-anchor inventory (agent + # output — anchor text is the raw material for entity extraction) and + # the URL-only frontier for ``site_crawler.crawl_site``. + link_records = extract_link_records(raw_html, url) return { "content": content, "metadata": metadata, "crawler_type": crawler_type, # Next-hop targets for ``site_crawler.crawl_site``; ignored by # single-URL callers. - "links": extract_links(raw_html, url), + "links": [ + r["url"] for r in link_records if r["kind"] not in ("email", "tel") + ], + "link_records": link_records, + # Lead-gen signals harvested from raw HTML (footer/legal boilerplate + # that Trafilatura strips from ``content``). Dict form so callers can + # pass it straight through without importing the dataclass. + "contacts": extract_contacts(raw_html).as_dict(), } @staticmethod diff --git a/surfsense_backend/app/proprietary/web_crawler/site_crawler.py b/surfsense_backend/app/proprietary/web_crawler/site_crawler.py index 49a7f872e..0c24f70d2 100644 --- a/surfsense_backend/app/proprietary/web_crawler/site_crawler.py +++ b/surfsense_backend/app/proprietary/web_crawler/site_crawler.py @@ -2,27 +2,40 @@ # # Part of the ``app.proprietary`` package; licensed separately from the # Apache-2.0 project root (see ``app/proprietary/LICENSE``). -"""Depth-bounded site crawl built on the single-URL engine (``crawl_url``). +"""Depth-bounded site crawl driven by Scrapling's spider engine. -Breadth-first frontier: fetch a page, follow its same-site links one hop deeper, -dedupe by canonical URL, stop at ``max_crawl_pages``. Every fetch runs through -``crawl_url`` so tiered fetch, proxy, and captcha handling are reused. +The traversal (frontier, dedupe, link filtering, same-site scope) is Scrapling's +``CrawlerEngine`` + ``LinkExtractor``; every *fetch* still goes through our +``WebCrawlerConnector.crawl_url`` so the tiered fetch ladder, proxy rotation, and +captcha handling are reused unchanged. -Sequential today; ``_fetch_page`` isolates one unit of work so a bounded worker -pool can replace the loop later without changing the traversal. +The bridge is ``_ConnectorSession``: a duck-typed Scrapling "session" whose +``fetch`` calls ``crawl_url`` and wraps the ``CrawlOutcome`` in a Scrapling +``Response`` (the outcome is stashed on the response so ``parse`` can rebuild a +``CrawlPage``). The engine is awaited directly on the caller's event loop — +``Spider.start()`` is avoided because it spins up its own loop via ``anyio.run``. """ from __future__ import annotations -from collections import deque +import logging +from collections.abc import Iterable from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from scrapling.spiders import CrawlerEngine, LinkExtractor, Response, Spider from app.proprietary.web_crawler.connector import ( CrawlOutcome, CrawlOutcomeStatus, WebCrawlerConnector, ) -from app.proprietary.web_crawler.url_policy import canonicalize_url, host_of, same_site +from app.proprietary.web_crawler.url_policy import host_of + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from scrapling.spiders import Request @dataclass(frozen=True) @@ -36,61 +49,130 @@ class CrawlPage: loaded_url: str | None = None content: str | None = None metadata: dict[str, str] | None = None + contacts: dict[str, list[str]] | None = None + links: list[dict[str, str]] | None = None error: str | None = None captcha_attempts: int = 0 captcha_solved: bool = False -async def crawl_site( - engine: WebCrawlerConnector, - start_urls: list[str], - *, - max_crawl_depth: int, - max_crawl_pages: int, -) -> list[CrawlPage]: - """Crawl ``start_urls`` up to ``max_crawl_depth`` hops / ``max_crawl_pages`` pages. +# HTTP status the fake Response reports per outcome. Only used for Scrapling's +# stats/logging; block detection is disabled (our connector owns that), so the +# exact codes never gate a retry. +_HTTP_STATUS: dict[CrawlOutcomeStatus, int] = { + CrawlOutcomeStatus.SUCCESS: 200, + CrawlOutcomeStatus.EMPTY: 204, + CrawlOutcomeStatus.FAILED: 502, +} - Depth 0 fetches only the start URLs. Links are followed only from successful - pages, under the depth cap, and only on a start URL's site. Start URLs count - toward ``max_crawl_pages``. Order is BFS from the seeds. + +class _ConnectorSession: + """Scrapling-compatible session that fetches via ``WebCrawlerConnector``. + + ``SessionManager`` treats any non-``FetcherSession`` object as a browser-style + session and calls ``await session.fetch(url=..., **kwargs)``. We satisfy that + contract, run the real crawl, and translate the ``CrawlOutcome`` into a + ``Response`` — attaching the outcome as ``response._outcome`` (not ``meta``, + which ``response.follow`` would copy onto every child request). """ - allowed_hosts = {host_of(url) for url in start_urls} - visited: set[str] = set() - frontier: deque[tuple[str, int, str | None]] = deque() - for seed in start_urls: - key = canonicalize_url(seed) - if key not in visited: - visited.add(key) - frontier.append((seed, 0, None)) - pages: list[CrawlPage] = [] - while frontier and len(pages) < max_crawl_pages: - url, depth, referrer = frontier.popleft() - page, outcome = await _fetch_page(engine, url, depth, referrer) - pages.append(page) + def __init__(self, connector: WebCrawlerConnector): + self._connector = connector + self._is_alive = False - if depth >= max_crawl_depth: - continue + async def __aenter__(self) -> _ConnectorSession: + self._is_alive = True + return self + + async def __aexit__(self, *_exc: object) -> None: + self._is_alive = False + + async def fetch(self, url: str, **_kwargs: Any) -> Response: + outcome = await self._connector.crawl_url(url) + result = outcome.result or {} + content = result.get("content") + # Selector chokes on empty content; a fetch that raises would be counted + # as a failed request (no parse), dropping the page from the output. Feed + # a harmless placeholder so failed/empty pages still reach ``parse``. + response = Response( + url=result.get("loaded_url") or url, + content=content or "", + status=_HTTP_STATUS[outcome.status], + reason=outcome.status.value, + cookies={}, + headers={}, + request_headers={}, + ) + response._outcome = outcome # type: ignore[attr-defined] + return response + + +class _SiteSpider(Spider): + """Depth/page-bounded spider whose fetching is delegated to the connector. + + Concurrency is pinned to 1 so the page cap — and the per-page billing derived + from it — stays exact and the output preserves breadth-first order. + ponytail: raising ``concurrent_requests`` needs an atomic page-budget guard to + avoid overshooting ``max_pages`` (and thus over-fetching / over-billing). + """ + + name = "surfsense_site" + concurrent_requests = 1 + max_blocked_retries = 0 + logging_level = logging.WARNING + + def __init__( + self, + connector: WebCrawlerConnector, + start_urls: list[str], + *, + max_depth: int, + max_pages: int, + link_extractor: LinkExtractor, + ): + self._connector = connector + self._max_depth = max_depth + self._max_pages = max_pages + self._link_extractor = link_extractor + self.pages: list[CrawlPage] = [] + super().__init__() + self.start_urls = list(start_urls) + + def configure_sessions(self, manager: Any) -> None: + manager.add("default", _ConnectorSession(self._connector)) + + async def is_blocked(self, response: Response) -> bool: + # The connector already classifies blocks and exhausts its own fallback + # ladder + proxy rotation; never let the spider re-fetch on top of that. + return False + + async def parse( + self, response: Response + ) -> AsyncGenerator[Request | None, None]: + outcome: CrawlOutcome = response._outcome # type: ignore[attr-defined] + depth: int = response.meta.get("_depth", 0) + referrer: str | None = response.meta.get("_referrer") + req_url = response.request.url if response.request else str(response.url) + + if len(self.pages) < self._max_pages: + self.pages.append(_to_page(req_url, outcome, depth, referrer)) + + # Cap reached: stop the engine so queued-but-unfetched links are abandoned + # (never fetched, never billed), matching the old BFS's per-fetch guard. + if len(self.pages) >= self._max_pages: + self.pause() + return + if depth >= self._max_depth: + return if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result: - continue + return + for link in outcome.result.get("links", []): - key = canonicalize_url(link) - if key in visited or not same_site(link, allowed_hosts): + if not self._link_extractor.matches(link): continue - visited.add(key) - frontier.append((link, depth + 1, url)) - return pages - - -async def _fetch_page( - engine: WebCrawlerConnector, - url: str, - depth: int, - referrer: str | None, -) -> tuple[CrawlPage, CrawlOutcome]: - """Fetch one URL and map it to a ``CrawlPage`` (the future concurrency unit).""" - outcome = await engine.crawl_url(url) - return _to_page(url, outcome, depth, referrer), outcome + yield response.follow( + link, meta={"_depth": depth + 1, "_referrer": req_url} + ) def _to_page( @@ -109,6 +191,8 @@ def _to_page( loaded_url=result.get("loaded_url") or url, content=result.get("content"), metadata=result.get("metadata"), + contacts=result.get("contacts"), + links=result.get("link_records"), captcha_attempts=outcome.captcha_attempts, captcha_solved=outcome.captcha_solved, ) @@ -121,3 +205,37 @@ def _to_page( captcha_attempts=outcome.captcha_attempts, captcha_solved=outcome.captcha_solved, ) + + +async def crawl_site( + engine: WebCrawlerConnector, + start_urls: list[str], + *, + max_crawl_depth: int, + max_crawl_pages: int, + include_patterns: Iterable[str] | None = None, + exclude_patterns: Iterable[str] | None = None, +) -> list[CrawlPage]: + """Crawl ``start_urls`` up to ``max_crawl_depth`` hops / ``max_crawl_pages`` pages. + + Depth 0 fetches only the start URLs. Links are followed only from successful + pages, under the depth cap, on the seeds' sites (subdomains included), and + matching ``include_patterns`` / not matching ``exclude_patterns`` (regexes). + Start URLs count toward ``max_crawl_pages``. Order is breadth-first. + """ + link_extractor = LinkExtractor( + allow=tuple(include_patterns or ()), + deny=tuple(exclude_patterns or ()), + allow_domains=tuple(host_of(u) for u in start_urls), + ) + spider = _SiteSpider( + engine, + start_urls, + max_depth=max_crawl_depth, + max_pages=max_crawl_pages, + link_extractor=link_extractor, + ) + crawler = CrawlerEngine(spider, spider._session_manager) + spider._engine = crawler # enable spider.pause() to stop at the page cap + await crawler.crawl() + return spider.pages diff --git a/surfsense_backend/app/proprietary/web_crawler/url_policy.py b/surfsense_backend/app/proprietary/web_crawler/url_policy.py index 21f9215a3..46f74c8c3 100644 --- a/surfsense_backend/app/proprietary/web_crawler/url_policy.py +++ b/surfsense_backend/app/proprietary/web_crawler/url_policy.py @@ -2,25 +2,93 @@ # # Part of the ``app.proprietary`` package; licensed separately from the # Apache-2.0 project root (see ``app/proprietary/LICENSE``). -"""URL helpers for the site crawler: link extraction, canonical key, same-site scope. +"""URL helpers for the crawler: link extraction (connector) and host scope (spider). -Pure functions (no I/O) so the crawl frontier stays deterministic and testable. +Pure functions (no I/O). Dedupe/canonicalization and same-site link filtering now +live in Scrapling's ``Scheduler`` / ``LinkExtractor`` (see ``site_crawler``); only +these two primitives remain SurfSense-owned. """ from __future__ import annotations -from urllib.parse import urldefrag, urljoin, urlsplit +import re +from typing import Any +from urllib.parse import unquote, urldefrag, urljoin, urlsplit from lxml import html as lxml_html from lxml.etree import ParserError -from w3lib.url import canonicalize_url as _w3lib_canonicalize_url + +from app.utils.crawl import is_social_host + +_WHITESPACE_RE = re.compile(r"\s+") + +# Anchor text cap: card-style links wrap whole article previews in one ; +# beyond this the text is a content dump, not a label. +_MAX_ANCHOR_TEXT = 200 + +# Context cap: nearest-ancestor text for icon-only anchors. Person/company +# cards ("Jane Doe General Partner") fit well under this; anything longer is +# a section dump and gets truncated rather than dropped. +_MAX_CONTEXT = 120 -def extract_links(page_html: str | None, base_url: str) -> list[str]: - """Absolute, http(s), fragment-free, de-duplicated ```` targets. +def _collapse(text: str) -> str: + return _WHITESPACE_RE.sub(" ", text).strip() - Relative hrefs resolve against ``base_url``; the page's own URL is dropped. - First-seen order is preserved to keep the frontier stable. + +def _node_text(node: Any) -> str: + # itertext + space-join keeps a word boundary between block elements, + # where text_content() would glue "Jane Doe

Partner" together. + return _collapse(" ".join(node.itertext())) + + +def _anchor_label(anchor: Any) -> str: + """Best label for an anchor: its text, else aria-label/title, else img alt.""" + text = _node_text(anchor) + if text: + return text + for attr in ("aria-label", "title"): + value = _collapse(anchor.get(attr) or "") + if value: + return value + for alt in anchor.xpath(".//img/@alt"): + value = _collapse(str(alt)) + if value: + return value + return "" + + +def _anchor_context(anchor: Any) -> str: + """Nearest ancestor's text for unlabeled anchors (icon-only social links). + + Team/profile cards put the person's name next to — not inside — the icon + link, so the closest ancestor with any text is the entity label we want. + """ + node = anchor.getparent() + while node is not None: + text = _node_text(node) + if text: + return text[:_MAX_CONTEXT] + node = node.getparent() + return "" + + +def extract_link_records( + page_html: str | None, base_url: str +) -> list[dict[str, str]]: + """Structured ```` inventory: ``{url, text, context, rel, kind}`` per target. + + ``kind`` is one of ``internal`` (same site as ``base_url``), ``external``, + ``social`` (known profile host), ``email`` (``mailto:``), or ``tel``. http(s) + targets are absolutized against ``base_url`` and fragment-stripped; the + page's own URL is dropped. De-duplicated by target URL (first-seen order), + keeping the first non-empty anchor text so a nav logo link doesn't shadow + the labeled one. + + ``text`` falls back to aria-label/title/img-alt for icon-only anchors. + ``context`` (social/email/tel only) is the nearest ancestor's text — team + pages label a person *next to* their LinkedIn icon, not inside it, so this + is what ties a profile URL to its entity. """ if not page_html or not page_html.strip(): return [] @@ -30,30 +98,70 @@ def extract_links(page_html: str | None, base_url: str) -> list[str]: return [] self_url, _ = urldefrag(base_url) - seen: set[str] = set() - links: list[str] = [] - for href in root.xpath("//a/@href"): - target, _ = urldefrag(urljoin(base_url, href.strip())) - if urlsplit(target).scheme not in ("http", "https"): + base_host = host_of(base_url) + records: dict[str, dict[str, str]] = {} + + for anchor in root.xpath("//a[@href]"): + href = str(anchor.get("href", "")).strip() + low = href.lower() + # unquote: hrefs URL-encode spaces etc. ("tel:+1%20408-629-1770") + if low.startswith("mailto:"): + target = unquote(urlsplit(href).path.split("?")[0]).strip() + kind = "email" + elif low.startswith("tel:"): + target = unquote(urlsplit(href).path).strip() + kind = "tel" + else: + target, _ = urldefrag(urljoin(base_url, href)) + if urlsplit(target).scheme not in ("http", "https"): + continue + if target == self_url: + continue + host = (urlsplit(target).hostname or "").lower() + if is_social_host(host): + kind = "social" + elif host_of(target) == base_host: + kind = "internal" + else: + kind = "external" + if not target: continue - if target == self_url or target in seen: - continue - seen.add(target) - links.append(target) - return links + + text = _anchor_label(anchor)[:_MAX_ANCHOR_TEXT] + record = { + "url": target, + "text": text, + "rel": str(anchor.get("rel", "")).strip(), + "kind": kind, + } + # Context only where entity attribution matters; internal/external nav + # context is boilerplate that would bloat every item. + if kind in ("social", "email", "tel"): + record["context"] = _anchor_context(anchor) if not text else "" + existing = records.get(target) + if existing is None: + records[target] = record + elif not existing["text"] and text: + existing["text"] = text + if "context" in existing: + existing["context"] = "" + return list(records.values()) -def canonicalize_url(url: str) -> str: - """Stable visited-set key: sorts query, normalizes encoding, drops fragment.""" - return _w3lib_canonicalize_url(url, keep_fragments=False) +def extract_links(page_html: str | None, base_url: str) -> list[str]: + """Absolute, http(s), fragment-free, de-duplicated ```` targets. + + URL-only view of ``extract_link_records`` for callers that just need the + frontier; first-seen order is preserved to keep it stable. + """ + return [ + record["url"] + for record in extract_link_records(page_html, base_url) + if record["kind"] not in ("email", "tel") + ] def host_of(url: str) -> str: """Lowercased host with a leading ``www.`` removed, for same-site matching.""" host = (urlsplit(url).hostname or "").lower() return host[4:] if host.startswith("www.") else host - - -def same_site(url: str, allowed_hosts: set[str]) -> bool: - """Whether ``url``'s host (``www.``-normalized) is in ``allowed_hosts``.""" - return host_of(url) in allowed_hosts diff --git a/surfsense_backend/app/utils/crawl/__init__.py b/surfsense_backend/app/utils/crawl/__init__.py index 66a38ce4f..3f4762414 100644 --- a/surfsense_backend/app/utils/crawl/__init__.py +++ b/surfsense_backend/app/utils/crawl/__init__.py @@ -11,8 +11,12 @@ the proprietary boundary in ``app/proprietary/web_crawler/`` (``stealth.py``). """ from app.utils.crawl.classifier import BlockType, classify_block +from app.utils.crawl.contacts import Contacts, extract_contacts, is_social_host __all__ = [ "BlockType", + "Contacts", "classify_block", + "extract_contacts", + "is_social_host", ] diff --git a/surfsense_backend/app/utils/crawl/contacts.py b/surfsense_backend/app/utils/crawl/contacts.py new file mode 100644 index 000000000..b487b10f1 --- /dev/null +++ b/surfsense_backend/app/utils/crawl/contacts.py @@ -0,0 +1,229 @@ +"""Pure contact/social-signal extraction from raw HTML (Apache-2.0, generic). + +Lead-gen / competitive-intelligence crawls need the emails, phone numbers, and +social profiles a site publishes — which almost always live in the footer, the +contact page, or the privacy/terms pages. Trafilatura's main-content extraction +deliberately drops that boilerplate, so these signals must be pulled from the +raw HTML, not the cleaned markdown. + +No I/O and no bypass logic, so this sits in the generic ``app/utils/crawl`` +package (mirrors ``classifier``) and is consumed by the proprietary connector. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from urllib.parse import unquote, urldefrag, urlsplit + +from lxml import html as lxml_html +from lxml.etree import ParserError + +# Social/profile hosts worth surfacing as leads. Matched on host == d or a +# subdomain of d. ``x.com``/``twitter.com`` both kept (rename churn). +_SOCIAL_HOSTS = ( + "twitter.com", + "x.com", + "linkedin.com", + "facebook.com", + "fb.com", + "instagram.com", + "youtube.com", + "youtu.be", + "github.com", + "gitlab.com", + "tiktok.com", + "discord.com", + "discord.gg", + "t.me", + "medium.com", + "threads.net", + "pinterest.com", + "reddit.com", + "crunchbase.com", + "wellfound.com", + "angel.co", + "mastodon.social", + "bsky.app", + # Regional networks — the primary business contact channel in much of the + # world (WhatsApp: LatAm/India/Africa; Line: JP/TH/TW; VK/OK: RU; + # Weibo/WeChat: CN; Xing: DACH; Kakao: KR). + "wa.me", + "whatsapp.com", + "line.me", + "lin.ee", + "vk.com", + "ok.ru", + "weibo.com", + "weixin.qq.com", + "xing.com", + "pf.kakao.com", +) + +# Email domains that are almost never a real contact (SDKs, CDNs, examples). +_NOISE_EMAIL_DOMAINS = frozenset( + { + "sentry.io", + "wixpress.com", + "example.com", + "example.org", + "domain.com", + "email.com", + # Unambiguous placeholder domains; ambiguous ones (business.com, + # company.com) are left to the placeholder local-part filter instead. + "yourcompany.com", + "yourdomain.com", + "yoursite.com", + "schema.org", + "w3.org", + "googleapis.com", + "gstatic.com", + "sentry-cdn.com", + "cloudflare.com", + } +) + +# File extensions that surface as bogus email "TLDs" when an asset ref (``logo@2x.png``) +# or version-pinned dep (``react@18.2.0.js``) matches the email shape. +_ASSET_TLDS = frozenset( + { + "png", "jpg", "jpeg", "gif", "svg", "webp", "ico", "bmp", + "css", "js", "mjs", "cjs", "ts", "map", "json", "xml", + "woff", "woff2", "ttf", "eot", "otf", "php", "html", "htm", + } +) + +_EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}") + +# Template/form placeholders, compared after stripping [._-] separators, so +# "your.email"/"your-email"/"youremail" all match. Deliberately excludes real +# common locals like hello/info/contact/support. +_PLACEHOLDER_EMAIL_LOCALS = frozenset( + { + "youremail", "yourname", "youraddress", "myemail", "email", "name", + "user", "username", "someone", "somebody", "johndoe", "janedoe", + "firstname", "lastname", "firstnamelastname", "firstlast", + "test", "example", "sample", "placeholder", + } +) + +# Placeholder profile handles left in site templates ("github.com/username"). +_PLACEHOLDER_SOCIAL_SEGMENTS = frozenset( + { + "username", "yourusername", "yourhandle", "handle", "user", + "profile", "yourprofile", "yourname", "yourpage", "pagename", + "youraccount", "account", "example", "placeholder", "yourcompany", + } +) + +_SEPARATORS_RE = re.compile(r"[._\-]+") + + +def _normalized(token: str) -> str: + return _SEPARATORS_RE.sub("", token.strip().lower().lstrip("@")) + + +@dataclass +class Contacts: + """Deduped contact signals harvested from one page's raw HTML.""" + + emails: list[str] = field(default_factory=list) + phones: list[str] = field(default_factory=list) + socials: list[str] = field(default_factory=list) + + def as_dict(self) -> dict[str, list[str]]: + return {"emails": self.emails, "phones": self.phones, "socials": self.socials} + + @property + def is_empty(self) -> bool: + return not (self.emails or self.phones or self.socials) + + +def is_social_host(host: str) -> bool: + """True when ``host`` is (a subdomain of) a known social/profile host.""" + return any(host == d or host.endswith("." + d) for d in _SOCIAL_HOSTS) + + +def _keep_email(email: str) -> bool: + local, _, domain = email.partition("@") + domain = domain.lower() + if domain in _NOISE_EMAIL_DOMAINS: + return False + if _normalized(local) in _PLACEHOLDER_EMAIL_LOCALS: + return False + # Drops asset/version false positives like ``logo@2x.png`` / ``react@18.2.0.js`` + # whose trailing token is a file extension, not a real TLD. + return domain.rsplit(".", 1)[-1] not in _ASSET_TLDS + + +def _keep_social(url: str) -> bool: + # ponytail: any placeholder-looking path segment drops the URL; a real + # handle literally named "username"/"example" is collateral. Upgrade path: + # per-host handle position rules (e.g. linkedin.com/in/). + return not any( + _normalized(segment) in _PLACEHOLDER_SOCIAL_SEGMENTS + for segment in urlsplit(url).path.split("/") + if segment + ) + + +def _dedup(values: list[str]) -> list[str]: + """Case-insensitive dedupe that preserves first-seen order.""" + seen: set[str] = set() + out: list[str] = [] + for value in values: + key = value.lower() + if key not in seen: + seen.add(key) + out.append(value) + return out + + +def extract_contacts(raw_html: str | None) -> Contacts: + """Harvest emails, phone numbers, and social profile URLs from raw HTML. + + Emails come from ``mailto:`` hrefs (high confidence) and a plaintext scan of + the source (noise-filtered). Phones come only from ``tel:`` hrefs — a text + scan for phone numbers is too noisy to be worth it. Socials are ``href`` + targets on known profile hosts. Any parse error yields empty results rather + than aborting the crawl. + """ + if not raw_html or not raw_html.strip(): + return Contacts() + + emails: list[str] = [] + phones: list[str] = [] + socials: list[str] = [] + + try: + root = lxml_html.fromstring(raw_html) + except (ParserError, ValueError): + root = None + + if root is not None: + for href in root.xpath("//a/@href | //link/@href"): + href = str(href).strip() + low = href.lower() + # unquote: hrefs URL-encode spaces etc. ("tel:+1%20408-629-1770") + if low.startswith("mailto:"): + addr = unquote(urlsplit(href).path.split("?")[0]).strip() + if addr: + emails.append(addr) + elif low.startswith("tel:"): + num = unquote(urlsplit(href).path).strip() + if num: + phones.append(num) + elif low.startswith(("http://", "https://")): + host = (urlsplit(href).hostname or "").lower() + if is_social_host(host): + socials.append(urldefrag(href)[0]) + + # Plaintext email scan over the source catches addresses rendered as text + # (e.g. "hello@site.com" in a footer) that never appear as a mailto href. + emails.extend(_EMAIL_RE.findall(raw_html)) + + return Contacts( + emails=[e for e in _dedup(emails) if _keep_email(e)], + phones=_dedup(phones), + socials=[s for s in _dedup(socials) if _keep_social(s)], + ) diff --git a/surfsense_backend/tests/unit/capabilities/test_run_truncation.py b/surfsense_backend/tests/unit/capabilities/test_run_truncation.py index 5a7d58f78..43f1519be 100644 --- a/surfsense_backend/tests/unit/capabilities/test_run_truncation.py +++ b/surfsense_backend/tests/unit/capabilities/test_run_truncation.py @@ -127,7 +127,9 @@ def _patch_session(monkeypatch, value, calls): def _tools(): - read_run, search_run = run_reader.build_run_reader_tools(workspace_id=7) + read_run, search_run, _export_run = run_reader.build_run_reader_tools( + workspace_id=7 + ) return read_run, search_run @@ -150,6 +152,46 @@ async def test_read_run_paginates(monkeypatch): assert "workspace_id" in calls[0] +@pytest.mark.asyncio +async def test_read_run_char_offset_pages_inside_one_huge_item(monkeypatch): + """A single item bigger than the cap is fully reachable via char_offset.""" + huge_line = "A" * RUN_OUTPUT_CHAR_CAP + "MARKER" + "B" * 1000 + _patch_session(monkeypatch, huge_line, []) + read_run, _ = _tools() + ref = "run_" + "0" * 8 + "-0000-0000-0000-000000000000" + + first = await read_run.ainvoke({"ref": ref, "offset": 0, "limit": 1}) + assert "MARKER" not in first # clipped at the cap + assert f"char_offset={RUN_OUTPUT_CHAR_CAP}" in first # continuation hint + + second = await read_run.ainvoke( + {"ref": ref, "offset": 0, "limit": 1, "char_offset": RUN_OUTPUT_CHAR_CAP} + ) + assert "MARKER" in second + assert "truncated" not in second # remainder fits + + past_end = await read_run.ainvoke( + {"ref": ref, "offset": 0, "limit": 1, "char_offset": len(huge_line) + 5} + ) + assert "No content at char_offset" in past_end + + +@pytest.mark.asyncio +async def test_search_run_excerpts_huge_matched_line(monkeypatch): + """A match inside a huge line returns a window around it, not the whole line.""" + huge_line = "x" * 100_000 + "NEEDLE" + "y" * 100_000 + _patch_session(monkeypatch, huge_line, []) + _, search_run = _tools() + + out = await search_run.ainvoke( + {"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", + "pattern": "NEEDLE"} + ) + assert "NEEDLE" in out + assert "match at char 100000" in out + assert len(out) < 2000 # excerpt, not the 200k line + + @pytest.mark.asyncio async def test_read_run_rejects_bad_ref(monkeypatch): _patch_session(monkeypatch, _BODY, []) @@ -180,6 +222,97 @@ async def test_search_run_matches(monkeypatch): assert "item_1" not in out.split("item_7")[0] +# --- export_run ------------------------------------------------------------ + + +_CRAWL_BODY = "\n".join( + [ + json.dumps( + { + "url": "https://x.com/team/", + "status": "success", + "links": [ + {"url": "https://x.com/author/jane/", "text": "Jane Doe", + "context": "Jane Doe General Partner", "kind": "internal"}, + {"url": "https://x.com/author/bob/", "text": "Bob Roe", + "context": "Bob Roe Operations", "kind": "internal"}, + # Duplicate of Jane (nav + card) — must dedupe. + {"url": "https://x.com/author/jane/", "text": "Jane Doe", + "context": "Jane Doe General Partner", "kind": "internal"}, + {"url": "https://x.com/about/", "text": "About", "kind": "internal"}, + ], + } + ), + json.dumps({"url": "https://x.com/jobs/", "status": "failed", "links": []}), + "not json — skipped", + ] +) + + +def test_rows_from_body_links_explode_and_items(): + links = run_reader._rows_from_body(_CRAWL_BODY, "links") + assert len(links) == 4 + assert links[0]["page"] == "https://x.com/team/" + assert links[0]["text"] == "Jane Doe" + + items = run_reader._rows_from_body(_CRAWL_BODY, "items") + assert [i["url"] for i in items] == ["https://x.com/team/", "https://x.com/jobs/"] + + +def test_rows_to_csv_dedupes_and_orders_columns(): + records = run_reader._rows_from_body(_CRAWL_BODY, "links") + csv_text, count = run_reader._rows_to_csv(records, ["page", "url", "text"]) + lines = csv_text.strip().split("\n") + assert lines[0] == "page,url,text" + assert count == 3 # 4 records - 1 duplicate + assert len(lines) == 4 # header + 3 rows + assert "Jane Doe" in lines[1] + + +@pytest.mark.asyncio +async def test_export_run_filters_and_saves(monkeypatch): + _patch_session(monkeypatch, _CRAWL_BODY, []) + saved: dict = {} + + async def _fake_save(*, virtual_path, content, workspace_id): + saved["path"] = virtual_path + saved["content"] = content + saved["workspace_id"] = workspace_id + return 42, "/documents/exports/team.csv" + + monkeypatch.setattr(run_reader, "_save_export_document", _fake_save) + _, _, export_run = run_reader.build_run_reader_tools(workspace_id=7) + + out = await export_run.ainvoke( + { + "ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", + "path": "exports/team.csv", + "rows": "links", + "include_pattern": "/author/", + } + ) + assert "Exported 2 rows" in out # Jane + Bob; About filtered; dupe deduped + assert "/documents/exports/team.csv" in out + assert "document id 42" in out + assert saved["workspace_id"] == 7 + assert "About" not in saved["content"] + assert "Bob Roe" in saved["content"] + + +@pytest.mark.asyncio +async def test_export_run_empty_filter_is_error(monkeypatch): + _patch_session(monkeypatch, _CRAWL_BODY, []) + _, _, export_run = run_reader.build_run_reader_tools(workspace_id=7) + out = await export_run.ainvoke( + { + "ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", + "path": "exports/none.csv", + "include_pattern": "no-such-thing-anywhere", + } + ) + assert out.startswith("Error: no rows to export") + + @pytest.mark.asyncio async def test_search_run_falls_back_on_bad_regex(monkeypatch): _patch_session(monkeypatch, _BODY, []) diff --git a/surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py b/surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py index 2d550abbc..d26846f93 100644 --- a/surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py +++ b/surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py @@ -94,6 +94,55 @@ async def test_failed_page_has_no_markdown_but_keeps_error() -> None: assert item.error == "boom" +async def test_aggregated_contacts_carry_provenance_and_site_wide_flag() -> None: + footer = "https://linkedin.com/company/e" + person = "https://linkedin.com/in/jane" + + class _ContactsEngine: + async def crawl_url(self, url: str) -> CrawlOutcome: + socials = [footer] + ([person] if url.endswith("/about") else []) + links = ["https://e.com/about", "https://e.com/blog"] if url == "https://e.com/" else [] + return CrawlOutcome( + status=_SUCCESS, + result={ + "content": "ok", + "metadata": {}, + "links": links, + "contacts": {"emails": [], "phones": [], "socials": socials}, + }, + ) + + execute = build_crawl_executor(engine=_ContactsEngine()) + out = await execute( + CrawlInput(startUrls=["https://e.com/"], maxCrawlDepth=1, maxCrawlPages=10) + ) + + by_value = {ref.value: ref for ref in out.contacts.socials} + assert by_value[footer].siteWide # on all 3 pages -> boilerplate + assert by_value[footer].pageCount == 3 + assert not by_value[person].siteWide # only on /about -> page-local entity + assert by_value[person].pages == ["https://e.com/about"] + + +async def test_single_page_crawl_marks_contacts_site_wide() -> None: + class _OnePageEngine: + async def crawl_url(self, url: str) -> CrawlOutcome: + return CrawlOutcome( + status=_SUCCESS, + result={ + "content": "ok", + "metadata": {}, + "links": [], + "contacts": {"emails": ["a@e.com"], "phones": [], "socials": []}, + }, + ) + + execute = build_crawl_executor(engine=_OnePageEngine()) + out = await execute(CrawlInput(startUrls=["https://e.com/"])) + + assert out.contacts.emails[0].siteWide # one page: no signal to split on + + async def test_captcha_telemetry_is_rolled_up_for_billing() -> None: class _CaptchaEngine: async def crawl_url(self, url: str) -> CrawlOutcome: diff --git a/surfsense_backend/tests/unit/platforms/reddit/test_search_budget.py b/surfsense_backend/tests/unit/platforms/reddit/test_search_budget.py new file mode 100644 index 000000000..ce58199c8 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/reddit/test_search_budget.py @@ -0,0 +1,69 @@ +"""Offline tests for multi-search budgeting in ``iter_reddit``. + +No network: ``_search_flow`` is faked. Asserts the maxItems budget is +fair-shared across concurrent searches (a noisy query can't starve the rest) +and that the same post surfacing via several queries is emitted once. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator + +from app.proprietary.platforms.reddit import scraper +from app.proprietary.platforms.reddit.schemas import RedditScrapeInput + + +def _fake_search_flow(results_by_query: dict[str, list[str]]): + """Fake flow yielding post dicts (id per entry), honoring ``max_items``.""" + calls: dict[str, int] = {} + + def flow( + query: str, + *, + input_model: RedditScrapeInput, + subreddit: str | None = None, + max_items: int | None = None, + ) -> AsyncIterator[dict]: + cap = input_model.maxItems if max_items is None else max_items + calls[query] = cap + + async def gen() -> AsyncIterator[dict]: + for pid in results_by_query.get(query, [])[:cap]: + yield {"dataType": "post", "id": pid, "title": pid} + + return gen() + + return flow, calls + + +async def test_budget_is_fair_shared_across_searches(monkeypatch): + # One noisy query with 100 hits must not starve the two precise ones. + data = { + "noisy": [f"n{i}" for i in range(100)], + "precise_a": ["a1", "a2", "a3"], + "precise_b": ["b1", "b2"], + } + flow, calls = _fake_search_flow(data) + monkeypatch.setattr(scraper, "_search_flow", flow) + + model = RedditScrapeInput(searches=list(data), maxItems=30) + items = await scraper.scrape_reddit(model, limit=30) + + ids = {i["id"] for i in items} + # Every precise result made it in; noisy filled only its ceil(30/3)=10 share. + assert {"a1", "a2", "a3", "b1", "b2"} <= ids + assert sum(1 for i in ids if i.startswith("n")) == 10 + assert all(cap == 10 for cap in calls.values()) + + +async def test_duplicate_posts_across_searches_emit_once(monkeypatch): + data = {"q1": ["dup", "x1"], "q2": ["dup", "x2"]} + flow, _ = _fake_search_flow(data) + monkeypatch.setattr(scraper, "_search_flow", flow) + + model = RedditScrapeInput(searches=["q1", "q2"], maxItems=10) + items = await scraper.scrape_reddit(model, limit=10) + + ids = [i["id"] for i in items] + assert ids.count("dup") == 1 + assert {"x1", "x2"} <= set(ids) diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py b/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py index f33d8c007..8db7c1ed1 100644 --- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py +++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py @@ -86,6 +86,100 @@ async def test_escalates_to_dynamic_on_static_miss( assert outcome.tier == "scrapling-dynamic" +def test_dropped_currency_amounts_detection() -> None: + """Fires only when the DOM has currency figures that the markdown lost.""" + dropped = connector_module.dropped_currency_amounts + html = "

" + assert dropped(html, "Pro plan without figures") + assert not dropped(html, "Pro plan $49/mo") # markdown kept it + assert not dropped("no prices here", "text") + # Country-agnostic: symbol-after-amount and ISO codes count too. + assert dropped("ab 49€ pro Monat", "ab pro Monat") + assert dropped("USD 2,500 per year", "per year") + # Script content is not visible: a JSON payload price must not trigger. + assert not dropped( + "hi", "hi" + ) + + +def test_build_result_repairs_pricing_card_loss() -> None: + """div-grid pricing cards dropped by trafilatura get recovered.""" + cards = "".join( + f"

{name}

${price}
" + f"
  • feature a
  • feature b
" + f"Choose {name}
" + for name, price in (("Free", 0), ("Pro", 49), ("Enterprise", 199)) + ) + html = ( + "Pricing" + "

Simple pricing

" + + "Choose the plan that fits your team best. " * 30 + + "

" + + cards + + "
" + ) + result = WebCrawlerConnector()._build_result( + html, "https://x.com/pricing", "t", allow_raw_fallback=False + ) + assert result is not None + assert "$49" in result["content"] + assert "$199" in result["content"] + + +def test_looks_like_js_shell_thresholds() -> None: + """Shell = huge HTML AND near-empty extraction; either alone is healthy.""" + shell = connector_module.looks_like_js_shell + assert shell(4_200_000, 597) # a16z.com/team + assert not shell(200_000, 13_092) # a16z investment-list: normal page + assert not shell(45_000, 1_356) # small brochure page: small is not thin + assert not shell(4_200_000, 50_000) # huge but content-rich (long article) + + +async def test_thin_static_shell_escalates_to_dynamic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A JS-shell static result escalates; the hydrated dynamic result wins.""" + crawler = WebCrawlerConnector() + + async def _thin_static(_url: str, *_args) -> dict: + return _result("scrapling-static") | {"thin_static": True} + + async def _dynamic(_url: str, *_args) -> dict: + return _result("scrapling-dynamic") + + monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _thin_static) + monkeypatch.setattr(crawler, "_crawl_with_dynamic", _dynamic) + + outcome = await crawler.crawl_url("https://example.com") + + assert outcome.status is CrawlOutcomeStatus.SUCCESS + assert outcome.tier == "scrapling-dynamic" + + +async def test_thin_static_is_fallback_when_browser_tiers_fail( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Browser tiers unavailable -> the partial static content still returns.""" + crawler = WebCrawlerConnector() + + async def _thin_static(_url: str, *_args) -> dict: + return _result("scrapling-static") | {"thin_static": True} + + async def _unavailable(_url: str, *_args) -> None: + raise NotImplementedError("no subprocess support") + + monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _thin_static) + monkeypatch.setattr(crawler, "_crawl_with_dynamic", _unavailable) + monkeypatch.setattr(crawler, "_crawl_with_stealthy", _unavailable) + + outcome = await crawler.crawl_url("https://example.com") + + assert outcome.status is CrawlOutcomeStatus.SUCCESS + assert outcome.tier == "scrapling-static" + assert outcome.result is not None + assert "thin_static" not in outcome.result # internal tag never leaks + + async def test_all_tiers_empty_is_empty(monkeypatch: pytest.MonkeyPatch) -> None: """Every tier fetched but extracted nothing -> EMPTY (not billable).""" crawler = WebCrawlerConnector() @@ -345,6 +439,46 @@ async def test_static_4xx_is_classified( assert block_state["block_type"] is BlockType.CLOUDFLARE +class _FakeScrollPage: + """Playwright-page stand-in: height grows per scroll until a plateau.""" + + def __init__(self, heights: list[int]): + self._heights = heights + self._i = 0 + self.scrolls = 0 + + def evaluate(self, script: str): + if "scrollHeight" in script and "scrollTo" not in script: + return self._heights[min(self._i, len(self._heights) - 1)] + self.scrolls += 1 + self._i += 1 + return None + + def wait_for_timeout(self, _ms: int) -> None: + pass + + +def test_scroll_to_bottom_stops_when_height_stops_growing() -> None: + page = _FakeScrollPage([1000, 2000, 3000, 3000]) + assert connector_module.scroll_to_bottom(page) is page + assert page.scrolls == 3 # scrolled at 1000/2000/3000; 3000-again broke the loop + + +def test_scroll_to_bottom_is_bounded_on_endless_feeds() -> None: + page = _FakeScrollPage([i * 1000 for i in range(1, 100)]) # never stabilizes + connector_module.scroll_to_bottom(page) + assert page.scrolls == connector_module._SCROLL_MAX_ROUNDS + + +def test_scroll_to_bottom_swallows_page_errors() -> None: + class _Broken: + def evaluate(self, _script: str): + raise RuntimeError("target closed") + + page = _Broken() + assert connector_module.scroll_to_bottom(page) is page # never raises + + def test_build_result_ok_on_real_content() -> None: """03e: a normal 200 page with content classifies OK.""" crawler = WebCrawlerConnector() diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py b/surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py index ca5ad06fa..9ff7fec26 100644 --- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py +++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py @@ -5,10 +5,9 @@ from __future__ import annotations import pytest from app.proprietary.web_crawler.url_policy import ( - canonicalize_url, + extract_link_records, extract_links, host_of, - same_site, ) pytestmark = pytest.mark.unit @@ -60,30 +59,67 @@ def test_extract_links_on_empty_or_blank_html_is_empty() -> None: assert extract_links(None, "https://e.com") == [] -def test_canonicalize_lowercases_host_sorts_query_and_drops_fragment() -> None: - assert ( - canonicalize_url("https://E.com/a?b=2&a=1#frag") == "https://e.com/a?a=1&b=2" - ) +def test_extract_link_records_classifies_kinds_and_keeps_anchor_text() -> None: + html = """ + About\n us + External + Jane Doe + Mail + Call + """ + records = {r["url"]: r for r in extract_link_records(html, "https://example.com/")} + assert records["https://example.com/about"]["kind"] == "internal" + assert records["https://example.com/about"]["text"] == "About us" # collapsed ws + assert records["https://other.com/x"]["kind"] == "external" + assert records["https://www.linkedin.com/in/jane"] == { + "url": "https://www.linkedin.com/in/jane", + "text": "Jane Doe", + "context": "", + "rel": "", + "kind": "social", + } + assert records["a@b.com"]["kind"] == "email" # mailto query stripped + assert records["+1-555-0100"]["kind"] == "tel" -def test_canonicalize_collapses_fragment_and_empty_query_to_one_key() -> None: - # The three forms must dedupe to the same visited-set key. - canonical = canonicalize_url("https://e.com/a") - assert canonicalize_url("https://e.com/a#frag") == canonical - assert canonicalize_url("https://e.com/a?") == canonical +def test_percent_encoded_tel_and_mailto_are_decoded() -> None: + """Seen live: must not leak %20.""" + html = """ + Call + Email + """ + records = {r["kind"]: r for r in extract_link_records(html, "https://example.com/")} + assert records["tel"]["url"] == "+1 408-629-1770" + assert records["email"]["url"] == "hello@acme.io" -def test_canonicalize_keeps_nondefault_port() -> None: - assert canonicalize_url("https://e.com:8443/x") == "https://e.com:8443/x" +def test_icon_only_social_link_gets_ancestor_context() -> None: + html = """ +
+

Jane Doe

General Partner

+ +
+ """ + (record,) = extract_link_records(html, "https://example.com/") + assert record["text"] == "" + assert record["context"] == "Jane Doe General Partner" + + +def test_icon_social_link_prefers_aria_label_over_context() -> None: + html = '
Footer
' + (record,) = extract_link_records(html, "https://example.com/") + assert record["text"] == "Acme on X" + assert record["context"] == "" + + +def test_extract_link_records_dedupes_keeping_first_nonempty_text() -> None: + html = 'Pricing' + records = extract_link_records(html, "https://example.com/") + assert records == [ + {"url": "https://example.com/p", "text": "Pricing", "rel": "", "kind": "internal"} + ] def test_host_of_strips_www_and_lowercases() -> None: assert host_of("https://www.Example.com/x") == "example.com" assert host_of("https://Example.com/x") == "example.com" - - -def test_same_site_matches_on_normalized_host() -> None: - allowed = {"example.com"} - assert same_site("https://www.example.com/a", allowed) is True - assert same_site("https://example.com/b", allowed) is True - assert same_site("https://other.com/c", allowed) is False diff --git a/surfsense_backend/tests/unit/utils/crawl/test_contacts.py b/surfsense_backend/tests/unit/utils/crawl/test_contacts.py new file mode 100644 index 000000000..c143ed820 --- /dev/null +++ b/surfsense_backend/tests/unit/utils/crawl/test_contacts.py @@ -0,0 +1,103 @@ +"""``extract_contacts`` behavior: harvest emails/phones/socials from raw HTML.""" + +from __future__ import annotations + +import pytest + +from app.utils.crawl import extract_contacts + +pytestmark = pytest.mark.unit + + +def test_harvests_mailto_tel_and_socials() -> None: + html = """ + + + + """ + c = extract_contacts(html) + assert c.emails == ["hello@acme.io"] # mailto query stripped + assert c.phones == ["+1-555-0100"] + assert c.socials == [ + "https://www.linkedin.com/company/acme", + "https://x.com/acme", + "https://github.com/acme/repo", # fragment stripped + ] + # Same-site, non-social link is not a contact signal. + assert "https://acme.io/about" not in c.socials + + +def test_plaintext_email_without_mailto_is_found() -> None: + html = "

Reach us at hello@cochat.ai for support.

" + assert extract_contacts(html).emails == ["hello@cochat.ai"] + + +def test_filters_noise_emails_and_asset_false_positives() -> None: + html = """ + + + +

ops@sentry.io

+ x + + """ + assert extract_contacts(html).emails == ["real@company.com"] + + +def test_filters_template_placeholders() -> None: + html = """ + +

youremail@business.com your.email@acme.io john-doe@acme.io

+ real + gh template + tw template + real person + + """ + c = extract_contacts(html) + assert c.emails == ["hello@acme.io"] # your.email + john-doe normalized away + assert c.socials == ["https://www.linkedin.com/in/jane-doe/"] + + +def test_regional_social_hosts_are_harvested() -> None: + """WhatsApp/Line/VK/Weibo etc. are the business contact channel outside the US.""" + html = """ + WhatsApp + Line + VK + Weibo + Xing + """ + assert len(extract_contacts(html).socials) == 5 + + +def test_percent_encoded_hrefs_are_decoded() -> None: + """Sites URL-encode tel/mailto hrefs (seen live: tel:+1%20408-629-1770).""" + html = """ + Call + Email + """ + c = extract_contacts(html) + assert c.phones == ["+1 408-629-1770"] + assert "hello@acme.io" in c.emails + + +def test_dedupes_case_insensitively_preserving_order() -> None: + html = """ + a + b + """ + assert extract_contacts(html).emails == ["Hello@Acme.io"] + + +def test_empty_or_unparseable_html_is_empty() -> None: + assert extract_contacts("").is_empty + assert extract_contacts(None).is_empty + assert extract_contacts(" ").is_empty diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index ccd52a516..c46b8d4fc 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -30,6 +30,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -58,6 +61,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -86,6 +92,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -114,6 +123,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -142,6 +154,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -170,6 +185,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -198,6 +216,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -226,6 +247,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -254,6 +278,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -282,6 +309,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -319,6 +349,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -347,6 +381,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -375,6 +412,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -403,6 +443,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -431,6 +474,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -459,6 +505,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -487,6 +536,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -515,6 +567,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -543,6 +598,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -571,6 +629,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -599,6 +660,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -636,6 +700,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -664,6 +732,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -692,6 +763,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -720,6 +794,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -748,6 +825,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -776,6 +856,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -804,6 +887,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -832,6 +918,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -860,6 +949,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -888,6 +980,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -916,6 +1011,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -953,6 +1051,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -981,6 +1083,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1036,6 +1141,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1064,6 +1175,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1092,6 +1206,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1147,6 +1264,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1202,6 +1325,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1230,6 +1359,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", ] conflicts = [[ @@ -5164,19 +5296,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/98/cb5ca20618d205a09d5bec7591fbc4130369c7e6308d9a676a28ff3ab22c/limits-5.8.0-py3-none-any.whl", hash = "sha256:ae1b008a43eb43073c3c579398bd4eb4c795de60952532dc24720ab45e1ac6b8", size = 60954 }, ] -[[package]] -name = "linkup-sdk" -version = "0.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/fa/d54d7086ceb8e8aa3777ae82f9876ceb7d15a6f583bbebf2fc2bea4cf69c/linkup_sdk-0.13.0.tar.gz", hash = "sha256:dab3f516bb955bdb9dd5815445bfdc7a2c9803dc57c3b4be4038d9e40f3d096a", size = 76440 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/b8/9a8be932db54dc673c0a2f033831e9963cb4395352ce5a54a423e2fb58de/linkup_sdk-0.13.0-py3-none-any.whl", hash = "sha256:d4f5f4698cbaf4711a3296473f6030613c9c8ac829c83172a51c34c6e653808a", size = 11750 }, -] - [[package]] name = "litellm" version = "1.88.1" @@ -10140,7 +10259,6 @@ dependencies = [ { name = "langchain-unstructured" }, { name = "langgraph" }, { name = "langgraph-checkpoint-postgres" }, - { name = "linkup-sdk" }, { name = "litellm" }, { name = "llama-cloud-services" }, { name = "markdown" }, @@ -10181,7 +10299,6 @@ dependencies = [ { name = "starlette" }, { name = "static-ffmpeg" }, { name = "stripe" }, - { name = "tavily-python" }, { name = "tornado" }, { name = "trafilatura" }, { name = "typst" }, @@ -10258,7 +10375,6 @@ requires-dist = [ { name = "langchain-unstructured", specifier = ">=1.0.1" }, { name = "langgraph", specifier = ">=1.1.3" }, { name = "langgraph-checkpoint-postgres", specifier = ">=3.0.2" }, - { name = "linkup-sdk", specifier = ">=0.2.4" }, { name = "litellm", specifier = ">=1.83.7" }, { name = "llama-cloud-services", specifier = ">=0.6.25" }, { name = "markdown", specifier = ">=3.7" }, @@ -10299,7 +10415,6 @@ requires-dist = [ { name = "starlette", specifier = ">=0.40.0,<0.51.0" }, { name = "static-ffmpeg", specifier = ">=2.13" }, { name = "stripe", specifier = ">=15.0.0" }, - { name = "tavily-python", specifier = ">=0.3.2" }, { name = "torch", marker = "sys_platform == 'linux' and extra == 'cpu'", specifier = "==2.11.0", index = "https://download.pytorch.org/whl/cpu", conflict = { package = "surf-new-backend", extra = "cpu" } }, { name = "torch", marker = "sys_platform == 'linux' and extra == 'cu126'", specifier = "==2.11.0", index = "https://download.pytorch.org/whl/cu126", conflict = { package = "surf-new-backend", extra = "cu126" } }, { name = "torch", marker = "sys_platform == 'linux' and extra == 'cu128'", specifier = "==2.11.0", index = "https://download.pytorch.org/whl/cu128", conflict = { package = "surf-new-backend", extra = "cu128" } }, @@ -10353,20 +10468,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814 }, ] -[[package]] -name = "tavily-python" -version = "0.7.23" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "requests" }, - { name = "tiktoken" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/89/d1/197419d6133643848514e5e84e8f41886e825b73bf91ae235a1595c964f5/tavily_python-0.7.23.tar.gz", hash = "sha256:3b92232e0e29ab68898b765f281bb4f2c650b02210b64affbc48e15292e96161", size = 25968 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/27/f9c6e9249367be0772fb754849e03cbbc6ad8d80a479bf30ea8811828b2e/tavily_python-0.7.23-py3-none-any.whl", hash = "sha256:52ef85c44b926bce3f257570cd32bc1bd4db54666acf3105617f27411a59e188", size = 19079 }, -] - [[package]] name = "tenacity" version = "9.1.4"