feat: update environment variables and enhance scraping capabilities

- Adjusted Google Maps and YouTube micro pricing in the .env.example file for better cost management. - Introduced new environment variables for captcha solving and stealth browser hardening to improve scraping resilience. - Removed outdated smoke test for scraper API endpoints to streamline testing. - Enhanced anonymous chat agent's system prompt to clarify capabilities and suggest account creation for advanced features. - Updated Reddit fetch logic to prioritize new session handling and improve resilience against IP-related issues. - Added compacting functionality for scraper results to optimize data handling and presentation. - Improved workspace and document management tools with clearer descriptions and enhanced functionality. - Introduced new UI components for agent setup guidance in the web application.
2026-07-08 22:22:17 +02:00 · 2026-07-06 20:27:36 -07:00 · 2026-07-06 20:27:36 -07:00 · 1fd58752a3
commit 1fd58752a3
parent 271a21aee6
24 changed files with 1326 additions and 320 deletions
--- a/surfsense_backend/app/agents/chat/anonymous_chat/agent.py
+++ b/surfsense_backend/app/agents/chat/anonymous_chat/agent.py
@ -41,9 +41,12 @@ _MAX_DOC_CHARS = 50_000
 def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str:
    """Build the system prompt for the minimal anonymous chat agent.

-    The prompt keeps the assistant focused on plain Q/A + web search, inlines
-    any uploaded document as read-only context, and redirects every other
-    SurfSense feature to account registration.
+    The prompt keeps the assistant focused on plain Q/A from model knowledge,
+    inlines any uploaded document as read-only context, and treats the chat as
+    a registration funnel: every other SurfSense capability (scraping, live
+    data, deliverables, knowledge base, automations) redirects to sign-up, and
+    the assistant softly suggests an account when the conversation reveals a
+    competitive-intelligence need the platform serves.
    """
    today = datetime.now(UTC).strftime("%A, %B %d, %Y")

@ -70,7 +73,13 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str

    return (
        "You are SurfSense's free AI assistant, available to everyone without "
-        "login.\n\n"
+        "login. SurfSense is the open-source competitive intelligence platform: "
+        "registered users get specialist agents that pull live market data from "
+        "Reddit, YouTube, Google Maps, Google Search, and the open web, turn it "
+        "into cited briefs, reports, podcasts, and presentations, keep findings "
+        "in a searchable knowledge base, and run scheduled monitoring "
+        "automations — plus a REST scraping API and MCP server for their own "
+        "agents.\n\n"
        f"Today's date is {today}.\n\n"
        "## How to help\n"
        "- Answer the user's questions directly and conversationally. You are "
@ -78,23 +87,42 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str
        "- Answer from your own knowledge. You do NOT have web access here, so "
        "for current, real-time, or fast-changing facts (news, prices, "
        "weather, recent events, live data) say you can't look them up in the "
-        "free experience and may be out of date, and invite the user to create "
-        "a free account for live web search.\n"
+        "free experience and may be out of date.\n"
        "- Be concise, accurate, and helpful. Use Markdown formatting when it "
        "improves readability."
        f"{doc_section}\n\n"
        "## What is not available here\n"
-        "This is the free, no-login experience. You CANNOT search the web, save "
-        "files or notes, generate reports, podcasts, resumes, presentations, or "
-        "images, search or build a knowledge base, connect to apps (Gmail, "
-        "Google Drive, Notion, Slack, Calendar, Discord, and similar), set up "
-        "automations, or remember anything across sessions.\n\n"
+        "This is the free, no-login experience. You CANNOT search the web or "
+        "scrape any platform (Reddit, YouTube, Google Maps, Google Search, "
+        "websites), save files or notes, upload additional files, generate "
+        "reports, podcasts, resumes, presentations, or images, search or build "
+        "a knowledge base, connect to apps (Gmail, Google Drive, Notion, "
+        "Slack, Calendar, Discord, and similar), set up automations or "
+        "monitoring, use the scraping API or MCP server, or remember anything "
+        "across sessions.\n\n"
        "If the user asks for any of these, do NOT pretend to do them and "
-        "never claim you saved, created, or stored anything. Instead, briefly "
-        "let them know the feature requires a free SurfSense account and "
-        "invite them to create one at https://www.surfsense.com. Then offer to "
-        "help with what you can do here (answering questions from your own "
-        "knowledge and about any uploaded document)."
+        "never claim you saved, created, scraped, or stored anything. Instead, "
+        "briefly say that this needs a free SurfSense account and invite them "
+        "to sign up at https://www.surfsense.com/register — name the specific "
+        "capability that would solve their request (e.g. 'a SurfSense account "
+        "can scrape those Reddit threads live and turn them into a cited "
+        "brief'). Then offer to help with what you can do here (answering from "
+        "your own knowledge and about any uploaded document).\n\n"
+        "## Suggesting SurfSense when it genuinely fits\n"
+        "You are also the front door to SurfSense. When the conversation "
+        "reveals a need the full platform serves — researching competitors, "
+        "tracking pricing or rankings, monitoring brand mentions or reviews, "
+        "gauging Reddit/YouTube sentiment, generating leads, needing current "
+        "web data, or wanting recurring reports — first answer as well as you "
+        "can from your own knowledge, then add ONE short sentence pointing out "
+        "that a free SurfSense account can do that with live data, linking "
+        "https://www.surfsense.com/register.\n"
+        "- Be helpful first, never salesy: the answer is the product; the "
+        "suggestion is a footnote.\n"
+        "- At most one suggestion per response, and stop suggesting entirely "
+        "if the user declines or ignores it.\n"
+        "- Do not suggest it for needs SurfSense does not serve (casual chat, "
+        "coding help, homework, creative writing)."
    )


--- a/surfsense_backend/app/proprietary/platforms/reddit/fetch.py
+++ b/surfsense_backend/app/proprietary/platforms/reddit/fetch.py
@ -8,7 +8,7 @@ Reddit deprecated *cold* unauthenticated ``.json`` (a bare anonymous GET now
 ``scripts/e2e_reddit_scraper.py`` step 0) is:

    warm one anonymous session cookie (``loid``) with a plain GET to
-    ``old.reddit.com`` (``www.reddit.com/svc/shreddit/<slug>`` fallback), then
+    ``www.reddit.com/svc/shreddit/<slug>`` (``old.reddit.com`` fallback), then
    GET ``www.reddit.com/<path>/.json?raw_json=1`` through that same
    Chrome-impersonated, sticky-IP session. Which warm URL mints ``loid`` is
    exit-IP dependent, so the order is a tiebreak — see :func:`warm_session`.
@ -84,6 +84,12 @@ _BACKOFF_BASE_S = 5.0
 _MIN_INTERVAL_S = 0.5
 _PACE_JITTER_S = 0.25

+# curl's default is 30s, so one dead sticky IP stalled a whole run for 30-50s
+# (seen live 2026-07-06). A healthy fetch lands in ~1s; cap at 10s so a dead IP
+# costs one bounded wait, then the timeout falls into the generic exception
+# branch of fetch_json and rotates to a fresh IP — same treatment as a 403.
+_REQUEST_TIMEOUT_S = 10.0
+
 _HEADERS = {"Accept-Language": "en-US,en;q=0.9"}

 # Age-gate opt-in, sent on every ``.json`` fetch so NSFW listings aren't blanked
@ -170,7 +176,10 @@ class _RotatingSession:
            self._cm = self.session = None
            return
        self._cm = FetcherSession(
-            proxy=proxy, stealthy_headers=True, impersonate="chrome"
+            proxy=proxy,
+            stealthy_headers=True,
+            impersonate="chrome",
+            timeout=_REQUEST_TIMEOUT_S,
        )
        self.session = await self._cm.__aenter__()

@ -234,16 +243,15 @@ async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool:
    Returns ``True`` when a ``loid`` was issued (the session can now reach
    ``.json``), else ``False`` (caller rotates the IP and retries).

-    Tries ``old.reddit`` (yt-dlp's primary), then ``svc/shreddit``. WHICH one
-    mints is exit-IP dependent and roughly random: live probes 2026-07-04 saw
-    both directions across sessions on the rotating residential/custom proxy
-    (one IP 403s ``old.reddit`` but mints on ``shreddit``; another does the
-    reverse, sometimes with an ``rdt`` bot-interstitial). So the order is a
-    tiebreak, not an optimization — a fresh session pays ~one wasted warm 403
-    either way. That cost is amortized: ``fan_out`` reuses one warmed session
-    per worker across many jobs, so warm-up runs once per worker, not per fetch.
-    The fallback is what actually matters — it preserves correctness whichever
-    way a given IP leans.
+    Tries ``svc/shreddit`` first, then ``old.reddit`` (yt-dlp's primary) as
+    the fallback. Live probes 2026-07-04 saw both directions across exit IPs,
+    but a live run 2026-07-06 had ``old.reddit`` 403 on 12/12 fresh IPs while
+    ``shreddit`` minted every time — Reddit appears to have shut old.reddit to
+    anonymous traffic, so shreddit-first saves one guaranteed-403 round trip
+    per warm-up. The fallback is what actually matters — it preserves
+    correctness if a given IP (or Reddit) flips back the other way. That cost
+    is amortized: ``fan_out`` reuses one warmed session per worker across many
+    jobs, so warm-up runs once per worker, not per fetch.

    ponytail: sequential two-source warm burns 1 wasted request on ~half of new
    sessions. A parallel warm (gather both, take whichever mints) removes the
@ -256,14 +264,14 @@ async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool:
    """
    seen: set[str] = set()
    with suppress(Exception):
-        page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS)
+        page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS)
        seen |= _response_cookie_names(page)
    if _LOID_COOKIE in seen:
        return True

-    # Fallback: mints loid on exit IPs where old.reddit 403s instead.
+    # Fallback: mints loid on exit IPs where shreddit 403s instead.
    with suppress(Exception):
-        page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS)
+        page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS)
        seen |= _response_cookie_names(page)
    return _LOID_COOKIE in seen

@ -278,6 +286,7 @@ async def _get_page(session: Any, url: str) -> Any:
        cookies=_OVER18_COOKIES,
        proxy=get_proxy_url(),
        stealthy_headers=True,
+        timeout=_REQUEST_TIMEOUT_S,
    )


--- a/surfsense_backend/test_scraper_api_smoke.py
+++ b/surfsense_backend/test_scraper_api_smoke.py
@ -1,106 +0,0 @@
-"""One-shot smoke test: hit every scraper API endpoint with a PAT, print PASS/FAIL."""
-
-import json
-import os
-import sys
-import time
-
-import httpx
-
-BASE = "http://localhost:8000/api/v1/workspaces/12/scrapers"
-PAT = os.environ["SURFSENSE_PAT"]  # export a ss_pat_... key before running
-HEADERS = {"Authorization": f"Bearer {PAT}", "Content-Type": "application/json"}
-
-# Minimal payloads: 1-3 items each to keep credit spend tiny.
-VERBS = [
-    ("google_search/scrape", {"queries": ["surfsense github"], "max_pages_per_query": 1}),
-    ("web/crawl", {"startUrls": ["https://example.com"], "maxCrawlDepth": 0}),
-    ("reddit/scrape", {"urls": ["https://www.reddit.com/r/Python/"], "max_items": 3}),
-    ("youtube/scrape", {"search_queries": ["python tutorial"], "max_results": 1}),
-    (
-        "youtube/comments",
-        {"urls": ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"], "max_comments": 3},
-    ),
-    (
-        "google_maps/scrape",
-        {"search_queries": ["coffee shop"], "location": "New York, USA", "max_places": 1},
-    ),
-    (
-        "google_maps/reviews",
-        # Google Sydney office, a stable well-known place id.
-        {"place_ids": ["ChIJN1t_tDeuEmsRUsoyG83frY4"], "max_reviews": 3},
-    ),
-]
-
-results = []
-client = httpx.Client(headers=HEADERS, timeout=300)
-sync_run_id = None
-
-for verb, payload in VERBS:
-    t0 = time.time()
-    try:
-        r = client.post(f"{BASE}/{verb}", json=payload)
-        dur = time.time() - t0
-        run_id = r.headers.get("x-run-id")
-        if r.status_code == 200:
-            body = r.json()
-            # Count items in whatever list field the output has.
-            items = next((len(v) for v in body.values() if isinstance(v, list)), "?")
-            results.append((verb, "PASS", f"{r.status_code} items={items} run={run_id} {dur:.1f}s"))
-            if sync_run_id is None and run_id:
-                sync_run_id = run_id
-        else:
-            results.append((verb, "FAIL", f"{r.status_code} {r.text[:200]} {dur:.1f}s"))
-    except Exception as e:
-        results.append((verb, "FAIL", f"{type(e).__name__}: {e}"))
-    print(f"[{results[-1][1]}] {verb}: {results[-1][2]}", flush=True)
-
-# --- Run history endpoints ---
-r = client.get(f"{BASE}/runs", params={"limit": 5})
-ok = r.status_code == 200 and isinstance(r.json(), list)
-results.append(("GET runs (list)", "PASS" if ok else "FAIL", f"{r.status_code} rows={len(r.json()) if ok else '?'}"))
-print(f"[{results[-1][1]}] GET runs: {results[-1][2]}", flush=True)
-
-if sync_run_id:
-    r = client.get(f"{BASE}/runs/{sync_run_id}")
-    ok = r.status_code == 200 and r.json().get("id") == sync_run_id
-    results.append(("GET runs/{id} (detail)", "PASS" if ok else "FAIL", f"{r.status_code} status={r.json().get('status') if ok else r.text[:100]}"))
-    print(f"[{results[-1][1]}] GET run detail: {results[-1][2]}", flush=True)
-
-# --- Async mode + SSE events ---
-r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"]})
-if r.status_code == 202:
-    async_id = r.json()["run_id"]
-    seen, finished = [], None
-    with client.stream("GET", f"{BASE}/runs/{async_id}/events", timeout=120) as s:
-        for line in s.iter_lines():
-            if line.startswith("data: "):
-                ev = json.loads(line[6:])
-                seen.append(ev["type"])
-                if ev["type"] == "run.finished":
-                    finished = ev.get("status")
-                    break
-    ok = finished == "success"
-    results.append(("async + SSE events", "PASS" if ok else "FAIL", f"202 events={seen} final={finished}"))
-else:
-    results.append(("async + SSE events", "FAIL", f"{r.status_code} {r.text[:200]}"))
-print(f"[{results[-1][1]}] async+SSE: {results[-1][2]}", flush=True)
-
-# --- Cancel endpoint ---
-r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"], "maxCrawlDepth": 2, "maxCrawlPages": 50})
-if r.status_code == 202:
-    cancel_id = r.json()["run_id"]
-    time.sleep(1)
-    r2 = client.post(f"{BASE}/runs/{cancel_id}/cancel")
-    ok = r2.status_code == 200 and r2.json().get("status") == "cancelled"
-    results.append(("POST runs/{id}/cancel", "PASS" if ok else "FAIL", f"{r2.status_code} {r2.text[:150]}"))
-else:
-    results.append(("POST runs/{id}/cancel", "FAIL", f"setup {r.status_code}"))
-print(f"[{results[-1][1]}] cancel: {results[-1][2]}", flush=True)
-
-print("\n===== SUMMARY =====")
-for name, status_, detail in results:
-    print(f"{status_:4} {name}: {detail}")
-failed = [r for r in results if r[1] == "FAIL"]
-print(f"\n{len(results) - len(failed)}/{len(results)} passed")
-sys.exit(1 if failed else 0)
--- a/surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py
+++ b/surfsense_backend/tests/unit/platforms/reddit/test_fetch_resilience.py
@ -95,8 +95,8 @@ def _no_sleep(monkeypatch) -> None:


 async def test_warms_then_returns_json():
-    # old.reddit is tried first and mints loid -> a single warm call.
-    holder = _FakeHolder([_FakeSession(200, old_loid=True)])
+    # shreddit is tried first and mints loid -> a single warm call.
+    holder = _FakeHolder([_FakeSession(200, shreddit_loid=True)])
    token = _current_session.set(holder)
    try:
        result = await fetch_json("r/python/hot")
@ -107,9 +107,9 @@ async def test_warms_then_returns_json():
    assert holder.session.warm_calls == 1  # warmed exactly once


-async def test_warm_falls_back_to_shreddit():
-    # old.reddit doesn't mint loid, shreddit does -> still warms on the same IP.
-    holder = _FakeHolder([_FakeSession(200, shreddit_loid=True, old_loid=False)])
+async def test_warm_falls_back_to_old_reddit():
+    # shreddit doesn't mint loid, old.reddit does -> still warms on the same IP.
+    holder = _FakeHolder([_FakeSession(200, shreddit_loid=False, old_loid=True)])
    token = _current_session.set(holder)
    try:
        result = await fetch_json("r/python/hot")