mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat: update environment variables and enhance scraping capabilities
- Adjusted Google Maps and YouTube micro pricing in the .env.example file for better cost management. - Introduced new environment variables for captcha solving and stealth browser hardening to improve scraping resilience. - Removed outdated smoke test for scraper API endpoints to streamline testing. - Enhanced anonymous chat agent's system prompt to clarify capabilities and suggest account creation for advanced features. - Updated Reddit fetch logic to prioritize new session handling and improve resilience against IP-related issues. - Added compacting functionality for scraper results to optimize data handling and presentation. - Improved workspace and document management tools with clearer descriptions and enhanced functionality. - Introduced new UI components for agent setup guidance in the web application.
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
271a21aee6
commit
1fd58752a3
24 changed files with 1326 additions and 320 deletions
|
|
@ -41,9 +41,12 @@ _MAX_DOC_CHARS = 50_000
|
|||
def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str:
|
||||
"""Build the system prompt for the minimal anonymous chat agent.
|
||||
|
||||
The prompt keeps the assistant focused on plain Q/A + web search, inlines
|
||||
any uploaded document as read-only context, and redirects every other
|
||||
SurfSense feature to account registration.
|
||||
The prompt keeps the assistant focused on plain Q/A from model knowledge,
|
||||
inlines any uploaded document as read-only context, and treats the chat as
|
||||
a registration funnel: every other SurfSense capability (scraping, live
|
||||
data, deliverables, knowledge base, automations) redirects to sign-up, and
|
||||
the assistant softly suggests an account when the conversation reveals a
|
||||
competitive-intelligence need the platform serves.
|
||||
"""
|
||||
today = datetime.now(UTC).strftime("%A, %B %d, %Y")
|
||||
|
||||
|
|
@ -70,7 +73,13 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str
|
|||
|
||||
return (
|
||||
"You are SurfSense's free AI assistant, available to everyone without "
|
||||
"login.\n\n"
|
||||
"login. SurfSense is the open-source competitive intelligence platform: "
|
||||
"registered users get specialist agents that pull live market data from "
|
||||
"Reddit, YouTube, Google Maps, Google Search, and the open web, turn it "
|
||||
"into cited briefs, reports, podcasts, and presentations, keep findings "
|
||||
"in a searchable knowledge base, and run scheduled monitoring "
|
||||
"automations — plus a REST scraping API and MCP server for their own "
|
||||
"agents.\n\n"
|
||||
f"Today's date is {today}.\n\n"
|
||||
"## How to help\n"
|
||||
"- Answer the user's questions directly and conversationally. You are "
|
||||
|
|
@ -78,23 +87,42 @@ def build_anonymous_system_prompt(anon_doc: dict[str, Any] | None = None) -> str
|
|||
"- Answer from your own knowledge. You do NOT have web access here, so "
|
||||
"for current, real-time, or fast-changing facts (news, prices, "
|
||||
"weather, recent events, live data) say you can't look them up in the "
|
||||
"free experience and may be out of date, and invite the user to create "
|
||||
"a free account for live web search.\n"
|
||||
"free experience and may be out of date.\n"
|
||||
"- Be concise, accurate, and helpful. Use Markdown formatting when it "
|
||||
"improves readability."
|
||||
f"{doc_section}\n\n"
|
||||
"## What is not available here\n"
|
||||
"This is the free, no-login experience. You CANNOT search the web, save "
|
||||
"files or notes, generate reports, podcasts, resumes, presentations, or "
|
||||
"images, search or build a knowledge base, connect to apps (Gmail, "
|
||||
"Google Drive, Notion, Slack, Calendar, Discord, and similar), set up "
|
||||
"automations, or remember anything across sessions.\n\n"
|
||||
"This is the free, no-login experience. You CANNOT search the web or "
|
||||
"scrape any platform (Reddit, YouTube, Google Maps, Google Search, "
|
||||
"websites), save files or notes, upload additional files, generate "
|
||||
"reports, podcasts, resumes, presentations, or images, search or build "
|
||||
"a knowledge base, connect to apps (Gmail, Google Drive, Notion, "
|
||||
"Slack, Calendar, Discord, and similar), set up automations or "
|
||||
"monitoring, use the scraping API or MCP server, or remember anything "
|
||||
"across sessions.\n\n"
|
||||
"If the user asks for any of these, do NOT pretend to do them and "
|
||||
"never claim you saved, created, or stored anything. Instead, briefly "
|
||||
"let them know the feature requires a free SurfSense account and "
|
||||
"invite them to create one at https://www.surfsense.com. Then offer to "
|
||||
"help with what you can do here (answering questions from your own "
|
||||
"knowledge and about any uploaded document)."
|
||||
"never claim you saved, created, scraped, or stored anything. Instead, "
|
||||
"briefly say that this needs a free SurfSense account and invite them "
|
||||
"to sign up at https://www.surfsense.com/register — name the specific "
|
||||
"capability that would solve their request (e.g. 'a SurfSense account "
|
||||
"can scrape those Reddit threads live and turn them into a cited "
|
||||
"brief'). Then offer to help with what you can do here (answering from "
|
||||
"your own knowledge and about any uploaded document).\n\n"
|
||||
"## Suggesting SurfSense when it genuinely fits\n"
|
||||
"You are also the front door to SurfSense. When the conversation "
|
||||
"reveals a need the full platform serves — researching competitors, "
|
||||
"tracking pricing or rankings, monitoring brand mentions or reviews, "
|
||||
"gauging Reddit/YouTube sentiment, generating leads, needing current "
|
||||
"web data, or wanting recurring reports — first answer as well as you "
|
||||
"can from your own knowledge, then add ONE short sentence pointing out "
|
||||
"that a free SurfSense account can do that with live data, linking "
|
||||
"https://www.surfsense.com/register.\n"
|
||||
"- Be helpful first, never salesy: the answer is the product; the "
|
||||
"suggestion is a footnote.\n"
|
||||
"- At most one suggestion per response, and stop suggesting entirely "
|
||||
"if the user declines or ignores it.\n"
|
||||
"- Do not suggest it for needs SurfSense does not serve (casual chat, "
|
||||
"coding help, homework, creative writing)."
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ Reddit deprecated *cold* unauthenticated ``.json`` (a bare anonymous GET now
|
|||
``scripts/e2e_reddit_scraper.py`` step 0) is:
|
||||
|
||||
warm one anonymous session cookie (``loid``) with a plain GET to
|
||||
``old.reddit.com`` (``www.reddit.com/svc/shreddit/<slug>`` fallback), then
|
||||
``www.reddit.com/svc/shreddit/<slug>`` (``old.reddit.com`` fallback), then
|
||||
GET ``www.reddit.com/<path>/.json?raw_json=1`` through that same
|
||||
Chrome-impersonated, sticky-IP session. Which warm URL mints ``loid`` is
|
||||
exit-IP dependent, so the order is a tiebreak — see :func:`warm_session`.
|
||||
|
|
@ -84,6 +84,12 @@ _BACKOFF_BASE_S = 5.0
|
|||
_MIN_INTERVAL_S = 0.5
|
||||
_PACE_JITTER_S = 0.25
|
||||
|
||||
# curl's default is 30s, so one dead sticky IP stalled a whole run for 30-50s
|
||||
# (seen live 2026-07-06). A healthy fetch lands in ~1s; cap at 10s so a dead IP
|
||||
# costs one bounded wait, then the timeout falls into the generic exception
|
||||
# branch of fetch_json and rotates to a fresh IP — same treatment as a 403.
|
||||
_REQUEST_TIMEOUT_S = 10.0
|
||||
|
||||
_HEADERS = {"Accept-Language": "en-US,en;q=0.9"}
|
||||
|
||||
# Age-gate opt-in, sent on every ``.json`` fetch so NSFW listings aren't blanked
|
||||
|
|
@ -170,7 +176,10 @@ class _RotatingSession:
|
|||
self._cm = self.session = None
|
||||
return
|
||||
self._cm = FetcherSession(
|
||||
proxy=proxy, stealthy_headers=True, impersonate="chrome"
|
||||
proxy=proxy,
|
||||
stealthy_headers=True,
|
||||
impersonate="chrome",
|
||||
timeout=_REQUEST_TIMEOUT_S,
|
||||
)
|
||||
self.session = await self._cm.__aenter__()
|
||||
|
||||
|
|
@ -234,16 +243,15 @@ async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool:
|
|||
Returns ``True`` when a ``loid`` was issued (the session can now reach
|
||||
``.json``), else ``False`` (caller rotates the IP and retries).
|
||||
|
||||
Tries ``old.reddit`` (yt-dlp's primary), then ``svc/shreddit``. WHICH one
|
||||
mints is exit-IP dependent and roughly random: live probes 2026-07-04 saw
|
||||
both directions across sessions on the rotating residential/custom proxy
|
||||
(one IP 403s ``old.reddit`` but mints on ``shreddit``; another does the
|
||||
reverse, sometimes with an ``rdt`` bot-interstitial). So the order is a
|
||||
tiebreak, not an optimization — a fresh session pays ~one wasted warm 403
|
||||
either way. That cost is amortized: ``fan_out`` reuses one warmed session
|
||||
per worker across many jobs, so warm-up runs once per worker, not per fetch.
|
||||
The fallback is what actually matters — it preserves correctness whichever
|
||||
way a given IP leans.
|
||||
Tries ``svc/shreddit`` first, then ``old.reddit`` (yt-dlp's primary) as
|
||||
the fallback. Live probes 2026-07-04 saw both directions across exit IPs,
|
||||
but a live run 2026-07-06 had ``old.reddit`` 403 on 12/12 fresh IPs while
|
||||
``shreddit`` minted every time — Reddit appears to have shut old.reddit to
|
||||
anonymous traffic, so shreddit-first saves one guaranteed-403 round trip
|
||||
per warm-up. The fallback is what actually matters — it preserves
|
||||
correctness if a given IP (or Reddit) flips back the other way. That cost
|
||||
is amortized: ``fan_out`` reuses one warmed session per worker across many
|
||||
jobs, so warm-up runs once per worker, not per fetch.
|
||||
|
||||
ponytail: sequential two-source warm burns 1 wasted request on ~half of new
|
||||
sessions. A parallel warm (gather both, take whichever mints) removes the
|
||||
|
|
@ -256,14 +264,14 @@ async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool:
|
|||
"""
|
||||
seen: set[str] = set()
|
||||
with suppress(Exception):
|
||||
page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS)
|
||||
page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS)
|
||||
seen |= _response_cookie_names(page)
|
||||
if _LOID_COOKIE in seen:
|
||||
return True
|
||||
|
||||
# Fallback: mints loid on exit IPs where old.reddit 403s instead.
|
||||
# Fallback: mints loid on exit IPs where shreddit 403s instead.
|
||||
with suppress(Exception):
|
||||
page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS)
|
||||
page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS)
|
||||
seen |= _response_cookie_names(page)
|
||||
return _LOID_COOKIE in seen
|
||||
|
||||
|
|
@ -278,6 +286,7 @@ async def _get_page(session: Any, url: str) -> Any:
|
|||
cookies=_OVER18_COOKIES,
|
||||
proxy=get_proxy_url(),
|
||||
stealthy_headers=True,
|
||||
timeout=_REQUEST_TIMEOUT_S,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue