mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat(reddit): implement Reddit scraping subagent and associated capabilities
- Added a new `reddit` subagent to scrape structured data from Reddit posts, comments, and users. - Introduced `reddit.scrape` capability for fetching data using URLs and search queries. - Implemented tools for scraping and parsing Reddit data, including handling pagination and rate limits. - Created input/output models for the Reddit scraper to define request and response structures. - Added documentation for the new Reddit scraping functionality and its usage. - Integrated the Reddit subagent into the existing multi-agent chat framework.
This commit is contained in:
parent
701f888b9e
commit
ff2e5f390f
35 changed files with 510 additions and 15 deletions
|
|
@ -30,6 +30,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = {
|
|||
"youtube": frozenset(),
|
||||
"google_maps": frozenset(),
|
||||
"google_search": frozenset(),
|
||||
"reddit": frozenset(),
|
||||
"airtable": frozenset({"AIRTABLE_CONNECTOR"}),
|
||||
"calendar": frozenset({"GOOGLE_CALENDAR_CONNECTOR"}),
|
||||
"clickup": frozenset({"CLICKUP_CONNECTOR"}),
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
"""``reddit`` builtin subagent: structured Reddit posts, comments, and users."""
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
"""``reddit`` route: ``SurfSenseSubagentSpec`` builder for deepagents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.language_models import BaseChatModel
|
||||
from langchain_core.tools import BaseTool
|
||||
|
||||
from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import (
|
||||
read_md_file,
|
||||
)
|
||||
from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec
|
||||
from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import (
|
||||
pack_subagent,
|
||||
)
|
||||
|
||||
from .tools.index import NAME, RULESET, load_tools
|
||||
|
||||
|
||||
def build_subagent(
|
||||
*,
|
||||
dependencies: dict[str, Any],
|
||||
model: BaseChatModel | None = None,
|
||||
middleware_stack: dict[str, Any] | None = None,
|
||||
mcp_tools: list[BaseTool] | None = None,
|
||||
) -> SurfSenseSubagentSpec:
|
||||
tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])]
|
||||
description = (
|
||||
read_md_file(__package__, "description").strip()
|
||||
or "Pulls structured data from Reddit posts, comments, subreddits, and users."
|
||||
)
|
||||
system_prompt = read_md_file(__package__, "system_prompt").strip()
|
||||
return pack_subagent(
|
||||
name=NAME,
|
||||
description=description,
|
||||
system_prompt=system_prompt,
|
||||
tools=tools,
|
||||
ruleset=RULESET,
|
||||
dependencies=dependencies,
|
||||
model=model,
|
||||
middleware_stack=middleware_stack,
|
||||
)
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
Reddit specialist: pulls structured Reddit data — posts (title, body, score, upvote ratio, comment count, subreddit, author, flair, timestamps), their comment threads, and community/user metadata. Discovers posts by search query (optionally scoped to a single subreddit), or scrapes a known post, subreddit, or user URL as-is, with sort (hot/top/new/rising) and time-window controls. Also compares fresh Reddit results against earlier findings in this chat.
|
||||
Use whenever the task is to find what people say on Reddit about a topic, gather posts/comments from a subreddit or user, discover discussion threads, or read community sentiment. Triggers include "search Reddit for X", "what does r/X say about Y", "find Reddit posts/threads about X", "top posts in r/X", and comparisons against earlier Reddit results in this chat. Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), or YouTube (use the YouTube specialist).
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
You are the SurfSense Reddit sub-agent.
|
||||
You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis.
|
||||
|
||||
<goal>
|
||||
Answer the delegated question from live Reddit data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it.
|
||||
</goal>
|
||||
|
||||
<available_tools>
|
||||
- `reddit_scrape`
|
||||
</available_tools>
|
||||
|
||||
<playbook>
|
||||
- Finding discussion on a topic: call `reddit_scrape` with `search_queries`; set `community` to scope the search to one subreddit (e.g. "python").
|
||||
- Scraping a subreddit's listing: pass `community` with no `search_queries`, and tune `sort` (hot/top/new/rising) and `time_filter` for top/controversial.
|
||||
- Scraping a specific post, subreddit, or user: pass its Reddit URL in `urls`.
|
||||
- Reading comment sentiment: keep `skip_comments` false and raise `max_comments`; set `skip_comments` true when you only need posts (faster).
|
||||
- Controlling volume: use `max_items` for the total cap, `max_posts` per target, `max_comments` per post.
|
||||
- Batch multiple search terms into one call rather than many single-term calls.
|
||||
- Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, score/rank changes).
|
||||
</playbook>
|
||||
|
||||
<tool_policy>
|
||||
- Use only tools in `<available_tools>`.
|
||||
- Report only results present in the tool output. Never invent titles, URLs, scores, authors, or comment text.
|
||||
</tool_policy>
|
||||
|
||||
<out_of_scope>
|
||||
- Do not read arbitrary web pages — that belongs to the web crawling specialist.
|
||||
- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on.
|
||||
- Google results belong to the Google Search specialist; YouTube belongs to the YouTube specialist.
|
||||
</out_of_scope>
|
||||
|
||||
<safety>
|
||||
- Report uncertainty explicitly when evidence is incomplete or conflicting.
|
||||
- Never present unverified claims as facts.
|
||||
</safety>
|
||||
|
||||
<failure_policy>
|
||||
- Underspecified request — no usable query, community, or URL — return `status=blocked` with the missing fields.
|
||||
- Tool failure: return `status=error` with a concise recovery `next_step`.
|
||||
- No useful evidence: return `status=blocked` with a narrower query or the scope you still need.
|
||||
</failure_policy>
|
||||
|
||||
<output_contract>
|
||||
Return **only** one JSON object (no markdown/prose):
|
||||
{
|
||||
"status": "success" | "partial" | "blocked" | "error",
|
||||
"action_summary": string,
|
||||
"evidence": {
|
||||
"findings": string[],
|
||||
"sources": string[],
|
||||
"confidence": "high" | "medium" | "low"
|
||||
},
|
||||
"next_step": string | null,
|
||||
"missing_fields": string[] | null,
|
||||
"assumptions": string[] | null
|
||||
}
|
||||
<include snippet="output_contract_base"/>
|
||||
Route-specific rules:
|
||||
- `evidence.findings`: max 10 entries, each a single sentence stating one distinct post, comment, or delta. Do not paste raw payloads.
|
||||
- `evidence.sources`: max 10 Reddit URLs, one per finding when applicable. List each URL once.
|
||||
</output_contract>
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
"""``reddit`` sub-agent tools: the Reddit scrape capability verb."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
|
||||
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
|
||||
from app.capabilities.core.access.agent import build_capability_tools
|
||||
from app.capabilities.reddit.scrape.definition import REDDIT_SCRAPE
|
||||
|
||||
NAME = "reddit"
|
||||
|
||||
RULESET = Ruleset(origin=NAME, rules=[])
|
||||
|
||||
_CI_VERBS = [REDDIT_SCRAPE]
|
||||
|
||||
|
||||
def load_tools(
|
||||
*, dependencies: dict[str, Any] | None = None, **kwargs: Any
|
||||
) -> list[BaseTool]:
|
||||
d = {**(dependencies or {}), **kwargs}
|
||||
return build_capability_tools(
|
||||
workspace_id=d.get("workspace_id"),
|
||||
capabilities=_CI_VERBS,
|
||||
)
|
||||
|
|
@ -27,6 +27,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent im
|
|||
from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import (
|
||||
build_subagent as build_memory_subagent,
|
||||
)
|
||||
from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import (
|
||||
build_subagent as build_reddit_subagent,
|
||||
)
|
||||
from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import (
|
||||
build_subagent as build_web_crawler_subagent,
|
||||
)
|
||||
|
|
@ -117,6 +120,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = {
|
|||
"memory": build_memory_subagent,
|
||||
"notion": build_notion_subagent,
|
||||
"onedrive": build_onedrive_subagent,
|
||||
"reddit": build_reddit_subagent,
|
||||
"slack": build_slack_subagent,
|
||||
"teams": build_teams_subagent,
|
||||
"web_crawler": build_web_crawler_subagent,
|
||||
|
|
|
|||
5
surfsense_backend/app/capabilities/reddit/__init__.py
Normal file
5
surfsense_backend/app/capabilities/reddit/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
"""``reddit.*`` namespace: platform-native Reddit data verbs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.reddit.scrape import definition as _scrape # noqa: F401
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
"""``reddit.scrape`` verb: Reddit URLs / search terms → posts, comments, users."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
"""``reddit.scrape`` capability registration (free — see 04-capabilities open item)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.core import Capability, register_capability
|
||||
from app.capabilities.reddit.scrape.executor import build_scrape_executor
|
||||
from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
||||
REDDIT_SCRAPE = Capability(
|
||||
name="reddit.scrape",
|
||||
description=(
|
||||
"Scrape public Reddit data. Give it Reddit URLs (post, subreddit, or "
|
||||
"user) and/or search terms, and it returns structured items — posts "
|
||||
"(title, body, score, comment count, subreddit, author), their comments, "
|
||||
"and community/user metadata. Use search_queries (optionally scoped to a "
|
||||
"community) to discover posts, or urls to pull a known post/subreddit/user."
|
||||
),
|
||||
input_schema=ScrapeInput,
|
||||
output_schema=ScrapeOutput,
|
||||
executor=build_scrape_executor(),
|
||||
billing_unit=None,
|
||||
)
|
||||
|
||||
register_capability(REDDIT_SCRAPE)
|
||||
49
surfsense_backend/app/capabilities/reddit/scrape/executor.py
Normal file
49
surfsense_backend/app/capabilities/reddit/scrape/executor.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""``reddit.scrape`` executor: verb input → scraper → Reddit items."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from app.capabilities.core import Executor
|
||||
from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
from app.exceptions import ForbiddenError
|
||||
from app.proprietary.platforms.reddit import (
|
||||
RedditAccessBlockedError,
|
||||
RedditScrapeInput,
|
||||
scrape_reddit,
|
||||
)
|
||||
|
||||
ScrapeFn = Callable[..., Awaitable[list[dict]]]
|
||||
|
||||
|
||||
def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
|
||||
"""Bind the executor to a scraper fn (defaults to the proprietary actor)."""
|
||||
scrape_fn = scrape_fn or scrape_reddit
|
||||
|
||||
async def execute(payload: ScrapeInput) -> ScrapeOutput:
|
||||
actor_input = RedditScrapeInput(
|
||||
startUrls=[{"url": url} for url in payload.urls],
|
||||
searches=payload.search_queries,
|
||||
searchCommunityName=payload.community,
|
||||
sort=payload.sort,
|
||||
time=payload.time_filter,
|
||||
includeNSFW=payload.include_nsfw,
|
||||
skipComments=payload.skip_comments,
|
||||
maxItems=payload.max_items,
|
||||
maxPostCount=payload.max_posts,
|
||||
maxComments=payload.max_comments,
|
||||
postDateLimit=payload.post_date_limit,
|
||||
commentDateLimit=payload.comment_date_limit,
|
||||
)
|
||||
try:
|
||||
items = await scrape_fn(actor_input, limit=payload.max_items)
|
||||
except RedditAccessBlockedError as exc:
|
||||
# Anonymous-only scraper; a hard block can't be retried with creds.
|
||||
# Mirror google_maps' SignInRequiredError -> ForbiddenError mapping.
|
||||
raise ForbiddenError(
|
||||
f"Reddit refused anonymous access: {exc}",
|
||||
code="REDDIT_ACCESS_BLOCKED",
|
||||
) from exc
|
||||
return ScrapeOutput(items=items)
|
||||
|
||||
return execute
|
||||
102
surfsense_backend/app/capabilities/reddit/scrape/schemas.py
Normal file
102
surfsense_backend/app/capabilities/reddit/scrape/schemas.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
"""``reddit.scrape`` I/O contracts.
|
||||
|
||||
A lean, agent-friendly surface over ``RedditScrapeInput``
|
||||
(``app/proprietary/platforms/reddit``). The executor maps this to the full
|
||||
scraper input; the scraper's ``RedditItem`` is reused verbatim as the output
|
||||
element.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from app.proprietary.platforms.reddit import RedditItem
|
||||
from app.proprietary.platforms.reddit.schemas import RedditSort, RedditTime
|
||||
|
||||
MAX_REDDIT_SOURCES = 20
|
||||
"""Per-call cap on urls + search_queries: bounds a synchronous request's fan-out."""
|
||||
|
||||
MAX_REDDIT_ITEMS = 100
|
||||
"""Hard ceiling on items returned per call, regardless of the per-target caps."""
|
||||
|
||||
|
||||
class ScrapeInput(BaseModel):
|
||||
urls: list[str] = Field(
|
||||
default_factory=list,
|
||||
max_length=MAX_REDDIT_SOURCES,
|
||||
description=(
|
||||
"Reddit URLs to scrape: a post, a subreddit (/r/<name>), a user "
|
||||
"(/user/<name>), or a search URL. Provide these OR search_queries/"
|
||||
"community (at least one source is required)."
|
||||
),
|
||||
)
|
||||
search_queries: list[str] = Field(
|
||||
default_factory=list,
|
||||
max_length=MAX_REDDIT_SOURCES,
|
||||
description=(
|
||||
"Search terms to run on Reddit; each returns up to max_items results. "
|
||||
"Scope to one subreddit with community."
|
||||
),
|
||||
)
|
||||
community: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Subreddit name (without 'r/') to scope search_queries to, e.g. "
|
||||
"'python'. With no search_queries, its listing is scraped."
|
||||
),
|
||||
)
|
||||
sort: RedditSort = Field(
|
||||
default="new",
|
||||
description="Result ordering: relevance, hot, top, new, rising, or comments.",
|
||||
)
|
||||
time_filter: RedditTime | None = Field(
|
||||
default=None,
|
||||
description="Time window for 'top'/'controversial' sorts: hour, day, week, month, year, all.",
|
||||
)
|
||||
include_nsfw: bool = Field(
|
||||
default=True,
|
||||
description="Include posts flagged over-18 (NSFW) in the results.",
|
||||
)
|
||||
skip_comments: bool = Field(
|
||||
default=False,
|
||||
description="Skip fetching comment trees (faster; posts/listings only).",
|
||||
)
|
||||
max_items: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=MAX_REDDIT_ITEMS,
|
||||
description="Max total items to return across all sources.",
|
||||
)
|
||||
max_posts: int = Field(
|
||||
default=10,
|
||||
ge=0,
|
||||
description="Max posts to pull per subreddit/user/search target.",
|
||||
)
|
||||
max_comments: int = Field(
|
||||
default=10,
|
||||
ge=0,
|
||||
description="Max comments to pull per post (0 = none).",
|
||||
)
|
||||
post_date_limit: str | None = Field(
|
||||
default=None,
|
||||
description="ISO date; only return posts newer than this (incremental scrape).",
|
||||
)
|
||||
comment_date_limit: str | None = Field(
|
||||
default=None,
|
||||
description="ISO date; only return comments newer than this (incremental scrape).",
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _require_a_source(self) -> ScrapeInput:
|
||||
if not self.urls and not self.search_queries and not self.community:
|
||||
raise ValueError(
|
||||
"Provide at least one of 'urls', 'search_queries', or 'community'."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class ScrapeOutput(BaseModel):
|
||||
items: list[RedditItem] = Field(
|
||||
default_factory=list,
|
||||
description="One item per result (post/comment/community/user), in emission order.",
|
||||
)
|
||||
|
|
@ -60,17 +60,17 @@ rotation surfaces as `RedditAccessBlockedError` (mirrors google_maps'
|
|||
|
||||
## Testing
|
||||
|
||||
- Offline unit tests: `tests/unit/scrapers/reddit/` — `test_skeleton.py`
|
||||
- Offline unit tests: `tests/unit/platforms/reddit/` — `test_skeleton.py`
|
||||
(schema + URL resolver), `test_parsers.py` (fixture-pinned mapping),
|
||||
`test_fetch_resilience.py` (warm / rotate / backoff loop with fake sessions,
|
||||
no network).
|
||||
- Live e2e (needs network + residential proxy): `scripts/e2e_reddit_scraper.py`
|
||||
— step 0 is the go/no-go `loid` probe; later steps exercise the flows and dump
|
||||
trimmed fixtures into `tests/unit/scrapers/reddit/fixtures/`.
|
||||
trimmed fixtures into `tests/unit/platforms/reddit/fixtures/`.
|
||||
|
||||
```bash
|
||||
cd surfsense_backend
|
||||
.venv/bin/python -m pytest tests/unit/scrapers/reddit/
|
||||
.venv/bin/python -m pytest tests/unit/platforms/reddit/
|
||||
.venv/bin/python scripts/e2e_reddit_scraper.py # live; regenerates fixtures
|
||||
```
|
||||
|
||||
|
|
@ -3,6 +3,7 @@ from fastapi import APIRouter, Depends
|
|||
# Import verb namespaces for their registration side effects before the door builds.
|
||||
import app.capabilities.google_maps
|
||||
import app.capabilities.google_search
|
||||
import app.capabilities.reddit
|
||||
import app.capabilities.web
|
||||
import app.capabilities.youtube # noqa: F401
|
||||
from app.automations.api import router as automations_router
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Manual functional e2e for the Reddit scraper (app/proprietary/scrapers/reddit).
|
||||
"""Manual functional e2e for the Reddit scraper (app/proprietary/platforms/reddit).
|
||||
|
||||
Run from the backend directory:
|
||||
cd surfsense_backend
|
||||
|
|
@ -18,7 +18,7 @@ It:
|
|||
Step 3 — run a search query.
|
||||
Step 4 — scrape a user profile.
|
||||
Step 5 — dump trimmed raw ``.json`` fixtures into
|
||||
tests/unit/scrapers/reddit/fixtures/ for the offline parser tests.
|
||||
tests/unit/platforms/reddit/fixtures/ for the offline parser tests.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
|
@ -36,22 +36,22 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
|
|||
load_dotenv(_candidate)
|
||||
break
|
||||
|
||||
from app.proprietary.scrapers.reddit import ( # noqa: E402
|
||||
from app.proprietary.platforms.reddit import ( # noqa: E402
|
||||
RedditScrapeInput,
|
||||
scrape_reddit,
|
||||
)
|
||||
from app.proprietary.scrapers.reddit.fetch import ( # noqa: E402
|
||||
from app.proprietary.platforms.reddit.fetch import ( # noqa: E402
|
||||
fetch_json,
|
||||
proxy_session,
|
||||
warm_session,
|
||||
)
|
||||
from app.proprietary.scrapers.reddit.parsers import children # noqa: E402
|
||||
from app.proprietary.platforms.reddit.parsers import children # noqa: E402
|
||||
|
||||
_SUBREDDIT = "python"
|
||||
_SEARCH_TERM = "async web scraping"
|
||||
_USER = "spez"
|
||||
|
||||
_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "scrapers" / "reddit" / "fixtures"
|
||||
_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "platforms" / "reddit" / "fixtures"
|
||||
|
||||
|
||||
def _hr(title: str) -> None:
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from app.agents.chat.multi_agent_chat.subagents.registry import (
|
|||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
# The full specialist roster the main agent composes from: 6 builtins + 15
|
||||
# The full specialist roster the main agent composes from: 8 builtins + 15
|
||||
# connector routes. Adding/removing a specialist is a deliberate product change
|
||||
# and must be reflected here.
|
||||
_EXPECTED_SUBAGENTS = frozenset(
|
||||
|
|
@ -34,6 +34,7 @@ _EXPECTED_SUBAGENTS = frozenset(
|
|||
"gmail",
|
||||
"google_drive",
|
||||
"google_maps",
|
||||
"google_search",
|
||||
"jira",
|
||||
"knowledge_base",
|
||||
"linear",
|
||||
|
|
@ -41,6 +42,7 @@ _EXPECTED_SUBAGENTS = frozenset(
|
|||
"memory",
|
||||
"notion",
|
||||
"onedrive",
|
||||
"reddit",
|
||||
"slack",
|
||||
"teams",
|
||||
"web_crawler",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
"""``reddit.scrape`` executor: verb input → actor input mapping → typed items.
|
||||
|
||||
Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's
|
||||
own payload→RedditScrapeInput mapping and the dict→RedditItem wrapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.capabilities.reddit.scrape.executor import build_scrape_executor
|
||||
from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
from app.exceptions import ForbiddenError
|
||||
from app.proprietary.platforms.reddit import RedditAccessBlockedError, RedditScrapeInput
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeScraper:
|
||||
"""Records the actor input + limit it was called with; returns canned items."""
|
||||
|
||||
def __init__(self, items: list[dict]):
|
||||
self._items = items
|
||||
self.calls: list[tuple[RedditScrapeInput, int | None]] = []
|
||||
|
||||
async def __call__(
|
||||
self, actor_input: RedditScrapeInput, *, limit: int | None = None
|
||||
) -> list[dict]:
|
||||
self.calls.append((actor_input, limit))
|
||||
return self._items
|
||||
|
||||
|
||||
async def test_maps_urls_to_start_urls_and_wraps_items():
|
||||
scraper = _FakeScraper([{"dataType": "post", "id": "abc", "title": "Hello"}])
|
||||
execute = build_scrape_executor(scrape_fn=scraper)
|
||||
|
||||
out = await execute(ScrapeInput(urls=["https://www.reddit.com/r/python/"]))
|
||||
|
||||
assert isinstance(out, ScrapeOutput)
|
||||
assert len(out.items) == 1
|
||||
assert out.items[0].id == "abc"
|
||||
assert out.items[0].title == "Hello"
|
||||
assert out.items[0].dataType == "post"
|
||||
|
||||
(actor_input, _limit) = scraper.calls[0]
|
||||
assert [u.url for u in actor_input.startUrls] == ["https://www.reddit.com/r/python/"]
|
||||
assert actor_input.searches == []
|
||||
|
||||
|
||||
async def test_forwards_search_queries_and_community():
|
||||
scraper = _FakeScraper([])
|
||||
execute = build_scrape_executor(scrape_fn=scraper)
|
||||
|
||||
await execute(ScrapeInput(search_queries=["a", "b"], community="python"))
|
||||
|
||||
(actor_input, _limit) = scraper.calls[0]
|
||||
assert actor_input.searches == ["a", "b"]
|
||||
assert actor_input.searchCommunityName == "python"
|
||||
assert actor_input.startUrls == []
|
||||
|
||||
|
||||
async def test_maps_caps_and_passes_limit():
|
||||
scraper = _FakeScraper([])
|
||||
execute = build_scrape_executor(scrape_fn=scraper)
|
||||
|
||||
await execute(
|
||||
ScrapeInput(
|
||||
search_queries=["x"],
|
||||
max_items=25,
|
||||
max_posts=7,
|
||||
max_comments=3,
|
||||
skip_comments=True,
|
||||
sort="top",
|
||||
time_filter="week",
|
||||
)
|
||||
)
|
||||
|
||||
(actor_input, limit) = scraper.calls[0]
|
||||
assert actor_input.maxItems == 25
|
||||
assert actor_input.maxPostCount == 7
|
||||
assert actor_input.maxComments == 3
|
||||
assert actor_input.skipComments is True
|
||||
assert actor_input.sort == "top"
|
||||
assert actor_input.time == "week"
|
||||
# The outer collection limit is the caller's total-item cap.
|
||||
assert limit == 25
|
||||
|
||||
|
||||
async def test_access_blocked_maps_to_forbidden():
|
||||
async def _blocked(actor_input: RedditScrapeInput, *, limit: int | None = None):
|
||||
raise RedditAccessBlockedError("all IPs refused")
|
||||
|
||||
execute = build_scrape_executor(scrape_fn=_blocked)
|
||||
|
||||
with pytest.raises(ForbiddenError):
|
||||
await execute(ScrapeInput(search_queries=["x"]))
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
"""``reddit.scrape`` input guards: a source is required and the batch is bounded."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.capabilities.reddit.scrape.schemas import (
|
||||
MAX_REDDIT_ITEMS,
|
||||
MAX_REDDIT_SOURCES,
|
||||
ScrapeInput,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_rejects_input_with_no_source():
|
||||
with pytest.raises(ValidationError):
|
||||
ScrapeInput()
|
||||
|
||||
|
||||
def test_accepts_urls_only():
|
||||
payload = ScrapeInput(urls=["https://www.reddit.com/r/python/"])
|
||||
assert payload.search_queries == []
|
||||
|
||||
|
||||
def test_accepts_search_queries_only():
|
||||
payload = ScrapeInput(search_queries=["notebooklm alternative"])
|
||||
assert payload.urls == []
|
||||
|
||||
|
||||
def test_accepts_community_only():
|
||||
payload = ScrapeInput(community="python")
|
||||
assert payload.community == "python"
|
||||
|
||||
|
||||
def test_defaults_and_bounds():
|
||||
payload = ScrapeInput(search_queries=["x"])
|
||||
assert payload.max_items == 10
|
||||
assert payload.sort == "new"
|
||||
assert payload.include_nsfw is True
|
||||
with pytest.raises(ValidationError):
|
||||
ScrapeInput(search_queries=["x"], max_items=0)
|
||||
with pytest.raises(ValidationError):
|
||||
ScrapeInput(search_queries=["x"], max_items=MAX_REDDIT_ITEMS + 1)
|
||||
|
||||
|
||||
def test_rejects_more_sources_than_the_cap():
|
||||
too_many = [f"https://redd.it/{i}" for i in range(MAX_REDDIT_SOURCES + 1)]
|
||||
with pytest.raises(ValidationError):
|
||||
ScrapeInput(urls=too_many)
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
"""The reddit namespace registers its verb as one Capability the doors/agent read."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.capabilities import (
|
||||
reddit, # noqa: F401 — importing the namespace registers its verbs
|
||||
)
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_reddit_scrape_is_registered_and_free():
|
||||
cap = get_capability("reddit.scrape")
|
||||
|
||||
assert cap.name == "reddit.scrape"
|
||||
assert cap.input_schema is ScrapeInput
|
||||
assert cap.output_schema is ScrapeOutput
|
||||
assert cap.billing_unit is None
|
||||
|
|
@ -11,8 +11,8 @@ from __future__ import annotations
|
|||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from app.proprietary.scrapers.reddit import fetch, scraper
|
||||
from app.proprietary.scrapers.reddit.fetch import (
|
||||
from app.proprietary.platforms.reddit import fetch, scraper
|
||||
from app.proprietary.platforms.reddit.fetch import (
|
||||
RedditAccessBlockedError,
|
||||
_current_session,
|
||||
fetch_json,
|
||||
|
|
@ -16,7 +16,7 @@ from pathlib import Path
|
|||
|
||||
import pytest
|
||||
|
||||
from app.proprietary.scrapers.reddit.parsers import (
|
||||
from app.proprietary.platforms.reddit.parsers import (
|
||||
after,
|
||||
children,
|
||||
flatten_comments,
|
||||
|
|
@ -6,8 +6,8 @@ input surface, the flat item serialization contract, and URL classification.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.proprietary.scrapers.reddit.schemas import RedditItem, RedditScrapeInput
|
||||
from app.proprietary.scrapers.reddit.url_resolver import resolve_url
|
||||
from app.proprietary.platforms.reddit.schemas import RedditItem, RedditScrapeInput
|
||||
from app.proprietary.platforms.reddit.url_resolver import resolve_url
|
||||
|
||||
|
||||
def test_input_has_no_auth_fields():
|
||||
Loading…
Add table
Add a link
Reference in a new issue