diff --git a/surfsense_backend/app/capabilities/youtube/comments/executor.py b/surfsense_backend/app/capabilities/youtube/comments/executor.py index f4f28bef5..a5f4338c9 100644 --- a/surfsense_backend/app/capabilities/youtube/comments/executor.py +++ b/surfsense_backend/app/capabilities/youtube/comments/executor.py @@ -6,7 +6,7 @@ from collections.abc import Awaitable, Callable from app.capabilities.core import Executor from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput -from app.proprietary.scrapers.youtube import ( +from app.proprietary.platforms.youtube import ( YouTubeCommentsInput, scrape_comments, ) diff --git a/surfsense_backend/app/capabilities/youtube/comments/schemas.py b/surfsense_backend/app/capabilities/youtube/comments/schemas.py index c656b1f55..62eb7c451 100644 --- a/surfsense_backend/app/capabilities/youtube/comments/schemas.py +++ b/surfsense_backend/app/capabilities/youtube/comments/schemas.py @@ -10,7 +10,7 @@ from typing import Literal from pydantic import BaseModel, Field -from app.proprietary.scrapers.youtube import CommentItem +from app.proprietary.platforms.youtube import CommentItem MAX_COMMENT_VIDEOS = 20 """Per-call cap on how many video URLs one request may harvest comments from.""" diff --git a/surfsense_backend/app/capabilities/youtube/scrape/executor.py b/surfsense_backend/app/capabilities/youtube/scrape/executor.py index 82760e536..46d4789dc 100644 --- a/surfsense_backend/app/capabilities/youtube/scrape/executor.py +++ b/surfsense_backend/app/capabilities/youtube/scrape/executor.py @@ -6,7 +6,7 @@ from collections.abc import Awaitable, Callable from app.capabilities.core import Executor from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput -from app.proprietary.scrapers.youtube import ( +from app.proprietary.platforms.youtube import ( YouTubeScrapeInput, scrape_youtube, ) diff --git a/surfsense_backend/app/capabilities/youtube/scrape/schemas.py b/surfsense_backend/app/capabilities/youtube/scrape/schemas.py index c24478b11..3c1f35ef1 100644 --- a/surfsense_backend/app/capabilities/youtube/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/youtube/scrape/schemas.py @@ -10,7 +10,7 @@ from __future__ import annotations from pydantic import BaseModel, Field, model_validator -from app.proprietary.scrapers.youtube import VideoItem +from app.proprietary.platforms.youtube import VideoItem MAX_YOUTUBE_SOURCES = 20 """Per-call cap on URLs + queries: bounds a synchronous request's fan-out (05).""" diff --git a/surfsense_backend/app/proprietary/platforms/__init__.py b/surfsense_backend/app/proprietary/platforms/__init__.py index 2c505c8ad..86c49efbb 100644 --- a/surfsense_backend/app/proprietary/platforms/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/__init__.py @@ -1,7 +1,6 @@ -"""Platform-specific crawl/extraction actors (non-Apache-2; see ../LICENSE). +"""Platform-native crawl/extraction actors (non-Apache-2; see ../LICENSE). -Scaffolded for future work (Phase 8 — Platform Actors). Each platform (e.g. -maps, professional networks) will get its own subpackage that reuses the shared -fetch strategies in ``app.proprietary.web_crawler`` under a platform-specific -structured extractor. Empty for now by design. +One subpackage per platform (e.g. ``youtube``), each a structured extractor for +that platform's data. Actors may reuse the shared fetch strategies in +``app.proprietary.web_crawler`` but expose their own platform-specific I/O. """ diff --git a/surfsense_backend/app/proprietary/scrapers/youtube/README.md b/surfsense_backend/app/proprietary/platforms/youtube/README.md similarity index 99% rename from surfsense_backend/app/proprietary/scrapers/youtube/README.md rename to surfsense_backend/app/proprietary/platforms/youtube/README.md index ca1105cbb..afcc718eb 100644 --- a/surfsense_backend/app/proprietary/scrapers/youtube/README.md +++ b/surfsense_backend/app/proprietary/platforms/youtube/README.md @@ -13,7 +13,7 @@ No API keys, no Apify account, no headless browser on the happy path. ## Quick start ```python -from app.proprietary.scrapers.youtube import ( +from app.proprietary.platforms.youtube import ( YouTubeScrapeInput, scrape_youtube, YouTubeCommentsInput, scrape_comments, ) diff --git a/surfsense_backend/app/proprietary/scrapers/youtube/__init__.py b/surfsense_backend/app/proprietary/platforms/youtube/__init__.py similarity index 100% rename from surfsense_backend/app/proprietary/scrapers/youtube/__init__.py rename to surfsense_backend/app/proprietary/platforms/youtube/__init__.py diff --git a/surfsense_backend/app/proprietary/scrapers/youtube/comments.py b/surfsense_backend/app/proprietary/platforms/youtube/comments.py similarity index 100% rename from surfsense_backend/app/proprietary/scrapers/youtube/comments.py rename to surfsense_backend/app/proprietary/platforms/youtube/comments.py index 5f60c7e01..c02331199 100644 --- a/surfsense_backend/app/proprietary/scrapers/youtube/comments.py +++ b/surfsense_backend/app/proprietary/platforms/youtube/comments.py @@ -27,8 +27,8 @@ from .parsers import ( parse_count, parse_video_page, ) -from .scraper import _post, _published_date, fan_out from .schemas import CommentItem, YouTubeCommentsInput +from .scraper import _post, _published_date, fan_out from .url_resolver import resolve_url logger = logging.getLogger(__name__) diff --git a/surfsense_backend/app/proprietary/scrapers/youtube/innertube.py b/surfsense_backend/app/proprietary/platforms/youtube/innertube.py similarity index 100% rename from surfsense_backend/app/proprietary/scrapers/youtube/innertube.py rename to surfsense_backend/app/proprietary/platforms/youtube/innertube.py diff --git a/surfsense_backend/app/proprietary/scrapers/youtube/parsers.py b/surfsense_backend/app/proprietary/platforms/youtube/parsers.py similarity index 100% rename from surfsense_backend/app/proprietary/scrapers/youtube/parsers.py rename to surfsense_backend/app/proprietary/platforms/youtube/parsers.py diff --git a/surfsense_backend/app/proprietary/scrapers/youtube/schemas.py b/surfsense_backend/app/proprietary/platforms/youtube/schemas.py similarity index 100% rename from surfsense_backend/app/proprietary/scrapers/youtube/schemas.py rename to surfsense_backend/app/proprietary/platforms/youtube/schemas.py diff --git a/surfsense_backend/app/proprietary/scrapers/youtube/scraper.py b/surfsense_backend/app/proprietary/platforms/youtube/scraper.py similarity index 100% rename from surfsense_backend/app/proprietary/scrapers/youtube/scraper.py rename to surfsense_backend/app/proprietary/platforms/youtube/scraper.py diff --git a/surfsense_backend/app/proprietary/scrapers/youtube/search_filters.py b/surfsense_backend/app/proprietary/platforms/youtube/search_filters.py similarity index 100% rename from surfsense_backend/app/proprietary/scrapers/youtube/search_filters.py rename to surfsense_backend/app/proprietary/platforms/youtube/search_filters.py diff --git a/surfsense_backend/app/proprietary/scrapers/youtube/subtitles.py b/surfsense_backend/app/proprietary/platforms/youtube/subtitles.py similarity index 100% rename from surfsense_backend/app/proprietary/scrapers/youtube/subtitles.py rename to surfsense_backend/app/proprietary/platforms/youtube/subtitles.py diff --git a/surfsense_backend/app/proprietary/scrapers/youtube/url_resolver.py b/surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py similarity index 100% rename from surfsense_backend/app/proprietary/scrapers/youtube/url_resolver.py rename to surfsense_backend/app/proprietary/platforms/youtube/url_resolver.py diff --git a/surfsense_backend/app/proprietary/scrapers/__init__.py b/surfsense_backend/app/proprietary/scrapers/__init__.py deleted file mode 100644 index 808d7dfe8..000000000 --- a/surfsense_backend/app/proprietary/scrapers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Platform-native scrapers (non-connector data extraction tools).""" diff --git a/surfsense_backend/app/routes/youtube_routes.py b/surfsense_backend/app/routes/youtube_routes.py index 22d8c6ace..76b489227 100644 --- a/surfsense_backend/app/routes/youtube_routes.py +++ b/surfsense_backend/app/routes/youtube_routes.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query from scrapling.fetchers import AsyncFetcher from app.auth.context import AuthContext -from app.proprietary.scrapers.youtube import ( +from app.proprietary.platforms.youtube import ( YouTubeCommentsInput, YouTubeScrapeInput, scrape_comments, diff --git a/surfsense_backend/scripts/e2e_youtube_scraper.py b/surfsense_backend/scripts/e2e_youtube_scraper.py index 92ca9635a..4a4634674 100644 --- a/surfsense_backend/scripts/e2e_youtube_scraper.py +++ b/surfsense_backend/scripts/e2e_youtube_scraper.py @@ -31,20 +31,20 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): load_dotenv(_candidate) break -from app.proprietary.scrapers.youtube import ( # noqa: E402 +from app.proprietary.platforms.youtube import ( # noqa: E402 YouTubeCommentsInput, YouTubeScrapeInput, scrape_comments, scrape_youtube, ) -from app.proprietary.scrapers.youtube.innertube import ( # noqa: E402 +from app.proprietary.platforms.youtube.innertube import ( # noqa: E402 INNERTUBE_PUBLIC_API_KEY, INNERTUBE_SEARCH_URL, build_innertube_payload, fetch_html, post_innertube, ) -from app.proprietary.scrapers.youtube.parsers import ( # noqa: E402 +from app.proprietary.platforms.youtube.parsers import ( # noqa: E402 extract_yt_initial_data, extract_yt_initial_player_response, ) diff --git a/surfsense_backend/tests/unit/capabilities/youtube/comments/test_executor.py b/surfsense_backend/tests/unit/capabilities/youtube/comments/test_executor.py index 94fe941ad..fd153fed5 100644 --- a/surfsense_backend/tests/unit/capabilities/youtube/comments/test_executor.py +++ b/surfsense_backend/tests/unit/capabilities/youtube/comments/test_executor.py @@ -6,7 +6,7 @@ import pytest from app.capabilities.youtube.comments.executor import build_comments_executor from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput -from app.proprietary.scrapers.youtube import YouTubeCommentsInput +from app.proprietary.platforms.youtube import YouTubeCommentsInput pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/capabilities/youtube/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/youtube/scrape/test_executor.py index 08f15981b..48c849cb5 100644 --- a/surfsense_backend/tests/unit/capabilities/youtube/scrape/test_executor.py +++ b/surfsense_backend/tests/unit/capabilities/youtube/scrape/test_executor.py @@ -10,7 +10,7 @@ import pytest from app.capabilities.youtube.scrape.executor import build_scrape_executor from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput -from app.proprietary.scrapers.youtube import YouTubeScrapeInput +from app.proprietary.platforms.youtube import YouTubeScrapeInput pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/platforms/__init__.py b/surfsense_backend/tests/unit/platforms/__init__.py new file mode 100644 index 000000000..116a188e2 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/__init__.py @@ -0,0 +1 @@ +"""Platform-native scraper packages live here (one subpackage per platform).""" diff --git a/surfsense_backend/tests/unit/scrapers/__init__.py b/surfsense_backend/tests/unit/platforms/youtube/__init__.py similarity index 100% rename from surfsense_backend/tests/unit/scrapers/__init__.py rename to surfsense_backend/tests/unit/platforms/youtube/__init__.py diff --git a/surfsense_backend/tests/unit/scrapers/youtube/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/youtube/test_fetch_resilience.py similarity index 96% rename from surfsense_backend/tests/unit/scrapers/youtube/test_fetch_resilience.py rename to surfsense_backend/tests/unit/platforms/youtube/test_fetch_resilience.py index 068c37b00..d69da5cb2 100644 --- a/surfsense_backend/tests/unit/scrapers/youtube/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/youtube/test_fetch_resilience.py @@ -9,10 +9,8 @@ from __future__ import annotations from collections.abc import AsyncIterator -import pytest - -from app.proprietary.scrapers.youtube import innertube, scraper -from app.proprietary.scrapers.youtube.innertube import ( +from app.proprietary.platforms.youtube import innertube, scraper +from app.proprietary.platforms.youtube.innertube import ( INNERTUBE_SEARCH_URL, _current_session, fetch_html, @@ -40,7 +38,7 @@ class _FakeSession: self.exc = exc self.calls = 0 - async def post(self, url, json=None): # noqa: A002 - mirror scrapling API + async def post(self, url, json=None): self.calls += 1 if self.exc: raise ConnectionError("boom") diff --git a/surfsense_backend/tests/unit/scrapers/youtube/test_parsers.py b/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py similarity index 99% rename from surfsense_backend/tests/unit/scrapers/youtube/test_parsers.py rename to surfsense_backend/tests/unit/platforms/youtube/test_parsers.py index 28c624f62..47a457260 100644 --- a/surfsense_backend/tests/unit/scrapers/youtube/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/youtube/test_parsers.py @@ -12,7 +12,7 @@ from pathlib import Path import pytest -from app.proprietary.scrapers.youtube.parsers import ( +from app.proprietary.platforms.youtube.parsers import ( channel_about_tokens, comment_next_token, comment_reply_tokens, @@ -21,23 +21,23 @@ from app.proprietary.scrapers.youtube.parsers import ( dig, find_all, parse_channel_about, - parse_collaborators, - parse_comment_entities, - parse_description_links, - parse_location, - parse_translation, parse_channel_metadata, parse_channel_shorts, - parse_playlist_video_ids, + parse_collaborators, + parse_comment_entities, parse_count, parse_date, + parse_description_links, + parse_location, + parse_playlist_video_ids, parse_search_response, + parse_translation, parse_video_page, seconds_to_duration, ) -from app.proprietary.scrapers.youtube.schemas import YouTubeScrapeInput -from app.proprietary.scrapers.youtube.search_filters import build_search_params -from app.proprietary.scrapers.youtube.url_resolver import resolve_url +from app.proprietary.platforms.youtube.schemas import YouTubeScrapeInput +from app.proprietary.platforms.youtube.search_filters import build_search_params +from app.proprietary.platforms.youtube.url_resolver import resolve_url pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/scrapers/youtube/__init__.py b/surfsense_backend/tests/unit/scrapers/youtube/__init__.py deleted file mode 100644 index e69de29bb..000000000