refactor(proprietary): move scrapers/ -> platforms/ (one subpackage per platform)

surfsense_backend/app/capabilities/youtube/comments/executor.py

@@ -6,7 +6,7 @@ from collections.abc import Awaitable, Callable
 
 from app.capabilities.core import Executor
 from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
-from app.proprietary.scrapers.youtube import (
+from app.proprietary.platforms.youtube import (
     YouTubeCommentsInput,
     scrape_comments,
 )

surfsense_backend/app/capabilities/youtube/comments/schemas.py

@@ -10,7 +10,7 @@ from typing import Literal
 
 from pydantic import BaseModel, Field
 
-from app.proprietary.scrapers.youtube import CommentItem
+from app.proprietary.platforms.youtube import CommentItem
 
 MAX_COMMENT_VIDEOS = 20
 """Per-call cap on how many video URLs one request may harvest comments from."""

surfsense_backend/app/capabilities/youtube/scrape/executor.py

@@ -6,7 +6,7 @@ from collections.abc import Awaitable, Callable
 
 from app.capabilities.core import Executor
 from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
-from app.proprietary.scrapers.youtube import (
+from app.proprietary.platforms.youtube import (
     YouTubeScrapeInput,
     scrape_youtube,
 )

surfsense_backend/app/capabilities/youtube/scrape/schemas.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 
 from pydantic import BaseModel, Field, model_validator
 
-from app.proprietary.scrapers.youtube import VideoItem
+from app.proprietary.platforms.youtube import VideoItem
 
 MAX_YOUTUBE_SOURCES = 20
 """Per-call cap on URLs + queries: bounds a synchronous request's fan-out (05)."""

surfsense_backend/app/proprietary/platforms/__init__.py

@@ -1,7 +1,6 @@
-"""Platform-specific crawl/extraction actors (non-Apache-2; see ../LICENSE).
+"""Platform-native crawl/extraction actors (non-Apache-2; see ../LICENSE).
 
-Scaffolded for future work (Phase 8 — Platform Actors). Each platform (e.g.
+One subpackage per platform (e.g. ``youtube``), each a structured extractor for
-maps, professional networks) will get its own subpackage that reuses the shared
+that platform's data. Actors may reuse the shared fetch strategies in
-fetch strategies in ``app.proprietary.web_crawler`` under a platform-specific
+``app.proprietary.web_crawler`` but expose their own platform-specific I/O.
-structured extractor. Empty for now by design.
 """

surfsense_backend/app/proprietary/platforms/youtube/README.md

@@ -13,7 +13,7 @@ No API keys, no Apify account, no headless browser on the happy path.
 ## Quick start
 
 ```python
-from app.proprietary.scrapers.youtube import (
+from app.proprietary.platforms.youtube import (
     YouTubeScrapeInput, scrape_youtube,
     YouTubeCommentsInput, scrape_comments,
 )

surfsense_backend/app/proprietary/platforms/youtube/comments.py

@@ -27,8 +27,8 @@ from .parsers import (
     parse_count,
     parse_video_page,
 )
-from .scraper import _post, _published_date, fan_out
 from .schemas import CommentItem, YouTubeCommentsInput
+from .scraper import _post, _published_date, fan_out
 from .url_resolver import resolve_url
 
 logger = logging.getLogger(__name__)

surfsense_backend/app/proprietary/scrapers/__init__.py

@@ -1 +0,0 @@
-"""Platform-native scrapers (non-connector data extraction tools)."""

surfsense_backend/app/routes/youtube_routes.py

@@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query
 from scrapling.fetchers import AsyncFetcher
 
 from app.auth.context import AuthContext
-from app.proprietary.scrapers.youtube import (
+from app.proprietary.platforms.youtube import (
     YouTubeCommentsInput,
     YouTubeScrapeInput,
     scrape_comments,

surfsense_backend/scripts/e2e_youtube_scraper.py

@@ -31,20 +31,20 @@ for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
         load_dotenv(_candidate)
         break
 
-from app.proprietary.scrapers.youtube import (  # noqa: E402
+from app.proprietary.platforms.youtube import (  # noqa: E402
     YouTubeCommentsInput,
     YouTubeScrapeInput,
     scrape_comments,
     scrape_youtube,
 )
-from app.proprietary.scrapers.youtube.innertube import (  # noqa: E402
+from app.proprietary.platforms.youtube.innertube import (  # noqa: E402
     INNERTUBE_PUBLIC_API_KEY,
     INNERTUBE_SEARCH_URL,
     build_innertube_payload,
     fetch_html,
     post_innertube,
 )
-from app.proprietary.scrapers.youtube.parsers import (  # noqa: E402
+from app.proprietary.platforms.youtube.parsers import (  # noqa: E402
     extract_yt_initial_data,
     extract_yt_initial_player_response,
 )

surfsense_backend/tests/unit/capabilities/youtube/comments/test_executor.py

@@ -6,7 +6,7 @@ import pytest
 
 from app.capabilities.youtube.comments.executor import build_comments_executor
 from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
-from app.proprietary.scrapers.youtube import YouTubeCommentsInput
+from app.proprietary.platforms.youtube import YouTubeCommentsInput
 
 pytestmark = pytest.mark.unit
 

surfsense_backend/tests/unit/capabilities/youtube/scrape/test_executor.py

@@ -10,7 +10,7 @@ import pytest
 
 from app.capabilities.youtube.scrape.executor import build_scrape_executor
 from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
-from app.proprietary.scrapers.youtube import YouTubeScrapeInput
+from app.proprietary.platforms.youtube import YouTubeScrapeInput
 
 pytestmark = pytest.mark.unit
 

surfsense_backend/tests/unit/platforms/__init__.py

@@ -0,0 +1 @@
+"""Platform-native scraper packages live here (one subpackage per platform)."""

surfsense_backend/tests/unit/platforms/youtube/test_fetch_resilience.py

@@ -9,10 +9,8 @@ from __future__ import annotations
 
 from collections.abc import AsyncIterator
 
-import pytest
+from app.proprietary.platforms.youtube import innertube, scraper
-
+from app.proprietary.platforms.youtube.innertube import (
-from app.proprietary.scrapers.youtube import innertube, scraper
-from app.proprietary.scrapers.youtube.innertube import (
     INNERTUBE_SEARCH_URL,
     _current_session,
     fetch_html,
@@ -40,7 +38,7 @@ class _FakeSession:
         self.exc = exc
         self.calls = 0
 
-    async def post(self, url, json=None):  # noqa: A002 - mirror scrapling API
+    async def post(self, url, json=None):
         self.calls += 1
         if self.exc:
             raise ConnectionError("boom")

surfsense_backend/tests/unit/platforms/youtube/test_parsers.py

@@ -12,7 +12,7 @@ from pathlib import Path
 
 import pytest
 
-from app.proprietary.scrapers.youtube.parsers import (
+from app.proprietary.platforms.youtube.parsers import (
     channel_about_tokens,
     comment_next_token,
     comment_reply_tokens,
@@ -21,23 +21,23 @@ from app.proprietary.scrapers.youtube.parsers import (
     dig,
     find_all,
     parse_channel_about,
-    parse_collaborators,
-    parse_comment_entities,
-    parse_description_links,
-    parse_location,
-    parse_translation,
     parse_channel_metadata,
     parse_channel_shorts,
-    parse_playlist_video_ids,
+    parse_collaborators,
+    parse_comment_entities,
     parse_count,
     parse_date,
+    parse_description_links,
+    parse_location,
+    parse_playlist_video_ids,
     parse_search_response,
+    parse_translation,
     parse_video_page,
     seconds_to_duration,
 )
-from app.proprietary.scrapers.youtube.schemas import YouTubeScrapeInput
+from app.proprietary.platforms.youtube.schemas import YouTubeScrapeInput
-from app.proprietary.scrapers.youtube.search_filters import build_search_params
+from app.proprietary.platforms.youtube.search_filters import build_search_params
-from app.proprietary.scrapers.youtube.url_resolver import resolve_url
+from app.proprietary.platforms.youtube.url_resolver import resolve_url
 
 pytestmark = pytest.mark.unit