refactor(proprietary): move scrapers/ -> platforms/ (one subpackage per platform)

This commit is contained in:
CREDO23 2026-07-03 11:01:45 +02:00
parent 9fe9c5b71d
commit 3312a442f3
25 changed files with 30 additions and 33 deletions

View file

@ -6,7 +6,7 @@ from collections.abc import Awaitable, Callable
from app.capabilities.core import Executor
from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
from app.proprietary.scrapers.youtube import (
from app.proprietary.platforms.youtube import (
YouTubeCommentsInput,
scrape_comments,
)

View file

@ -10,7 +10,7 @@ from typing import Literal
from pydantic import BaseModel, Field
from app.proprietary.scrapers.youtube import CommentItem
from app.proprietary.platforms.youtube import CommentItem
MAX_COMMENT_VIDEOS = 20
"""Per-call cap on how many video URLs one request may harvest comments from."""

View file

@ -6,7 +6,7 @@ from collections.abc import Awaitable, Callable
from app.capabilities.core import Executor
from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
from app.proprietary.scrapers.youtube import (
from app.proprietary.platforms.youtube import (
YouTubeScrapeInput,
scrape_youtube,
)

View file

@ -10,7 +10,7 @@ from __future__ import annotations
from pydantic import BaseModel, Field, model_validator
from app.proprietary.scrapers.youtube import VideoItem
from app.proprietary.platforms.youtube import VideoItem
MAX_YOUTUBE_SOURCES = 20
"""Per-call cap on URLs + queries: bounds a synchronous request's fan-out (05)."""

View file

@ -1,7 +1,6 @@
"""Platform-specific crawl/extraction actors (non-Apache-2; see ../LICENSE).
"""Platform-native crawl/extraction actors (non-Apache-2; see ../LICENSE).
Scaffolded for future work (Phase 8 Platform Actors). Each platform (e.g.
maps, professional networks) will get its own subpackage that reuses the shared
fetch strategies in ``app.proprietary.web_crawler`` under a platform-specific
structured extractor. Empty for now by design.
One subpackage per platform (e.g. ``youtube``), each a structured extractor for
that platform's data. Actors may reuse the shared fetch strategies in
``app.proprietary.web_crawler`` but expose their own platform-specific I/O.
"""

View file

@ -13,7 +13,7 @@ No API keys, no Apify account, no headless browser on the happy path.
## Quick start
```python
from app.proprietary.scrapers.youtube import (
from app.proprietary.platforms.youtube import (
YouTubeScrapeInput, scrape_youtube,
YouTubeCommentsInput, scrape_comments,
)

View file

@ -27,8 +27,8 @@ from .parsers import (
parse_count,
parse_video_page,
)
from .scraper import _post, _published_date, fan_out
from .schemas import CommentItem, YouTubeCommentsInput
from .scraper import _post, _published_date, fan_out
from .url_resolver import resolve_url
logger = logging.getLogger(__name__)

View file

@ -1 +0,0 @@
"""Platform-native scrapers (non-connector data extraction tools)."""

View file

@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query
from scrapling.fetchers import AsyncFetcher
from app.auth.context import AuthContext
from app.proprietary.scrapers.youtube import (
from app.proprietary.platforms.youtube import (
YouTubeCommentsInput,
YouTubeScrapeInput,
scrape_comments,