mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-10 22:32:16 +02:00
refactor(proprietary): move scrapers/ -> platforms/ (one subpackage per platform)
This commit is contained in:
parent
9fe9c5b71d
commit
3312a442f3
25 changed files with 30 additions and 33 deletions
|
|
@ -6,7 +6,7 @@ from collections.abc import Awaitable, Callable
|
|||
|
||||
from app.capabilities.core import Executor
|
||||
from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput
|
||||
from app.proprietary.scrapers.youtube import (
|
||||
from app.proprietary.platforms.youtube import (
|
||||
YouTubeCommentsInput,
|
||||
scrape_comments,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from typing import Literal
|
|||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.proprietary.scrapers.youtube import CommentItem
|
||||
from app.proprietary.platforms.youtube import CommentItem
|
||||
|
||||
MAX_COMMENT_VIDEOS = 20
|
||||
"""Per-call cap on how many video URLs one request may harvest comments from."""
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from collections.abc import Awaitable, Callable
|
|||
|
||||
from app.capabilities.core import Executor
|
||||
from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
from app.proprietary.scrapers.youtube import (
|
||||
from app.proprietary.platforms.youtube import (
|
||||
YouTubeScrapeInput,
|
||||
scrape_youtube,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from __future__ import annotations
|
|||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from app.proprietary.scrapers.youtube import VideoItem
|
||||
from app.proprietary.platforms.youtube import VideoItem
|
||||
|
||||
MAX_YOUTUBE_SOURCES = 20
|
||||
"""Per-call cap on URLs + queries: bounds a synchronous request's fan-out (05)."""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""Platform-specific crawl/extraction actors (non-Apache-2; see ../LICENSE).
|
||||
"""Platform-native crawl/extraction actors (non-Apache-2; see ../LICENSE).
|
||||
|
||||
Scaffolded for future work (Phase 8 — Platform Actors). Each platform (e.g.
|
||||
maps, professional networks) will get its own subpackage that reuses the shared
|
||||
fetch strategies in ``app.proprietary.web_crawler`` under a platform-specific
|
||||
structured extractor. Empty for now by design.
|
||||
One subpackage per platform (e.g. ``youtube``), each a structured extractor for
|
||||
that platform's data. Actors may reuse the shared fetch strategies in
|
||||
``app.proprietary.web_crawler`` but expose their own platform-specific I/O.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ No API keys, no Apify account, no headless browser on the happy path.
|
|||
## Quick start
|
||||
|
||||
```python
|
||||
from app.proprietary.scrapers.youtube import (
|
||||
from app.proprietary.platforms.youtube import (
|
||||
YouTubeScrapeInput, scrape_youtube,
|
||||
YouTubeCommentsInput, scrape_comments,
|
||||
)
|
||||
|
|
@ -27,8 +27,8 @@ from .parsers import (
|
|||
parse_count,
|
||||
parse_video_page,
|
||||
)
|
||||
from .scraper import _post, _published_date, fan_out
|
||||
from .schemas import CommentItem, YouTubeCommentsInput
|
||||
from .scraper import _post, _published_date, fan_out
|
||||
from .url_resolver import resolve_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -1 +0,0 @@
|
|||
"""Platform-native scrapers (non-connector data extraction tools)."""
|
||||
|
|
@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query
|
|||
from scrapling.fetchers import AsyncFetcher
|
||||
|
||||
from app.auth.context import AuthContext
|
||||
from app.proprietary.scrapers.youtube import (
|
||||
from app.proprietary.platforms.youtube import (
|
||||
YouTubeCommentsInput,
|
||||
YouTubeScrapeInput,
|
||||
scrape_comments,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue