mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-14 22:52:15 +02:00
refactor(proprietary): move scrapers/ -> platforms/ (one subpackage per platform)
This commit is contained in:
parent
9fe9c5b71d
commit
3312a442f3
25 changed files with 30 additions and 33 deletions
|
|
@ -1,7 +1,6 @@
|
|||
"""Platform-specific crawl/extraction actors (non-Apache-2; see ../LICENSE).
|
||||
"""Platform-native crawl/extraction actors (non-Apache-2; see ../LICENSE).
|
||||
|
||||
Scaffolded for future work (Phase 8 — Platform Actors). Each platform (e.g.
|
||||
maps, professional networks) will get its own subpackage that reuses the shared
|
||||
fetch strategies in ``app.proprietary.web_crawler`` under a platform-specific
|
||||
structured extractor. Empty for now by design.
|
||||
One subpackage per platform (e.g. ``youtube``), each a structured extractor for
|
||||
that platform's data. Actors may reuse the shared fetch strategies in
|
||||
``app.proprietary.web_crawler`` but expose their own platform-specific I/O.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ No API keys, no Apify account, no headless browser on the happy path.
|
|||
## Quick start
|
||||
|
||||
```python
|
||||
from app.proprietary.scrapers.youtube import (
|
||||
from app.proprietary.platforms.youtube import (
|
||||
YouTubeScrapeInput, scrape_youtube,
|
||||
YouTubeCommentsInput, scrape_comments,
|
||||
)
|
||||
|
|
@ -27,8 +27,8 @@ from .parsers import (
|
|||
parse_count,
|
||||
parse_video_page,
|
||||
)
|
||||
from .scraper import _post, _published_date, fan_out
|
||||
from .schemas import CommentItem, YouTubeCommentsInput
|
||||
from .scraper import _post, _published_date, fan_out
|
||||
from .url_resolver import resolve_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -1 +0,0 @@
|
|||
"""Platform-native scrapers (non-connector data extraction tools)."""
|
||||
Loading…
Add table
Add a link
Reference in a new issue