apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-12 22:42:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(native-connector): added google maps places & reviews scrapers

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-02 21:58:24 -07:00
parent cc99bc4cae
commit 7185079bd6
114 changed files with 4688 additions and 425 deletions
Download patch file Download diff file Expand all files Collapse all files

23
surfsense_backend/scripts/e2e_youtube_scraper.py
View file

@ -125,9 +125,7 @@ async def step4_dump_fixtures() -> bool:
(_FIXTURE_DIR / "video_initial_data.json").write_text(
json.dumps(initial), encoding="utf-8"
)
return _check(
"dumped fixtures", bool(player), f"-> {_FIXTURE_DIR}"
)
return _check("dumped fixtures", bool(player), f"-> {_FIXTURE_DIR}")
async def step5_comments() -> bool:
@ -137,11 +135,16 @@ async def step5_comments() -> bool:
)
items = await scrape_comments(inp)
for it in items[:6]:
print(f" - [{it.get('type')}] {it.get('author')} | {it.get('voteCount')} votes")
print(
f" - [{it.get('type')}] {it.get('author')} | {it.get('voteCount')} votes"
)
ok = bool(items) and all(it.get("cid") and it.get("videoId") for it in items)
has_reply = any(it.get("type") == "reply" and it.get("replyToCid") for it in items)
return _check("comments scraped (cid+videoId, reply linkage)", ok and has_reply,
f"{len(items)} items")
return _check(
"comments scraped (cid+videoId, reply linkage)",
ok and has_reply,
f"{len(items)} items",
)
async def step6_location_collaborators() -> bool:
@ -165,13 +168,17 @@ async def step6_location_collaborators() -> bool:
async def step7_translation() -> bool:
_hr("STEP 7 — translatedTitle/translatedText (subtitlesLanguage=es)")
items = await scrape_youtube(
YouTubeScrapeInput(startUrls=[{"url": _TRANSLATED_VIDEO_URL}], subtitlesLanguage="es")
YouTubeScrapeInput(
startUrls=[{"url": _TRANSLATED_VIDEO_URL}], subtitlesLanguage="es"
)
)
it = items[0] if items else {}
print(f" title : {it.get('title')}")
print(f" translatedTitle: {it.get('translatedTitle')}")
# A localized video's translated title differs from the canonical English one.
ok = bool(it.get("translatedTitle")) and it.get("translatedTitle") != it.get("title")
ok = bool(it.get("translatedTitle")) and it.get("translatedTitle") != it.get(
"title"
)
return _check("translatedTitle differs from original", ok)