diff --git a/surfsense_backend/scripts/e2e_instagram_scraper.py b/surfsense_backend/scripts/e2e_instagram_scraper.py index 95625fd06..c81075932 100644 --- a/surfsense_backend/scripts/e2e_instagram_scraper.py +++ b/surfsense_backend/scripts/e2e_instagram_scraper.py @@ -14,11 +14,14 @@ It: approach is invalid — later steps are skipped. Step 1 — scrape a profile's posts. Step 2 — scrape a profile's reels. - Step 3 — fetch comments for a discovered post URL. - Step 4 — fetch profile / hashtag / place details. - Step 5 — run a discovery search. + Step 3 — anonymous single-post extraction for a discovered ``/p/`` URL. + Step 4 — fetch profile details. + Step 5 — run a profile discovery search (Google-backed). Step 6 — dump trimmed, PII-anonymized raw fixtures into tests/unit/platforms/instagram/fixtures/ for the offline parser tests. + +Anonymous-only: hashtag/place feeds and comment threads are login-walled and are +not part of the scraper, so there are no steps for them. """ import asyncio @@ -41,15 +44,15 @@ from app.proprietary.platforms.instagram import ( # noqa: E402 scrape_instagram, ) from app.proprietary.platforms.instagram.fetch import ( # noqa: E402 + fetch_html, fetch_json, proxy_session, warm_session, ) +from app.proprietary.platforms.instagram.url_resolver import resolve_url # noqa: E402 _PROFILE = "natgeo" -_HASHTAG = "https://www.instagram.com/explore/tags/travel/" -_PLACE = "https://www.instagram.com/explore/locations/213385402/new-york-new-york/" -_SEARCH_TERM = "coffee" +_SEARCH_TERM = "national geographic" _FIXTURE_DIR = ( _BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures" @@ -129,30 +132,27 @@ async def step2_reels() -> bool: return _check("reels returned items", len(items) >= 0, f"{len(items)} reels") -async def step3_comments(post_url: str | None) -> bool: - _hr("STEP 3 — comments for a post") +async def step3_single_post(post_url: str | None) -> bool: + _hr("STEP 3 — single-post extraction for a /p/ URL") if not post_url: return _check("had a post URL", False, "step 1 found no post") items = await scrape_instagram( InstagramScrapeInput( - resultsType="comments", directUrls=[post_url], resultsLimit=10 + resultsType="posts", directUrls=[post_url], resultsLimit=1 ), - limit=10, + limit=1, ) - print(f" {len(items)} comments for {post_url}") - return _check("comments returned", len(items) >= 0, f"{len(items)} comments") + got = items[0] if items else {} + print(f" {len(items)} item for {post_url} | owner={got.get('ownerUsername')}") + return _check("single post returned an item", len(items) > 0, post_url) async def step4_details() -> bool: - _hr("STEP 4 — profile / hashtag / place details") + _hr("STEP 4 — profile details") items = await scrape_instagram( InstagramScrapeInput( resultsType="details", - directUrls=[ - f"https://www.instagram.com/{_PROFILE}/", - _HASHTAG, - _PLACE, - ], + directUrls=[f"https://www.instagram.com/{_PROFILE}/"], ), limit=10, ) @@ -162,12 +162,12 @@ async def step4_details() -> bool: async def step5_search() -> bool: - _hr("STEP 5 — discovery search") + _hr("STEP 5 — profile discovery search (Google-backed)") items = await scrape_instagram( InstagramScrapeInput( resultsType="posts", search=_SEARCH_TERM, - searchType="hashtag", + searchType="profile", searchLimit=3, resultsLimit=3, ), @@ -177,7 +177,7 @@ async def step5_search() -> bool: return _check("search returned results", len(items) >= 0, f"{len(items)} items") -async def step6_dump_fixtures() -> bool: +async def step6_dump_fixtures(post_url: str | None) -> bool: _hr("STEP 6 — dump trimmed, anonymized fixtures for offline tests") profile = await fetch_json( "api/v1/users/web_profile_info/", {"username": _PROFILE} @@ -189,6 +189,17 @@ async def step6_dump_fixtures() -> bool: json.dumps(_anonymize(profile)), encoding="utf-8" ) wrote.append("profile.json") + resolved = resolve_url(post_url) if post_url else None + if resolved is not None and resolved.kind in ("post", "reel"): + html = await fetch_html(f"p/{resolved.value}/") + if html: + (_FIXTURE_DIR / "post.json").write_text( + json.dumps( + {"url": post_url, "shortcode": resolved.value, "html": html} + ), + encoding="utf-8", + ) + wrote.append("post.json") return _check("dumped fixtures", bool(wrote), f"{wrote} -> {_FIXTURE_DIR}") @@ -214,10 +225,10 @@ async def main() -> int: results.append(await step1_posts()) results.append(await step2_reels()) post_url = await _first_post_url() - results.append(await step3_comments(post_url)) + results.append(await step3_single_post(post_url)) results.append(await step4_details()) results.append(await step5_search()) - results.append(await step6_dump_fixtures()) + results.append(await step6_dump_fixtures(post_url)) _hr("SUMMARY") print(f" {sum(results)}/{len(results)} steps passed") return 0 if all(results) else 1