chore(scripts): update Instagram e2e scraper script

surfsense_backend/scripts/e2e_instagram_scraper.py

@@ -14,11 +14,14 @@ It:
       approach is invalid — later steps are skipped.
   Step 1 — scrape a profile's posts.
   Step 2 — scrape a profile's reels.
-  Step 3 — fetch comments for a discovered post URL.
-  Step 4 — fetch profile / hashtag / place details.
-  Step 5 — run a discovery search.
+  Step 3 — anonymous single-post extraction for a discovered ``/p/`` URL.
+  Step 4 — fetch profile details.
+  Step 5 — run a profile discovery search (Google-backed).
   Step 6 — dump trimmed, PII-anonymized raw fixtures into
       tests/unit/platforms/instagram/fixtures/ for the offline parser tests.
+
+Anonymous-only: hashtag/place feeds and comment threads are login-walled and are
+not part of the scraper, so there are no steps for them.
 """
 
 import asyncio
@@ -41,15 +44,15 @@ from app.proprietary.platforms.instagram import (  # noqa: E402
     scrape_instagram,
 )
 from app.proprietary.platforms.instagram.fetch import (  # noqa: E402
+    fetch_html,
     fetch_json,
     proxy_session,
     warm_session,
 )
+from app.proprietary.platforms.instagram.url_resolver import resolve_url  # noqa: E402
 
 _PROFILE = "natgeo"
-_HASHTAG = "https://www.instagram.com/explore/tags/travel/"
-_PLACE = "https://www.instagram.com/explore/locations/213385402/new-york-new-york/"
-_SEARCH_TERM = "coffee"
+_SEARCH_TERM = "national geographic"
 
 _FIXTURE_DIR = (
     _BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures"
@@ -129,30 +132,27 @@ async def step2_reels() -> bool:
     return _check("reels returned items", len(items) >= 0, f"{len(items)} reels")
 
 
-async def step3_comments(post_url: str | None) -> bool:
-    _hr("STEP 3 — comments for a post")
+async def step3_single_post(post_url: str | None) -> bool:
+    _hr("STEP 3 — single-post extraction for a /p/ URL")
     if not post_url:
         return _check("had a post URL", False, "step 1 found no post")
     items = await scrape_instagram(
         InstagramScrapeInput(
-            resultsType="comments", directUrls=[post_url], resultsLimit=10
+            resultsType="posts", directUrls=[post_url], resultsLimit=1
         ),
-        limit=10,
+        limit=1,
     )
-    print(f"    {len(items)} comments for {post_url}")
-    return _check("comments returned", len(items) >= 0, f"{len(items)} comments")
+    got = items[0] if items else {}
+    print(f"    {len(items)} item for {post_url} | owner={got.get('ownerUsername')}")
+    return _check("single post returned an item", len(items) > 0, post_url)
 
 
 async def step4_details() -> bool:
-    _hr("STEP 4 — profile / hashtag / place details")
+    _hr("STEP 4 — profile details")
     items = await scrape_instagram(
         InstagramScrapeInput(
             resultsType="details",
-            directUrls=[
-                f"https://www.instagram.com/{_PROFILE}/",
-                _HASHTAG,
-                _PLACE,
-            ],
+            directUrls=[f"https://www.instagram.com/{_PROFILE}/"],
         ),
         limit=10,
     )
@@ -162,12 +162,12 @@ async def step4_details() -> bool:
 
 
 async def step5_search() -> bool:
-    _hr("STEP 5 — discovery search")
+    _hr("STEP 5 — profile discovery search (Google-backed)")
     items = await scrape_instagram(
         InstagramScrapeInput(
             resultsType="posts",
             search=_SEARCH_TERM,
-            searchType="hashtag",
+            searchType="profile",
             searchLimit=3,
             resultsLimit=3,
         ),
@@ -177,7 +177,7 @@ async def step5_search() -> bool:
     return _check("search returned results", len(items) >= 0, f"{len(items)} items")
 
 
-async def step6_dump_fixtures() -> bool:
+async def step6_dump_fixtures(post_url: str | None) -> bool:
     _hr("STEP 6 — dump trimmed, anonymized fixtures for offline tests")
     profile = await fetch_json(
         "api/v1/users/web_profile_info/", {"username": _PROFILE}
@@ -189,6 +189,17 @@ async def step6_dump_fixtures() -> bool:
             json.dumps(_anonymize(profile)), encoding="utf-8"
         )
         wrote.append("profile.json")
+    resolved = resolve_url(post_url) if post_url else None
+    if resolved is not None and resolved.kind in ("post", "reel"):
+        html = await fetch_html(f"p/{resolved.value}/")
+        if html:
+            (_FIXTURE_DIR / "post.json").write_text(
+                json.dumps(
+                    {"url": post_url, "shortcode": resolved.value, "html": html}
+                ),
+                encoding="utf-8",
+            )
+            wrote.append("post.json")
     return _check("dumped fixtures", bool(wrote), f"{wrote} -> {_FIXTURE_DIR}")
 
 
@@ -214,10 +225,10 @@ async def main() -> int:
     results.append(await step1_posts())
     results.append(await step2_reels())
     post_url = await _first_post_url()
-    results.append(await step3_comments(post_url))
+    results.append(await step3_single_post(post_url))
     results.append(await step4_details())
     results.append(await step5_search())
-    results.append(await step6_dump_fixtures())
+    results.append(await step6_dump_fixtures(post_url))
     _hr("SUMMARY")
     print(f"  {sum(results)}/{len(results)} steps passed")
     return 0 if all(results) else 1