From 3bd864ed958df144983b27fb6e946fab715bb447 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:08:10 +0530 Subject: [PATCH 01/72] feat(env): add Instagram scraping configuration parameters to .env.example --- surfsense_backend/.env.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 6bbf32d3f..a030b75aa 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -290,6 +290,8 @@ MICROS_PER_PAGE=1000 # GOOGLE_MAPS_MICROS_PER_REVIEW=1500 # YOUTUBE_MICROS_PER_VIDEO=2500 # YOUTUBE_MICROS_PER_COMMENT=1500 +# INSTAGRAM_SCRAPE_MICROS_PER_ITEM=3500 +# INSTAGRAM_SCRAPE_MICROS_PER_COMMENT=1500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 From 8486a4f91471539f966c330edcf2b9496e4315bb Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:08:23 +0530 Subject: [PATCH 02/72] feat(constants): add Instagram to SUBAGENT_TO_REQUIRED_CONNECTOR_MAP for enhanced multi-agent chat capabilities --- surfsense_backend/app/agents/chat/multi_agent_chat/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index b132618d7..40e8dd6d5 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -36,6 +36,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "google_maps": frozenset(), "google_search": frozenset(), "reddit": frozenset(), + "instagram": frozenset(), "mcp_discovery": frozenset( { "SLACK_CONNECTOR", From f204db7bce422ebac048b035936cd453f0c93e50 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:57:44 +0530 Subject: [PATCH 03/72] feat(billing): add INSTAGRAM_ITEM and INSTAGRAM_COMMENT units --- surfsense_backend/app/capabilities/core/types.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index c87601832..e0fef5379 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -25,6 +25,8 @@ class BillingUnit(StrEnum): GOOGLE_MAPS_REVIEW = "google_maps_review" YOUTUBE_VIDEO = "youtube_video" YOUTUBE_COMMENT = "youtube_comment" + INSTAGRAM_ITEM = "instagram_item" + INSTAGRAM_COMMENT = "instagram_comment" class BillableInput(Protocol): From 43660ee51bf346a547fc1a9e970ac46a94b1ac16 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:57:53 +0530 Subject: [PATCH 04/72] feat(billing): map Instagram units to rate keys and display nouns --- surfsense_backend/app/capabilities/core/billing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 71aa45c58..7c2fbba9f 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -35,6 +35,8 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.GOOGLE_MAPS_REVIEW: "GOOGLE_MAPS_MICROS_PER_REVIEW", BillingUnit.YOUTUBE_VIDEO: "YOUTUBE_MICROS_PER_VIDEO", BillingUnit.YOUTUBE_COMMENT: "YOUTUBE_MICROS_PER_COMMENT", + BillingUnit.INSTAGRAM_ITEM: "INSTAGRAM_SCRAPE_MICROS_PER_ITEM", + BillingUnit.INSTAGRAM_COMMENT: "INSTAGRAM_SCRAPE_MICROS_PER_COMMENT", } @@ -51,6 +53,8 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.GOOGLE_MAPS_REVIEW: "review", BillingUnit.YOUTUBE_VIDEO: "video", BillingUnit.YOUTUBE_COMMENT: "comment", + BillingUnit.INSTAGRAM_ITEM: "item", + BillingUnit.INSTAGRAM_COMMENT: "comment", } From 51c2dfdfe69e3362e7a3d9833487e55eaf6700fa Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:57:59 +0530 Subject: [PATCH 05/72] feat(config): add Instagram per-item and per-comment scrape rates --- surfsense_backend/app/config/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 74f7ec6c4..23411d979 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -719,6 +719,14 @@ class Config: # Kept separate from the video rate so comments can be re-tuned toward the # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing. YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500")) + INSTAGRAM_SCRAPE_MICROS_PER_ITEM = int( + os.getenv("INSTAGRAM_SCRAPE_MICROS_PER_ITEM", "3500") + ) + # Kept separate from the item rate so comments can be re-tuned toward the + # cheaper per-comment market without touching post/reel pricing. + INSTAGRAM_SCRAPE_MICROS_PER_COMMENT = int( + os.getenv("INSTAGRAM_SCRAPE_MICROS_PER_COMMENT", "1500") + ) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. From 842a0e666d7e3befea5a8bea506b29cf821d85ec Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:11 +0530 Subject: [PATCH 06/72] feat(instagram): add platform input/output schemas --- .../platforms/instagram/schemas.py | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/schemas.py diff --git a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py new file mode 100644 index 000000000..2175744a2 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py @@ -0,0 +1,187 @@ +# ruff: noqa: N815 +"""Input/output models for the Instagram scraper. + +The models mirror the public Instagram scraper actor spec so the endpoint can be +a drop-in: the input accepts the full documented surface, and every output field +is emitted (``None``/``[]`` when the anonymous web endpoints cannot source it +yet) so the contract expands additively — the same rule the Google Search and +YouTube models follow. + +**Anonymous only.** There is deliberately **no** authentication field on the +input (no username/password/token/cookie/``login*``) — the scraper holds only +Instagram's anonymous web-session cookies (``csrftoken``/``mid``) and can never +log in. Anything auth-shaped a caller sends lands in ``extra`` and is ignored. +""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +InstagramResultsType = Literal[ + "posts", "details", "comments", "reels", "mentions", "stories" +] +InstagramSearchType = Literal["hashtag", "profile", "place", "user"] +InstagramDetailKind = Literal["profile", "hashtag", "place"] + + +class InstagramScrapeInput(BaseModel): + """Instagram scraper input surface (anonymous, no auth fields). + + Field names mirror the public actor spec verbatim. ``resultsLimit`` / + ``searchLimit`` are collector policy applied by :func:`scrape_instagram`, + NOT ceilings baked into the streaming flows. Fields the acquisition layer + doesn't source yet are still accepted via ``extra="allow"`` for parity. + """ + + model_config = ConfigDict(extra="allow") + resultsType: InstagramResultsType = "posts" + directUrls: list[str] = Field(default_factory=list) + resultsLimit: int | None = Field(default=None, ge=1) + onlyPostsNewerThan: str | None = None + search: str | None = None + searchType: InstagramSearchType = "hashtag" + searchLimit: int | None = Field(default=None, ge=1, le=250) + addParentData: bool = False + skipPinnedPosts: bool = False + isNewestComments: bool = False + includeNestedComments: bool = False + addProfileStatistics: bool = False + + +class _ItemBase(BaseModel): + """Common error / provenance fields carried on every output item. + + Errors surface as item-level fields (never exceptions) so a partial run + still returns the items it could source, mirroring the actor's shape. + """ + + model_config = ConfigDict(extra="allow") + inputUrl: str | None = None + error: str | None = None + errorDescription: str | None = None + requestErrorMessages: list[str] = Field(default_factory=list) + + def to_output(self) -> dict[str, Any]: + """Serialize to the flat output dict shape (keeps extras).""" + return self.model_dump(exclude_none=False) + + +class InstagramMediaItem(_ItemBase): + """A post / reel / mention. One flat schema per the actor FAQ.""" + + id: str | None = None + type: Literal["Image", "Video", "Sidecar"] | None = None + shortCode: str | None = None + caption: str | None = None + hashtags: list[str] = Field(default_factory=list) + mentions: list[str] = Field(default_factory=list) + url: str | None = None + commentsCount: int | None = None + firstComment: str | None = None + latestComments: list[dict[str, Any]] = Field(default_factory=list) + dimensionsHeight: int | None = None + dimensionsWidth: int | None = None + displayUrl: str | None = None + images: list[str] = Field(default_factory=list) + videoUrl: str | None = None + alt: str | None = None + likesCount: int | None = None + videoViewCount: int | None = None + videoPlayCount: int | None = None + reshareCount: int | None = None + timestamp: str | None = None + childPosts: list[dict[str, Any]] = Field(default_factory=list) + ownerUsername: str | None = None + ownerId: str | None = None + ownerFullName: str | None = None + isPinned: bool | None = None + productType: str | None = None + videoDuration: float | None = None + paidPartnership: bool | None = None + taggedUsers: list[dict[str, Any]] = Field(default_factory=list) + musicInfo: dict[str, Any] | None = None + coauthorProducers: list[dict[str, Any]] = Field(default_factory=list) + locationName: str | None = None + locationId: str | None = None + isCommentsDisabled: bool | None = None + dataSource: dict[str, Any] | None = None + + +class InstagramComment(_ItemBase): + """A comment on a post / reel.""" + + id: str | None = None + postUrl: str | None = None + commentUrl: str | None = None + text: str | None = None + ownerUsername: str | None = None + ownerProfilePicUrl: str | None = None + timestamp: str | None = None + repliesCount: int | None = None + replies: list[dict[str, Any]] = Field(default_factory=list) + likesCount: int | None = None + owner: dict[str, Any] | None = None + + +class InstagramProfile(_ItemBase): + """A profile detail item (``detailKind = "profile"``).""" + + detailKind: Literal["profile"] = "profile" + id: str | None = None + username: str | None = None + url: str | None = None + fullName: str | None = None + biography: str | None = None + externalUrl: str | None = None + followersCount: int | None = None + followsCount: int | None = None + postsCount: int | None = None + highlightReelCount: int | None = None + igtvVideoCount: int | None = None + isBusinessAccount: bool | None = None + businessCategoryName: str | None = None + private: bool | None = None + verified: bool | None = None + profilePicUrl: str | None = None + profilePicUrlHD: str | None = None + relatedProfiles: list[dict[str, Any]] = Field(default_factory=list) + latestPosts: list[dict[str, Any]] = Field(default_factory=list) + statistics: dict[str, Any] | None = None + + +class InstagramHashtag(_ItemBase): + """A hashtag detail item (``detailKind = "hashtag"``).""" + + detailKind: Literal["hashtag"] = "hashtag" + id: str | None = None + name: str | None = None + url: str | None = None + postsCount: int | None = None + topPosts: list[dict[str, Any]] = Field(default_factory=list) + posts: list[dict[str, Any]] = Field(default_factory=list) + related: list[dict[str, Any]] = Field(default_factory=list) + searchTerm: str | None = None + searchSource: str | None = None + + +class InstagramPlace(_ItemBase): + """A place detail item (``detailKind = "place"``).""" + + detailKind: Literal["place"] = "place" + name: str | None = None + location_id: str | None = None + slug: str | None = None + lat: float | None = None + lng: float | None = None + location_address: str | None = None + location_city: str | None = None + location_zip: str | None = None + phone: str | None = None + website: str | None = None + category: str | None = None + media_count: int | None = None + posts: list[dict[str, Any]] = Field(default_factory=list) + searchTerm: str | None = None + searchSource: str | None = None From 8dc8ad6a36aa9a398ebd682a404b947e13512797 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:15 +0530 Subject: [PATCH 07/72] feat(instagram): add URL classifier and normalizer --- .../platforms/instagram/url_resolver.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py diff --git a/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py new file mode 100644 index 000000000..596b85c2c --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py @@ -0,0 +1,96 @@ +"""Classify and normalize an Instagram URL into a scrape job. + +Covers the supported ``directUrls`` shapes: a profile, a post (``/p/``), a reel +(``/reel/``), a hashtag (``/explore/tags/``), and a place +(``/explore/locations/``), plus bare profile IDs. Non-Instagram hosts resolve to +``None`` so the orchestrator can skip them. Mirrors the frozen ``ResolvedUrl`` +dataclass shape of ``../reddit/url_resolver.py``. + +Normalization rules (from the reference spec): +- ``_u/`` and ``/profilecard/`` segments are stripped. +- Story URLs (``/stories//...``) reduce to the profile. +- Location URLs are valid with the numeric ID alone (no trailing slug). +- Numeric post-ID URLs are only valid for the ``comments`` flow; elsewhere the + shortCode form is required, so a numeric-ID URL resolves with + ``numeric_post_id`` set and callers reject it outside comments. +- ``share/`` redirect resolution is handled at fetch time (network), not here. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal +from urllib.parse import urlparse + +ResolvedKind = Literal["profile", "post", "reel", "hashtag", "place"] + +_INSTAGRAM_HOSTS = frozenset( + {"m.instagram.com", "www.instagram.com", "instagram.com"} +) +_STRIP_SEGMENTS = frozenset({"_u", "profilecard"}) +_RESERVED = frozenset( + {"p", "s", "tv", "reel", "reels", "share", "explore", "stories", "accounts"} +) + + +@dataclass(frozen=True) +class ResolvedUrl: + """A classified Instagram target: the kind, its key, and the source URL.""" + + kind: ResolvedKind + value: str + url: str + slug: str | None = None + numeric_post_id: bool = False + + +def _is_instagram_host(hostname: str | None) -> bool: + if not hostname: + return False + return hostname.lower() in _INSTAGRAM_HOSTS + + +def _segments(url: str) -> list[str]: + parsed = urlparse(url) + if not _is_instagram_host(parsed.hostname): + return [] + if not parsed.path: + return [] + return [s for s in parsed.path.split("/") if s and s not in _STRIP_SEGMENTS] + + +def resolve_url(url: str) -> ResolvedUrl | None: + """Classify an Instagram URL into a scrape job, or ``None`` if unrecognized. + + A bare token with no ``http`` prefix and no ``/`` is treated as a profile ID + (the reference accepts bare profile handles alongside full URLs). + """ + if "instagram.com" not in url.lower(): + token = url.strip().lstrip("@") + if token and "/" not in token and "." not in token: + return ResolvedUrl( + "profile", token, f"https://www.instagram.com/{token}/" + ) + segments = _segments(url) + if not segments: + return None + head = segments[0] + if head == "explore" and len(segments) >= 3 and segments[1] == "tags": + return ResolvedUrl("hashtag", segments[2], url) + if head == "explore" and len(segments) >= 3 and segments[1] == "locations": + slug = segments[3] if len(segments) >= 4 else None + return ResolvedUrl("place", segments[2], url, slug=slug) + if head == "p" and len(segments) >= 2: + code = segments[1] + return ResolvedUrl("post", code, url, numeric_post_id=code.isdigit()) + if head in ("reel", "reels") and len(segments) >= 2: + code = segments[1] + return ResolvedUrl("reel", code, url, numeric_post_id=code.isdigit()) + if head == "stories" and len(segments) >= 2: + user = segments[1] + return ResolvedUrl( + "profile", user, f"https://www.instagram.com/{user}/" + ) + if head not in _RESERVED: + return ResolvedUrl("profile", head, url) + return None From 2d2900100483798940d84c8a806071fb27b810a5 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:27 +0530 Subject: [PATCH 08/72] feat(instagram): add web-JSON parsers --- .../platforms/instagram/parsers.py | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/parsers.py diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py new file mode 100644 index 000000000..96d66a2c5 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -0,0 +1,223 @@ +"""Pure JSON -> item mapping for the Instagram scraper. + +Framework-agnostic and I/O-free so it can be unit-tested against captured +fixtures. Every function takes a raw Instagram web-JSON node and returns a plain +dict shaped like the public actor spec — no network, no proxy, no ``scrapedAt`` +stamp (the orchestrator adds provenance so these stay deterministic under +fixture tests). + +Instagram's web JSON nests media under GraphQL-style ``edge_*`` containers +(``edge_media_to_caption``, ``edge_liked_by``, ...) with ``taken_at_timestamp`` +epoch seconds. These parsers flatten that into the actor's camelCase item shape. +Fields the anonymous endpoints don't expose are left unset (``None``/``[]``) so +parity is additive. +""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime +from typing import Any + +_BASE = "https://www.instagram.com" +_HASHTAG_RE = re.compile(r"#(\w+)") +_MENTION_RE = re.compile(r"@([\w.]+)") +_TYPE_MAP = { + "GraphImage": "Image", + "GraphVideo": "Video", + "GraphSidecar": "Sidecar", + "XDTGraphImage": "Image", + "XDTGraphVideo": "Video", + "XDTGraphSidecar": "Sidecar", +} + + +def _int(value: Any) -> int | None: + """Coerce to int, or ``None`` (never coerces bools).""" + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + return None + + +def _utc_from_sec(value: Any) -> str | None: + """Epoch seconds -> millisecond ISO string, or ``None``.""" + if not isinstance(value, int | float) or isinstance(value, bool): + return None + dt = datetime.fromtimestamp(float(value), tz=UTC) + return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def _edge_count(node: dict[str, Any], key: str) -> int | None: + """``node[key].count`` for a GraphQL ``edge_*`` container.""" + container = node.get(key) + if isinstance(container, dict): + return _int(container.get("count")) + return None + + +def _edges(container: Any) -> list[dict[str, Any]]: + """``container.edges[].node`` list for a GraphQL ``edge_*`` container.""" + if not isinstance(container, dict): + return [] + out = [] + for edge in container.get("edges") or []: + node = edge.get("node") if isinstance(edge, dict) else None + if not isinstance(node, dict): + continue + out.append(node) + return out + + +def _caption_text(node: dict[str, Any]) -> str | None: + """First caption edge's text (web feed) or a flat ``caption`` fallback.""" + edges = _edges(node.get("edge_media_to_caption")) + if edges: + text = edges[0].get("text") + if isinstance(text, str): + return text + cap = node.get("caption") + if isinstance(cap, dict): + cap = cap.get("text") + if isinstance(cap, str): + return cap + return None + + +def _likes_count(node: dict[str, Any]) -> int | None: + """Like count. ``-1`` (creator hid it) is passed through, never coerced.""" + for key in ("edge_liked_by", "edge_media_preview_like"): + count = _edge_count(node, key) + if count is not None: + return count + return _int(node.get("like_count")) + + +def _shortcode(node: dict[str, Any]) -> str | None: + code = node.get("shortcode") or node.get("code") + if isinstance(code, str): + return code + return None + + +def parse_media(node: dict[str, Any]) -> dict[str, Any]: + """Map a raw timeline/feed media node to a flat media item dict.""" + code = _shortcode(node) + caption = _caption_text(node) + typename = node.get("__typename") + owner = node.get("owner") if isinstance(node.get("owner"), dict) else {} + dims = node.get("dimensions") if isinstance(node.get("dimensions"), dict) else {} + is_video = bool(node.get("is_video")) + return { + "id": node.get("id"), + "type": _TYPE_MAP.get(typename) or ("Video" if is_video else "Image"), + "shortCode": code, + "caption": caption, + "hashtags": _HASHTAG_RE.findall(caption) if caption else [], + "mentions": _MENTION_RE.findall(caption) if caption else [], + "url": f"{_BASE}/p/{code}/" if code else None, + "commentsCount": _edge_count(node, "edge_media_to_comment") + or _int(node.get("comment_count")), + "dimensionsHeight": _int(dims.get("height")), + "dimensionsWidth": _int(dims.get("width")), + "displayUrl": node.get("display_url"), + "videoUrl": node.get("video_url") if is_video else None, + "alt": node.get("accessibility_caption"), + "likesCount": _likes_count(node), + "videoViewCount": _int(node.get("video_view_count")) if is_video else None, + "timestamp": _utc_from_sec(node.get("taken_at_timestamp")), + "ownerUsername": owner.get("username"), + "ownerId": owner.get("id") or node.get("owner_id"), + "ownerFullName": owner.get("full_name"), + "isCommentsDisabled": node.get("comments_disabled"), + } + + +def parse_comment(node: dict[str, Any], *, post_url: str | None) -> dict[str, Any]: + """Map a raw comment node to a flat comment item dict.""" + owner = node.get("owner") if isinstance(node.get("owner"), dict) else {} + code = _shortcode(node) + return { + "id": node.get("id"), + "postUrl": post_url, + "commentUrl": f"{_BASE}/p/{code}/c/{node.get('id')}/" if code else None, + "text": node.get("text"), + "ownerUsername": owner.get("username"), + "ownerProfilePicUrl": owner.get("profile_pic_url"), + "timestamp": _utc_from_sec(node.get("created_at")), + "repliesCount": _edge_count(node, "edge_threaded_comments") + or _int(node.get("child_comment_count")), + "likesCount": _edge_count(node, "edge_liked_by") + or _int(node.get("comment_like_count")), + "owner": {"id": owner.get("id"), "username": owner.get("username")} + if owner + else None, + } + + +def parse_profile(user: dict[str, Any]) -> dict[str, Any]: + """Map a raw ``web_profile_info`` ``data.user`` to a flat profile item dict.""" + username = user.get("username") + latest = [parse_media(n) for n in _edges(user.get("edge_owner_to_timeline_media"))] + return { + "detailKind": "profile", + "id": user.get("id"), + "username": username, + "url": f"{_BASE}/{username}/" if username else None, + "fullName": user.get("full_name"), + "biography": user.get("biography"), + "externalUrl": user.get("external_url"), + "followersCount": _edge_count(user, "edge_followed_by"), + "followsCount": _edge_count(user, "edge_follow"), + "postsCount": _edge_count(user, "edge_owner_to_timeline_media"), + "highlightReelCount": _int(user.get("highlight_reel_count")), + "igtvVideoCount": _edge_count(user, "edge_felix_video_timeline"), + "isBusinessAccount": user.get("is_business_account"), + "businessCategoryName": user.get("business_category_name"), + "private": user.get("is_private"), + "verified": user.get("is_verified"), + "profilePicUrl": user.get("profile_pic_url"), + "profilePicUrlHD": user.get("profile_pic_url_hd"), + "latestPosts": latest, + } + + +def parse_hashtag(data: dict[str, Any]) -> dict[str, Any]: + """Map a raw ``tags/web_info`` payload to a flat hashtag item dict.""" + node = data.get("data") if isinstance(data.get("data"), dict) else data + name = node.get("name") + top = _edges(node.get("edge_hashtag_to_top_posts")) + recent = _edges(node.get("edge_hashtag_to_media")) + return { + "detailKind": "hashtag", + "id": node.get("id"), + "name": name, + "url": f"{_BASE}/explore/tags/{name}/" if name else None, + "postsCount": _edge_count(node, "edge_hashtag_to_media"), + "topPosts": [parse_media(n) for n in top], + "posts": [parse_media(n) for n in recent], + } + + +def parse_place(data: dict[str, Any]) -> dict[str, Any]: + """Map a raw ``locations/web_info`` payload to a flat place item dict.""" + loc = data.get("location") if isinstance(data.get("location"), dict) else data + recent = _edges(loc.get("edge_location_to_media")) + return { + "detailKind": "place", + "name": loc.get("name"), + "location_id": str(loc.get("id")) if loc.get("id") is not None else None, + "slug": loc.get("slug"), + "lat": loc.get("lat"), + "lng": loc.get("lng"), + "location_address": loc.get("address_json") or loc.get("address"), + "location_city": loc.get("city"), + "phone": loc.get("phone"), + "website": loc.get("website"), + "category": loc.get("category"), + "media_count": _edge_count(loc, "edge_location_to_media"), + "posts": [parse_media(n) for n in recent], + } From b1eff478fd0dcb1e55ecc94e344c5795fc414b23 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:32 +0530 Subject: [PATCH 09/72] feat(instagram): add proxy-aware fetch with warm-up and rotate-on-block --- .../proprietary/platforms/instagram/fetch.py | 354 ++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/fetch.py diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py new file mode 100644 index 000000000..82b0bcc14 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -0,0 +1,354 @@ +"""Proxy-aware fetch seam for the Instagram scraper (no browser). + +All network I/O flows through :func:`fetch_json` and always egresses through the +residential proxy (a direct hit would expose and risk-block the server IP). + +Instagram's public web app exposes anonymous JSON endpoints that a logged-out +browser calls, guarded by the ``X-IG-App-ID`` web app id and a warmed +``csrftoken``/``mid`` cookie pair: + + warm one anonymous session (plain GET to ``www.instagram.com/`` mints + ``csrftoken``/``mid``), then GET the ``api/v1/*/web_info`` / + ``web_profile_info`` endpoints through that same Chrome-impersonated, + sticky-IP session with the ``X-IG-App-ID`` header. + +This is a direct port of ``../reddit/fetch.py``'s rotate-on-block sticky-session +pattern (``_RotatingSession`` + ``_current_session`` ContextVar + +``open_proxy_holder``/``bind_proxy_holder``/``proxy_session``), with an +Instagram-specific :func:`warm_session` and header set. + +Honest ceiling: anonymous Instagram access is the most hostile of our platforms. +Login walls appear as 401/403 and rotate the exit IP; 429 backs off on the same +IP. Observed per-IP/session limits are documented in ``README.md``; the safe +``_FANOUT_CONCURRENCY`` is deliberately low. ponytail: the pacing/rotation +constants are calibrated to residential exits and may need retuning per pool — +watch for 401/403/429 log spam and adjust. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import random +import time +from contextlib import asynccontextmanager, suppress +from contextvars import ContextVar +from datetime import UTC, datetime +from typing import Any +from urllib.parse import urlencode + +from scrapling.fetchers import AsyncFetcher, FetcherSession + +from app.utils.proxy import get_proxy_url + +logger = logging.getLogger(__name__) + + +class InstagramAccessBlockedError(RuntimeError): + """Raised when every rotated IP is refused anonymous access. + + This is the Instagram login-wall branch: after warming and rotating, the + exit IPs still 401/403. We are anonymous-only and cannot log in, so instead + of silently returning nothing we surface it as a clear error (mirrors + reddit's ``RedditAccessBlockedError`` / google_maps' ``SignInRequiredError``). + The executor turns it into a 403 for REST callers. + """ + + +# Per-flow proxy session, set by ``bind_proxy_holder`` around one continuation +# chain. Reusing one keep-alive connection pins a single sticky exit IP so the +# warmed ``csrftoken``/``mid`` cookies (bound to that IP) stay valid across the +# warm-up + every subsequent web-endpoint fetch. A ContextVar keeps each +# concurrent fan-out flow on its own session/IP without threading a param +# through every call. +_current_session: ContextVar[_RotatingSession | None] = ContextVar( + "instagram_proxy_session", default=None +) + +# 401/403 => this IP hit the login wall; rotate to a fresh one and re-warm. +# 429 => rate limited; back off on the SAME IP (rotating wouldn't help and burns +# the pool). +_ROTATE_STATUSES = frozenset({401, 403}) +_BACKOFF_STATUS = 429 +_MAX_ROTATIONS = 3 +_MAX_BACKOFFS = 4 +_BACKOFF_BASE_S = 5.0 + +# Instagram 429s hard on bursts. Pace each sticky session so a fast IP can't +# burst past the per-IP threshold. ponytail: 1.5s is tuned to residential exits; +# a pool with a stricter per-IP cap may need it raised — watch for 429 log spam. +_MIN_INTERVAL_S = 1.5 +_PACE_JITTER_S = 0.5 + +# A healthy fetch lands in ~1-2s; cap at 15s so a dead sticky IP costs one +# bounded wait, then the timeout falls into the generic exception branch of +# fetch_json and rotates to a fresh IP — same treatment as a 403. +_REQUEST_TIMEOUT_S = 15.0 + +# The anonymous web app id every logged-out instagram.com XHR carries. Without +# it the api/v1/*/web_info and web_profile_info endpoints 403 outright. +_IG_APP_ID = "936619743392459" +_HEADERS = { + "Accept-Language": "en-US,en;q=0.9", + "X-IG-App-ID": _IG_APP_ID, + "X-Requested-With": "XMLHttpRequest", + "Referer": "https://www.instagram.com/", +} + +# A plain GET to the home page mints the anonymous csrftoken/mid cookie pair. +_WARM_URL = "https://www.instagram.com/" +_BASE = "https://www.instagram.com" +_CSRF_COOKIE = "csrftoken" + + +def now_iso() -> str: + """UTC timestamp in the millisecond ISO shape used by scraper output.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def _response_cookie_names(page: Any) -> set[str]: + """Cookie names set by a response (best-effort across scrapling shapes).""" + cookies = getattr(page, "cookies", None) + if isinstance(cookies, dict): + return set(cookies.keys()) + return set() + + +def _parse_json(page: Any) -> Any | None: + """Parse a scrapling response body into JSON, or ``None``. + + Prefers ``page.json()``; falls back to ``json.loads`` on the raw body when + the impersonated response hands back text. + """ + fn = getattr(page, "json", None) + if callable(fn): + with suppress(Exception): + return fn() + for attr in ("body", "text"): + val = getattr(page, attr, None) + if isinstance(val, bytes): + val = val.decode("utf-8", "replace") + if isinstance(val, str) and val.strip(): + with suppress(Exception): + return json.loads(val) + return None + return None + + +def _build_url(path: str, params: dict[str, Any] | None) -> str: + """Absolute URL for an instagram.com path (accepts already-absolute URLs).""" + base = path if path.startswith("http") else f"{_BASE}/{path.strip('/')}/" + if not params: + return base + qs = urlencode({k: v for k, v in params.items() if v is not None}) + sep = "&" if "?" in base else "?" + return f"{base}{sep}{qs}" if qs else base + + +class _RotatingSession: + """Owns one live ``FetcherSession`` (sticky IP) and can swap it for a fresh one. + + ``rotate()`` closes the current keep-alive connection and opens a new one, so + the rotating gateway hands out a different residential exit IP. Because the + warmed cookies bind to the exit IP, ``rotate()`` also drops the warmed state + — the next fetch re-warms on the new IP. Used sequentially within a single + flow (never shared across concurrent tasks), so no locking is needed. + ``session`` is ``None`` only when no proxy is configured. + """ + + def __init__(self) -> None: + self._cm: Any | None = None + self.session: Any | None = None + self.rotations = 0 + self.warmed = False + self._last_at = 0.0 + + async def _open(self) -> None: + proxy = get_proxy_url() + self.warmed = False + if proxy is None: + self._cm = self.session = None + return + self._cm = FetcherSession( + proxy=proxy, + stealthy_headers=True, + impersonate="chrome", + timeout=_REQUEST_TIMEOUT_S, + ) + self.session = await self._cm.__aenter__() + + async def close(self) -> None: + if self._cm is not None: + with suppress(Exception): # best-effort teardown + await self._cm.__aexit__(None, None, None) + self._cm = self.session = None + + async def rotate(self) -> Any | None: + """Drop the current IP and connect through a fresh one. Returns new session.""" + await self.close() + self.rotations += 1 + await self._open() + logger.info( + "[instagram] rotated proxy session (rotation #%d)", self.rotations + ) + return self.session + + async def pace(self) -> None: + """Sleep to hold this sticky IP under Instagram's per-IP rate threshold.""" + wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_at) + if wait > 0: + await asyncio.sleep(wait + random.uniform(0, _PACE_JITTER_S)) + self._last_at = time.monotonic() + + +async def open_proxy_holder() -> _RotatingSession: + """Open a warm rotate-on-block session holder (caller owns ``close()``).""" + holder = _RotatingSession() + await holder._open() + return holder + + +@asynccontextmanager +async def bind_proxy_holder(holder: _RotatingSession): + """Route this task's fetches through ``holder`` for the enclosed block. + + Does NOT close the holder — enables pooling warm sessions across sequential + jobs so each job skips the proxy handshake AND the cookie warm-up. + """ + token = _current_session.set(holder) + try: + yield holder + finally: + _current_session.reset(token) + + +@asynccontextmanager +async def proxy_session(): + """Open one reused, rotate-on-block proxy session for a continuation chain.""" + holder = await open_proxy_holder() + try: + async with bind_proxy_holder(holder): + yield holder + finally: + await holder.close() + + +async def warm_session(session: Any) -> bool: + """Mint anonymous ``csrftoken``/``mid`` cookies on a freshly opened session. + + Returns ``True`` when a ``csrftoken`` was issued (the session can now reach + the web endpoints), else ``False`` (caller rotates the IP and retries). + + Takes an already-open ``session`` (never constructs one) so tests can drive + warm/rotate deterministically with a fake session, exactly like the reddit + sibling's fetch-resilience tests. + """ + seen: set[str] = set() + with suppress(Exception): + page = await session.get(_WARM_URL, headers=_HEADERS) + seen |= _response_cookie_names(page) + return _CSRF_COOKIE in seen + + +async def _get_page(session: Any, url: str) -> Any: + """GET through the warmed sticky session, or a one-shot proxied fetch.""" + if session is not None: + return await session.get(url, headers=_HEADERS) + return await AsyncFetcher.get( + url, + headers=_HEADERS, + proxy=get_proxy_url(), + stealthy_headers=True, + timeout=_REQUEST_TIMEOUT_S, + ) + + +async def resolve_redirect(url: str) -> str | None: + """Follow a ``share/`` short URL to its canonical target, or ``None``. + + ``share/`` links redirect to the real post/profile URL; the resolver records + the original as ``redirectedFromUrl``. Best-effort: returns the final URL + when the session exposes it, else ``None``. + """ + holder = _current_session.get() + if holder is None: + async with proxy_session(): + return await resolve_redirect(url) + with suppress(Exception): + page = await _get_page(holder.session, url) + final = getattr(page, "url", None) + if isinstance(final, str) and final and final != url: + return final + return None + + +async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | None: + """GET an Instagram web endpoint through a warmed HTTP session. + + Returns parsed JSON (dict or list), or ``None`` on 404 / non-block failure. + Warms cookies once per session; rotates the residential IP and re-warms on + 401/403; backs off on 429. Raises :class:`InstagramAccessBlockedError` only + when every rotated IP refuses anonymous access (the login-wall branch, which + we cannot satisfy). + """ + holder = _current_session.get() + if holder is None: + # No bound session (e.g. a direct call outside fan_out): open a + # short-lived warmed session for this one fetch, then tear it down. + async with proxy_session(): + return await fetch_json(path, params) + + url = _build_url(path, params) + attempt = 0 + backoffs = 0 + while True: + session = holder.session + try: + if session is not None and not holder.warmed: + warmed_ok = await warm_session(session) + holder.warmed = True # attempted; don't re-warm this IP + if not warmed_ok: + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + raise InstagramAccessBlockedError( + f"could not warm session after {attempt} IP rotations for {path}" + ) + + await holder.pace() + page = await _get_page(session, url) + status = page.status + + if status == 200: + return _parse_json(page) + if status == 404: + return None + if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS: + backoffs += 1 + delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1)) + logger.warning( + "[instagram] 429 on %s; backing off %.1fs", path, delay + ) + await asyncio.sleep(delay + random.uniform(0, 1)) + continue + if status in _ROTATE_STATUSES and attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + if status in _ROTATE_STATUSES: + raise InstagramAccessBlockedError( + f"Instagram refused {path} on {attempt} rotated IPs ({status})" + ) + logger.warning("[instagram] GET %s returned %s", path, status) + return None + except InstagramAccessBlockedError: + raise + except Exception as e: + logger.warning("[instagram] GET %s failed: %s", path, e) + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue + return None From 6270250f74e34024da568b7046b76aac34922b04 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:38 +0530 Subject: [PATCH 10/72] feat(instagram): add fan-out orchestrator with deterministic early-stop cleanup --- .../platforms/instagram/__init__.py | 24 + .../platforms/instagram/scraper.py | 480 ++++++++++++++++++ 2 files changed, 504 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/scraper.py diff --git a/surfsense_backend/app/proprietary/platforms/instagram/__init__.py b/surfsense_backend/app/proprietary/platforms/instagram/__init__.py new file mode 100644 index 000000000..e3a2a122a --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/__init__.py @@ -0,0 +1,24 @@ +"""Platform-native Instagram scraper (anonymous, no browser).""" + +from .fetch import InstagramAccessBlockedError +from .schemas import ( + InstagramComment, + InstagramHashtag, + InstagramMediaItem, + InstagramPlace, + InstagramProfile, + InstagramScrapeInput, +) +from .scraper import iter_instagram, scrape_instagram + +__all__ = [ + "InstagramAccessBlockedError", + "InstagramComment", + "InstagramHashtag", + "InstagramMediaItem", + "InstagramPlace", + "InstagramProfile", + "InstagramScrapeInput", + "iter_instagram", + "scrape_instagram", +] diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py new file mode 100644 index 000000000..46569f4dd --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -0,0 +1,480 @@ +"""Orchestrator for the Instagram scraper. + +The core is the async generator :func:`iter_instagram` (unbounded); +:func:`scrape_instagram` is a thin collector with a caller-supplied ``limit`` +guard. Any cap is caller policy, never baked into flow logic. + +Independent targets (one per ``directUrl`` / discovered entity) fan out +concurrently on a pool of warm sessions (sticky IPs); each target's own paging +stays sequential. ``fan_out`` is ported from ``../reddit/scraper.py`` but bound +to *this* module's proxy holders so every worker warms its own session once and +reuses it. + +Flows are selected by ``resultsType``: +- ``posts`` / ``reels`` / ``mentions`` -> media items (profile / hashtag feeds, + or discovery search) +- ``comments`` -> comment items for post/reel URLs +- ``details`` -> profile / hashtag / place metadata (by URL or discovery search) + +ponytail: deep feed pagination (past the first web page of media) needs the +GraphQL cursor endpoint whose doc-id drifts; v1 emits the first page and stops. +The upgrade path is a ``_paginate_feed`` helper in this file plus a doc-id in +``fetch.py`` — contained to these two files, per the acquisition-seam rule. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator +from contextlib import aclosing +from datetime import UTC, datetime, timedelta +from typing import Any + +from .fetch import ( + InstagramAccessBlockedError, + bind_proxy_holder, + fetch_json, + now_iso, + open_proxy_holder, +) +from .parsers import ( + parse_comment, + parse_hashtag, + parse_media, + parse_place, + parse_profile, +) +from .schemas import InstagramScrapeInput +from .url_resolver import ResolvedUrl, resolve_url + +logger = logging.getLogger(__name__) + +__all__ = [ + "InstagramAccessBlockedError", + "iter_instagram", + "scrape_instagram", +] + +# Independent jobs run concurrently on a pool of warm proxy sessions. Anonymous +# Instagram is the most hostile platform, so this stays low to avoid burning the +# residential pool with parallel login walls. +_FANOUT_CONCURRENCY = 8 + +# Per-post comment fetches fan across their own warm sessions; kept below the +# top-level width so N concurrent targets x this can't explode the IP count. +_COMMENT_CONCURRENCY = 4 + +_PROFILE_PATH = "api/v1/users/web_profile_info/" +_HASHTAG_PATH = "api/v1/tags/web_info/" +_LOCATION_PATH = "api/v1/locations/web_info/" +_SEARCH_PATH = "web/search/topsearch/" + + +def _parse_newer_than(value: str | None) -> datetime | None: + """Parse ``onlyPostsNewerThan`` (ISO, YYYY-MM-DD, or relative) to UTC. + + Relative forms: ``" "`` where unit is minute/hour/day/week/month/ + year (singular or plural). Anything unparseable returns ``None`` (no filter). + """ + if not value: + return None + text = value.strip().lower() + parts = text.split() + if len(parts) == 2 and parts[0].isdigit(): + n = int(parts[0]) + unit = parts[1].rstrip("s") + days = { + "minute": n / 1440, + "hour": n / 24, + "day": n, + "week": n * 7, + "month": n * 30, + "year": n * 365, + }.get(unit) + if days is None: + return None + return datetime.now(UTC) - timedelta(days=days) + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + if dt.tzinfo: + return dt + return dt.replace(tzinfo=UTC) + except ValueError: + return None + + +def _is_after(timestamp: str | None, cutoff: datetime | None) -> bool: + """True if the item ``timestamp`` (ISO) is at/after the cutoff (or no cutoff).""" + if cutoff is None: + return True + if not timestamp: + return True + try: + dt = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + return dt >= cutoff + except ValueError: + return True + + +async def fan_out( + jobs: list[AsyncIterator[dict[str, Any]]], *, concurrency: int = _FANOUT_CONCURRENCY +) -> AsyncIterator[dict[str, Any]]: + """Stream items from independent async-iterator jobs via a warm worker pool. + + Each worker opens ONE proxy session and reuses it across the sequential jobs + it pulls, so only the first job per worker pays the proxy handshake + the + cookie warm-up. A bad job yields nothing rather than aborting the batch; + workers are cancelled and their sessions closed if the consumer stops early. + """ + if not jobs: + return + job_queue: asyncio.Queue[AsyncIterator[dict[str, Any]]] = asyncio.Queue() + for job in jobs: + job_queue.put_nowait(job) + results: asyncio.Queue[list[dict[str, Any]]] = asyncio.Queue() + + async def worker() -> None: + holder = None + try: + holder = await open_proxy_holder() + except Exception as e: # no session: jobs still run via one-shot fetches + logger.warning("[instagram] proxy session open failed: %s", e) + try: + while True: + try: + job = job_queue.get_nowait() + except asyncio.QueueEmpty: + return + items: list[dict[str, Any]] = [] + try: + if holder is not None: + async with bind_proxy_holder(holder): + items = [item async for item in job] + else: + items = [item async for item in job] + except InstagramAccessBlockedError: + raise # a hard login wall must abort the batch, not be swallowed + except Exception as e: # one bad target must not kill the run + logger.warning("[instagram] fan-out job failed: %s", e) + await results.put(items) + finally: + if holder is not None: + await holder.close() + + tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))] + try: + for _ in range(len(jobs)): + for item in await results.get(): + yield item + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +def _emit(partial: dict[str, Any], *, input_url: str | None) -> dict[str, Any]: + """Stamp provenance and serialize (parsers return plain dicts).""" + out = {**partial, "scrapedAt": now_iso()} + if input_url is not None: + out.setdefault("inputUrl", input_url) + return out + + +async def _profile_user(username: str) -> dict[str, Any] | None: + """Fetch a profile's ``data.user`` node, or ``None``.""" + data = await fetch_json(_PROFILE_PATH, {"username": username}) + if isinstance(data, dict): + user = ( + data.get("data", {}).get("user") + if isinstance(data.get("data"), dict) + else None + ) + if isinstance(user, dict): + return user + return None + return None + + +def _media_matches(item: dict[str, Any], result_type: str) -> bool: + """Filter a media item by feed type. ``reels`` keeps clips/videos only.""" + if result_type == "reels": + return item.get("type") == "Video" or item.get("productType") == "clips" + return True + + +async def _media_flow( + resolved: ResolvedUrl, + *, + input_model: InstagramScrapeInput, + cutoff: datetime | None, + per_target: int, +) -> AsyncIterator[dict[str, Any]]: + """Emit media items for a profile / hashtag / place URL.""" + from .parsers import _edges + + result_type = input_model.resultsType + if resolved.kind == "profile": + user = await _profile_user(resolved.value) + if user is None: + return + nodes = _edges(user.get("edge_owner_to_timeline_media")) + emitted = 0 + for node in nodes: + item = parse_media(node) + if input_model.skipPinnedPosts and item.get("isPinned"): + continue + if not _media_matches(item, result_type): + continue + if not _is_after(item.get("timestamp"), cutoff): + continue + yield _emit(item, input_url=resolved.url) + emitted += 1 + if emitted >= per_target: + return + return + if resolved.kind == "hashtag": + data = await fetch_json(_HASHTAG_PATH, {"tag_name": resolved.value}) + if isinstance(data, dict): + parsed = parse_hashtag(data) + emitted = 0 + for node in [*parsed.get("topPosts", []), *parsed.get("posts", [])]: + if not _media_matches(node, result_type): + continue + if not _is_after(node.get("timestamp"), cutoff): + continue + yield _emit(node, input_url=resolved.url) + emitted += 1 + if emitted >= per_target: + return + return + if resolved.kind == "place": + data = await fetch_json(_LOCATION_PATH, {"location_id": resolved.value}) + if isinstance(data, dict): + parsed = parse_place(data) + emitted = 0 + for node in parsed.get("posts", []): + if not _is_after(node.get("timestamp"), cutoff): + continue + yield _emit(node, input_url=resolved.url) + emitted += 1 + if emitted >= per_target: + return + return + + +async def _comments_flow( + resolved: ResolvedUrl, + *, + input_model: InstagramScrapeInput, + per_target: int, +) -> AsyncIterator[dict[str, Any]]: + """Emit comment items for a post / reel URL. + + ponytail: the anonymous comment page uses a GraphQL cursor whose doc-id + drifts; v1 sources the comments embedded in the media info payload and caps + at the actor's 50/post ceiling. Deeper paging is the upgrade path in + ``fetch.py``. + """ + from .parsers import _edges + + path = f"p/{resolved.value}/" + data = await fetch_json(path, {"__a": 1, "__d": "dis"}) + node = None + if isinstance(data, dict): + items = data.get("items") + if isinstance(items, list) and items: + node = items[0] + else: + gql = data.get("graphql") + node = gql.get("shortcode_media") if isinstance(gql, dict) else None + if not isinstance(node, dict): + return + comment_nodes = _edges(node.get("edge_media_to_parent_comment")) or _edges( + node.get("edge_media_to_comment") + ) + cap = min(per_target, 50) + emitted = 0 + for cnode in comment_nodes: + item = parse_comment(cnode, post_url=resolved.url) + yield _emit(item, input_url=resolved.url) + emitted += 1 + if input_model.includeNestedComments: + for reply in _edges(cnode.get("edge_threaded_comments")): + if emitted >= cap: + return + yield _emit( + parse_comment(reply, post_url=resolved.url), + input_url=resolved.url, + ) + emitted += 1 + if emitted >= cap: + return + + +async def _details_flow( + resolved: ResolvedUrl, *, input_model: InstagramScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Emit one profile / hashtag / place detail item for a URL.""" + if resolved.kind == "profile": + user = await _profile_user(resolved.value) + if user is not None: + yield _emit(parse_profile(user), input_url=resolved.url) + return + if resolved.kind == "hashtag": + data = await fetch_json(_HASHTAG_PATH, {"tag_name": resolved.value}) + if isinstance(data, dict): + yield _emit(parse_hashtag(data), input_url=resolved.url) + return + if resolved.kind == "place": + data = await fetch_json(_LOCATION_PATH, {"location_id": resolved.value}) + if isinstance(data, dict): + yield _emit(parse_place(data), input_url=resolved.url) + return + + +async def _discover( + query: str, *, search_type: str, limit: int +) -> list[ResolvedUrl]: + """Resolve a discovery query into target URLs via topsearch.""" + data = await fetch_json(_SEARCH_PATH, {"query": query, "context": "blended"}) + if not isinstance(data, dict): + return [] + out: list[ResolvedUrl] = [] + if search_type in ("profile", "user"): + for entry in data.get("users", []): + user = entry.get("user", {}) if isinstance(entry, dict) else {} + name = user.get("username") + if not name: + continue + out.append( + ResolvedUrl("profile", name, f"https://www.instagram.com/{name}/") + ) + elif search_type == "hashtag": + for entry in data.get("hashtags", []): + tag = entry.get("hashtag", {}) if isinstance(entry, dict) else {} + name = tag.get("name") + if not name: + continue + out.append( + ResolvedUrl( + "hashtag", + name, + f"https://www.instagram.com/explore/tags/{name}/", + ) + ) + elif search_type == "place": + for entry in data.get("places", []): + place = entry.get("place", {}) if isinstance(entry, dict) else {} + loc = place.get("location", {}) if isinstance(place, dict) else {} + pk = loc.get("pk") or loc.get("id") + if not pk: + continue + out.append( + ResolvedUrl( + "place", + str(pk), + f"https://www.instagram.com/explore/locations/{pk}/", + ) + ) + return out[:limit] + + +def _resolve_inputs(input_model: InstagramScrapeInput) -> list[ResolvedUrl]: + """Resolve ``directUrls`` (URLs take priority over ``search``).""" + resolved: list[ResolvedUrl] = [] + for url in input_model.directUrls: + r = resolve_url(url) + if r is None: + logger.warning("[instagram] unrecognized URL: %s", url) + continue + resolved.append(r) + return resolved + + +async def _targets(input_model: InstagramScrapeInput) -> list[ResolvedUrl]: + """The resolved targets for this run: direct URLs, else discovery search.""" + if input_model.directUrls: + return _resolve_inputs(input_model) + if not input_model.search: + return [] + limit = input_model.searchLimit or 10 + queries = [q.strip() for q in input_model.search.split(",") if q.strip()] + targets: list[ResolvedUrl] = [] + for query in queries: + targets.extend( + await _discover(query, search_type=input_model.searchType, limit=limit) + ) + return targets + + +async def iter_instagram( + input_model: InstagramScrapeInput, +) -> AsyncIterator[dict[str, Any]]: + """Yield flat Instagram items. ``directUrls`` override ``search``. + + Independent targets fan out concurrently; each target's paging stays + sequential. De-dupes media by ``id`` across targets. + """ + targets = await _targets(input_model) + if not targets: + return + result_type = input_model.resultsType + cutoff = _parse_newer_than(input_model.onlyPostsNewerThan) + per_target = input_model.resultsLimit or 10 + + if result_type == "comments": + jobs = [ + _comments_flow(r, input_model=input_model, per_target=per_target) + for r in targets + if r.kind in ("post", "reel") + ] + async with aclosing(fan_out(jobs, concurrency=_COMMENT_CONCURRENCY)) as stream: + async for item in stream: + yield item + return + + if result_type == "details": + jobs = [_details_flow(r, input_model=input_model) for r in targets] + async with aclosing(fan_out(jobs)) as stream: + async for item in stream: + yield item + return + + # posts / reels / mentions -> media feeds, de-duped by id across targets. + jobs = [ + _media_flow( + r, input_model=input_model, cutoff=cutoff, per_target=per_target + ) + for r in targets + ] + seen: set[str] = set() + async with aclosing(fan_out(jobs)) as stream: + async for item in stream: + item_id = item.get("id") + if isinstance(item_id, str): + if item_id in seen: + continue + seen.add(item_id) + yield item + + +async def scrape_instagram( + input_model: InstagramScrapeInput, *, limit: int | None = None +) -> list[dict[str, Any]]: + """Collect :func:`iter_instagram` into a list, honoring an optional ``limit``. + + ``limit`` is a request-time policy guard, NOT a ceiling in the streaming + core. + """ + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + async with aclosing(iter_instagram(input_model)) as stream: + async for item in stream: + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + break + return results From eccc142974e1b6cb8960cd55e340ed25aadec669 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:47 +0530 Subject: [PATCH 11/72] docs(instagram): add platform scraper README --- .../proprietary/platforms/instagram/README.md | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/instagram/README.md diff --git a/surfsense_backend/app/proprietary/platforms/instagram/README.md b/surfsense_backend/app/proprietary/platforms/instagram/README.md new file mode 100644 index 000000000..fa42e30a6 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/instagram/README.md @@ -0,0 +1,105 @@ +# Instagram scraper (anonymous, no browser) + +Platform-native Instagram scraper (anonymous, no browser). Standalone module: it +depends only on `app.utils.proxy` + `scrapling` and exposes a stable public API. +Its input/output surface is a **reference-compatible** mirror of the public +Instagram scraper actor spec (same `resultsType` / `directUrls` / camelCase field +names, additive `extra="allow"` parity), so callers written against that surface +work unchanged. It is **not** wired into ingestion or Celery — the capability +layer under `app/capabilities/instagram/` is what turns these primitives into +REST + agent + MCP surfaces. + +## Approach + +Instagram's public web app exposes anonymous, logged-out JSON behind a handful of +`www.instagram.com` endpoints once a session carries an anonymous `csrftoken` + +`mid` cookie pair and the `x-ig-app-id` web header: + +> Warm an anonymous session with one plain GET to `www.instagram.com/` (mints +> `csrftoken` + `mid`), then GET the web JSON endpoints through that same +> Chrome-impersonated, sticky-IP session. Rotate the residential IP + re-warm on +> a login wall (401/403), back off on 429. + +Endpoints used (anonymous web tier only): + +| Flow | Endpoint | +|---|---| +| profile / posts / reels | `api/v1/users/web_profile_info/?username=…` | +| comments | `p//?__a=1&__d=dis` | +| hashtag | `api/v1/tags/web_info/?tag_name=…` | +| place | `api/v1/locations/web_info/?location_id=…` | +| discovery search | `web/search/topsearch/?query=…` | + +**No browser, no Chromium, no `solve_cloudflare`** — this stays on the cheap HTTP +tier the sibling scrapers already use. + +## Anonymous only — no authentication, ever + +No login, no `sessionid` account cookie, no app password. The only cookies held +are the anonymous `csrftoken` / `mid` minted by the warm-up. There is **no** +authenticate option in the input surface or the fetch layer, by design. A +persistent block after IP rotation surfaces as `InstagramAccessBlockedError` +(mirrors Reddit's `RedditAccessBlockedError`) rather than a silent empty result, +so the capability layer can map it to a `403 INSTAGRAM_ACCESS_BLOCKED`. + +## Module map + +| File | Responsibility | +|---|---| +| `__init__.py` | Public exports: `InstagramScrapeInput`, item models, `iter_instagram`, `scrape_instagram`, `InstagramAccessBlockedError`. | +| `schemas.py` | `InstagramScrapeInput` (`extra="allow"`, no auth fields) + optional-field item models (`InstagramMediaItem`, `InstagramComment`, `InstagramProfile`, `InstagramHashtag`, `InstagramPlace`) each with `to_output()`. | +| `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (csrftoken/mid) + `fetch_json`. No browser imports. | +| `url_resolver.py` | Classify an Instagram URL → `profile`/`post`/`reel`/`hashtag`/`place`; non-Instagram → `None`. Strips `_u/`, `profilecard/`; story → profile. | +| `parsers.py` | Pure JSON→dict mapping (`parse_media`, `parse_comment`, `parse_profile`, `parse_hashtag`, `parse_place`, `_edges`). I/O-free. | +| `scraper.py` | Orchestrator: `_media_flow`/`_comments_flow`/`_details_flow`/`_discover`, `_targets`, `fan_out`, `iter_instagram`, `scrape_instagram`. | + +## How it works + +1. `iter_instagram` resolves `directUrls` (or runs a discovery `search` per the + comma-split queries) into targets and fans them out on a pool of warm proxy + sessions (`fan_out`, 8-way; 4-way for comments). Each worker opens one + sticky-IP session and warms `csrftoken`/`mid` once, reusing it across the + sequential targets it pulls. +2. `resultsType` selects the flow: `posts`/`reels`/`mentions` → media feeds, + `comments` → per-post comment items, `details` → profile/hashtag/place + metadata. Media items de-dupe by `id` across targets. +3. `fetch_json` warms the session on first use, rotates the IP + re-warms on + 401/403, backs off on 429, returns `None` on 404. +4. Parsers map raw web JSON to flat dicts; the orchestrator stamps `scrapedAt` + and applies `resultsLimit` / `onlyPostsNewerThan` as request-time policy. + +## Observed limits & calibration caveats + +- Anonymous web JSON is rate-limited per IP; the sticky-session pool keeps each + IP's request rate modest but a hot pool will still hit login walls — that's the + `InstagramAccessBlockedError` path, not a bug. +- `likesCount` is frequently withheld on anonymous responses (surfaces as `-1` or + absent upstream); treat it as best-effort. +- Comments on the anonymous media page cap at ~50/post; deeper paging needs the + GraphQL cursor endpoint whose doc-id drifts (see the `ponytail:` note in + `scraper.py`/`fetch.py`). +- The `$3.50 / 1k items` default meter assumes the proxy-bytes-per-item measured + on the reference targets; re-measure with the `references/` scale harness before + high-volume production use. + +## Testing + +- Offline unit tests: `tests/unit/platforms/instagram/` — `test_skeleton.py` + (schema + URL resolver), `test_parsers.py` (fixture-pinned mapping), + `test_fetch_resilience.py` (warm / rotate / backoff loop with fake sessions, no + network), `test_budget.py` (fair-share caps + de-dup). +- Live e2e (needs network + residential proxy): `scripts/e2e_instagram_scraper.py` + — step 0 is the go/no-go cookie probe; later steps exercise the flows and dump + trimmed, PII-anonymized fixtures. + +```bash +cd surfsense_backend +.venv/bin/python -m pytest tests/unit/platforms/instagram/ +.venv/bin/python scripts/e2e_instagram_scraper.py # live; regenerates fixtures +``` + +## TODO / out of scope (v1) + +- Deep feed pagination past the first web page of media (GraphQL cursor doc-id). +- Deep comment pagination past the ~50/post embedded ceiling. +- Sticky-IP provider parity (same `__sid` caveat as the Reddit sibling). From ff55537bce0144a14fca0bc1bedc55ba80d0229e Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:54 +0530 Subject: [PATCH 12/72] feat(instagram): add scrape capability verb --- .../capabilities/instagram/scrape/__init__.py | 3 + .../instagram/scrape/definition.py | 23 ++++ .../capabilities/instagram/scrape/executor.py | 54 +++++++++ .../capabilities/instagram/scrape/schemas.py | 105 ++++++++++++++++++ 4 files changed, 185 insertions(+) create mode 100644 surfsense_backend/app/capabilities/instagram/scrape/__init__.py create mode 100644 surfsense_backend/app/capabilities/instagram/scrape/definition.py create mode 100644 surfsense_backend/app/capabilities/instagram/scrape/executor.py create mode 100644 surfsense_backend/app/capabilities/instagram/scrape/schemas.py diff --git a/surfsense_backend/app/capabilities/instagram/scrape/__init__.py b/surfsense_backend/app/capabilities/instagram/scrape/__init__.py new file mode 100644 index 000000000..de8b3c7c4 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/scrape/__init__.py @@ -0,0 +1,3 @@ +"""``instagram.scrape`` verb: Instagram URLs / search terms → posts, reels, mentions.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/instagram/scrape/definition.py b/surfsense_backend/app/capabilities/instagram/scrape/definition.py new file mode 100644 index 000000000..e84ca4938 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/scrape/definition.py @@ -0,0 +1,23 @@ +"""``instagram.scrape`` capability registration (billed per item; see config +``INSTAGRAM_SCRAPE_MICROS_PER_ITEM``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.instagram.scrape.executor import build_scrape_executor +from app.capabilities.instagram.scrape.schemas import ScrapeInput, ScrapeOutput + +INSTAGRAM_SCRAPE = Capability( + name="instagram.scrape", + description=( + "Scrape public Instagram posts, reels, or mentions from " + "profile/post/hashtag/place URLs, or discover content via search queries." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.INSTAGRAM_ITEM, + docs_url="/docs/connectors/native/instagram", +) + +register_capability(INSTAGRAM_SCRAPE) diff --git a/surfsense_backend/app/capabilities/instagram/scrape/executor.py b/surfsense_backend/app/capabilities/instagram/scrape/executor.py new file mode 100644 index 000000000..a76722dcd --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/scrape/executor.py @@ -0,0 +1,54 @@ +"""``instagram.scrape`` executor: verb input → scraper → media items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.instagram.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.instagram import ( + InstagramAccessBlockedError, + InstagramScrapeInput, + scrape_instagram, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_instagram + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + actor_input = InstagramScrapeInput( + resultsType=payload.result_type, + directUrls=payload.urls, + search=",".join(payload.search_queries), + searchType=payload.search_type, + resultsLimit=payload.max_per_target, + onlyPostsNewerThan=payload.newer_than, + skipPinnedPosts=payload.skip_pinned_posts, + addParentData=payload.add_parent_data, + ) + emit_progress( + "starting", + "Resolving Instagram targets", + total=payload.max_items, + unit="item", + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except InstagramAccessBlockedError as exc: + # Anonymous-only scraper; a hard block can't be retried with creds. + raise ForbiddenError( + f"Instagram refused anonymous access: {exc}", + code="INSTAGRAM_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item" + ) + return ScrapeOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/instagram/scrape/schemas.py b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py new file mode 100644 index 000000000..0e5646dc7 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py @@ -0,0 +1,105 @@ +"""``instagram.scrape`` I/O contracts. + +A lean, agent-friendly surface over ``InstagramScrapeInput`` +(``app/proprietary/platforms/instagram``). The executor maps this to the full +scraper input; the scraper's ``InstagramMediaItem`` is reused verbatim as the +output element. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.instagram import InstagramMediaItem + +MAX_INSTAGRAM_SOURCES = 20 +"""Per-call cap on urls + search_queries: bounds a synchronous request's fan-out.""" + +MAX_INSTAGRAM_ITEMS = 100 +"""Hard ceiling on items returned per call, regardless of the per-target caps.""" + + +class ScrapeInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_INSTAGRAM_SOURCES, + description=( + "Instagram URLs or bare profile IDs: profile, post (/p/), reel " + "(/reel/), hashtag (/explore/tags/), or place (/explore/locations/). " + "Provide these OR search_queries (never both)." + ), + ) + search_queries: list[str] = Field( + default_factory=list, + max_length=MAX_INSTAGRAM_SOURCES, + description=( + "Discovery keywords (hashtags as plaintext, no '#'). Provide these " + "OR urls (never both)." + ), + ) + search_type: Literal["hashtag", "profile", "place", "user"] = Field( + default="hashtag", + description="What to discover from search_queries. Only used with search_queries.", + ) + result_type: Literal["posts", "reels", "mentions"] = Field( + default="posts", + description="Which feed to return. 'mentions' requires profile URLs.", + ) + newer_than: str | None = Field( + default=None, + description=( + "Only return posts newer than this: YYYY-MM-DD, ISO timestamp, or " + "relative ('1 day', '2 months'); UTC." + ), + ) + skip_pinned_posts: bool = Field( + default=False, + description="Exclude pinned posts (posts mode).", + ) + max_per_target: int = Field( + default=10, + ge=1, + description="Max results per URL or per discovered target.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_INSTAGRAM_ITEMS, + description="Max total items to return across all sources.", + ) + add_parent_data: bool = Field( + default=False, + description="Attach a dataSource block to each feed item.", + ) + + @model_validator(mode="after") + def _exactly_one_source(self) -> ScrapeInput: + if not self.urls and not self.search_queries: + raise ValueError( + "Provide at least one of 'urls' or 'search_queries'." + ) + if self.urls and self.search_queries: + raise ValueError( + "Provide 'urls' OR 'search_queries', not both (they cannot be combined)." + ) + return self + + @property + def estimated_units(self) -> int: + """Worst-case billable items for the pre-flight gate: ``max_items`` is a + hard cross-source ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class ScrapeOutput(BaseModel): + items: list[InstagramMediaItem] = Field( + default_factory=list, + description="One media item per result (post/reel/mention), in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned item = one billable unit.""" + return len(self.items) From 38a26b1ccf0fad75615e25946517d756a1ec3145 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:58:58 +0530 Subject: [PATCH 13/72] feat(instagram): add comments capability verb --- .../instagram/comments/__init__.py | 3 + .../instagram/comments/definition.py | 22 +++++++ .../instagram/comments/executor.py | 53 ++++++++++++++++ .../instagram/comments/schemas.py | 61 +++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 surfsense_backend/app/capabilities/instagram/comments/__init__.py create mode 100644 surfsense_backend/app/capabilities/instagram/comments/definition.py create mode 100644 surfsense_backend/app/capabilities/instagram/comments/executor.py create mode 100644 surfsense_backend/app/capabilities/instagram/comments/schemas.py diff --git a/surfsense_backend/app/capabilities/instagram/comments/__init__.py b/surfsense_backend/app/capabilities/instagram/comments/__init__.py new file mode 100644 index 000000000..84899d758 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/comments/__init__.py @@ -0,0 +1,3 @@ +"""``instagram.comments`` verb: post/reel URLs → comments (and replies).""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/instagram/comments/definition.py b/surfsense_backend/app/capabilities/instagram/comments/definition.py new file mode 100644 index 000000000..7794228ac --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/comments/definition.py @@ -0,0 +1,22 @@ +"""``instagram.comments`` capability registration (billed per comment; see config +``INSTAGRAM_SCRAPE_MICROS_PER_COMMENT``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.instagram.comments.executor import build_comments_executor +from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput + +INSTAGRAM_COMMENTS = Capability( + name="instagram.comments", + description=( + "Fetch comments (and optionally replies) for Instagram post or reel URLs." + ), + input_schema=CommentsInput, + output_schema=CommentsOutput, + executor=build_comments_executor(), + billing_unit=BillingUnit.INSTAGRAM_COMMENT, + docs_url="/docs/connectors/native/instagram", +) + +register_capability(INSTAGRAM_COMMENTS) diff --git a/surfsense_backend/app/capabilities/instagram/comments/executor.py b/surfsense_backend/app/capabilities/instagram/comments/executor.py new file mode 100644 index 000000000..260ee06c9 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/comments/executor.py @@ -0,0 +1,53 @@ +"""``instagram.comments`` executor: verb input → scraper → comment items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.instagram import ( + InstagramAccessBlockedError, + InstagramScrapeInput, + scrape_instagram, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_comments_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_instagram + + async def execute(payload: CommentsInput) -> CommentsOutput: + actor_input = InstagramScrapeInput( + resultsType="comments", + directUrls=payload.urls, + resultsLimit=payload.max_comments_per_post, + isNewestComments=payload.newest_first, + includeNestedComments=payload.include_replies, + ) + emit_progress( + "starting", + "Fetching Instagram comments", + total=payload.max_items, + unit="comment", + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except InstagramAccessBlockedError as exc: + raise ForbiddenError( + f"Instagram refused anonymous access: {exc}", + code="INSTAGRAM_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", + f"Scraped {len(items)} comment(s)", + current=len(items), + unit="comment", + ) + return CommentsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/instagram/comments/schemas.py b/surfsense_backend/app/capabilities/instagram/comments/schemas.py new file mode 100644 index 000000000..fadb83215 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/comments/schemas.py @@ -0,0 +1,61 @@ +"""``instagram.comments`` I/O contracts. + +A lean surface over ``InstagramScrapeInput`` (``resultsType="comments"``). The +scraper's ``InstagramComment`` is reused verbatim as the output element. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from app.capabilities.instagram.scrape.schemas import ( + MAX_INSTAGRAM_ITEMS, + MAX_INSTAGRAM_SOURCES, +) +from app.proprietary.platforms.instagram import InstagramComment + +MAX_COMMENTS_PER_POST = 50 +"""Anonymous web media pages surface at most ~50 comments per post.""" + + +class CommentsInput(BaseModel): + urls: list[str] = Field( + min_length=1, + max_length=MAX_INSTAGRAM_SOURCES, + description="Post or reel URLs to fetch comments for (shortCode or numeric-ID forms).", + ) + newest_first: bool = Field( + default=False, + description="Return newest comments first.", + ) + include_replies: bool = Field( + default=False, + description="Include nested replies; each reply is a separate billable item.", + ) + max_comments_per_post: int = Field( + default=10, + ge=1, + le=MAX_COMMENTS_PER_POST, + description="Max comments per post (Instagram caps at 50).", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_INSTAGRAM_ITEMS, + description="Max total comments to return across all posts.", + ) + + @property + def estimated_units(self) -> int: + return self.max_items + + +class CommentsOutput(BaseModel): + items: list[InstagramComment] = Field( + default_factory=list, + description="One item per comment (or reply), in emission order.", + ) + + @property + def billable_units(self) -> int: + return len(self.items) From 7ea474be938da3b70d21672a41bb1d2b070ca831 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:01:38 +0530 Subject: [PATCH 14/72] feat(instagram): add details capability verb --- .../instagram/details/__init__.py | 3 + .../instagram/details/definition.py | 23 +++++ .../instagram/details/executor.py | 50 +++++++++++ .../capabilities/instagram/details/schemas.py | 86 +++++++++++++++++++ 4 files changed, 162 insertions(+) create mode 100644 surfsense_backend/app/capabilities/instagram/details/__init__.py create mode 100644 surfsense_backend/app/capabilities/instagram/details/definition.py create mode 100644 surfsense_backend/app/capabilities/instagram/details/executor.py create mode 100644 surfsense_backend/app/capabilities/instagram/details/schemas.py diff --git a/surfsense_backend/app/capabilities/instagram/details/__init__.py b/surfsense_backend/app/capabilities/instagram/details/__init__.py new file mode 100644 index 000000000..8e4c85258 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/details/__init__.py @@ -0,0 +1,3 @@ +"""``instagram.details`` verb: profile/hashtag/place URLs or search → metadata.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/instagram/details/definition.py b/surfsense_backend/app/capabilities/instagram/details/definition.py new file mode 100644 index 000000000..fa7f912d8 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/details/definition.py @@ -0,0 +1,23 @@ +"""``instagram.details`` capability registration (billed per item; see config +``INSTAGRAM_SCRAPE_MICROS_PER_ITEM``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.instagram.details.executor import build_details_executor +from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput + +INSTAGRAM_DETAILS = Capability( + name="instagram.details", + description=( + "Fetch Instagram profile, hashtag, or place metadata by URL or discovery " + "search. Each item carries a detailKind discriminator." + ), + input_schema=DetailsInput, + output_schema=DetailsOutput, + executor=build_details_executor(), + billing_unit=BillingUnit.INSTAGRAM_ITEM, + docs_url="/docs/connectors/native/instagram", +) + +register_capability(INSTAGRAM_DETAILS) diff --git a/surfsense_backend/app/capabilities/instagram/details/executor.py b/surfsense_backend/app/capabilities/instagram/details/executor.py new file mode 100644 index 000000000..f9a152f50 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/details/executor.py @@ -0,0 +1,50 @@ +"""``instagram.details`` executor: verb input → scraper → detail items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.instagram import ( + InstagramAccessBlockedError, + InstagramScrapeInput, + scrape_instagram, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_details_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_instagram + + async def execute(payload: DetailsInput) -> DetailsOutput: + actor_input = InstagramScrapeInput( + resultsType="details", + directUrls=payload.urls, + search=",".join(payload.search_queries), + searchType=payload.search_type, + searchLimit=payload.search_limit, + ) + emit_progress( + "starting", + "Resolving Instagram detail targets", + total=payload.max_items, + unit="item", + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except InstagramAccessBlockedError as exc: + raise ForbiddenError( + f"Instagram refused anonymous access: {exc}", + code="INSTAGRAM_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item" + ) + return DetailsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/instagram/details/schemas.py b/surfsense_backend/app/capabilities/instagram/details/schemas.py new file mode 100644 index 000000000..31972b095 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/details/schemas.py @@ -0,0 +1,86 @@ +"""``instagram.details`` I/O contracts. + +A lean surface over ``InstagramScrapeInput`` (``resultsType="details"``). Each +output item is a profile / hashtag / place, discriminated by the synthesized +``detailKind`` field (a SurfSense addition; every other field mirrors the actor). +""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import BaseModel, Field, model_validator + +from app.capabilities.instagram.scrape.schemas import ( + MAX_INSTAGRAM_ITEMS, + MAX_INSTAGRAM_SOURCES, +) +from app.proprietary.platforms.instagram import ( + InstagramHashtag, + InstagramPlace, + InstagramProfile, +) + +InstagramDetailItem = Annotated[ + InstagramProfile | InstagramHashtag | InstagramPlace, + Field(discriminator="detailKind"), +] + + +class DetailsInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_INSTAGRAM_SOURCES, + description=( + "Profile / hashtag / place URLs (or bare profile IDs). The URL type " + "determines the detail kind. Provide these OR search_queries." + ), + ) + search_queries: list[str] = Field( + default_factory=list, + max_length=MAX_INSTAGRAM_SOURCES, + description="Discovery keywords. Provide these OR urls (never both).", + ) + search_type: Literal["hashtag", "profile", "place"] = Field( + default="hashtag", + description="What to discover from search_queries (no 'user' — use instagram.scrape).", + ) + search_limit: int = Field( + default=10, + ge=1, + le=MAX_INSTAGRAM_ITEMS, + description="Max discovered entities per query.", + ) + max_items: int = Field( + default=10, + ge=1, + le=MAX_INSTAGRAM_ITEMS, + description="Max total detail items to return.", + ) + + @model_validator(mode="after") + def _exactly_one_source(self) -> DetailsInput: + if not self.urls and not self.search_queries: + raise ValueError( + "Provide at least one of 'urls' or 'search_queries'." + ) + if self.urls and self.search_queries: + raise ValueError( + "Provide 'urls' OR 'search_queries', not both (they cannot be combined)." + ) + return self + + @property + def estimated_units(self) -> int: + return self.max_items + + +class DetailsOutput(BaseModel): + items: list[InstagramDetailItem] = Field( + default_factory=list, + description="One item per profile/hashtag/place, keyed by detailKind.", + ) + + @property + def billable_units(self) -> int: + return len(self.items) From 929efe8152a83abd4cb4c7ae78aa58aa16bf8df5 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:01:49 +0530 Subject: [PATCH 15/72] feat(instagram): register instagram.* capability namespace --- surfsense_backend/app/capabilities/instagram/__init__.py | 7 +++++++ surfsense_backend/app/routes/__init__.py | 1 + 2 files changed, 8 insertions(+) create mode 100644 surfsense_backend/app/capabilities/instagram/__init__.py diff --git a/surfsense_backend/app/capabilities/instagram/__init__.py b/surfsense_backend/app/capabilities/instagram/__init__.py new file mode 100644 index 000000000..4004f86c7 --- /dev/null +++ b/surfsense_backend/app/capabilities/instagram/__init__.py @@ -0,0 +1,7 @@ +"""``instagram.*`` namespace: platform-native Instagram data verbs.""" + +from __future__ import annotations + +from app.capabilities.instagram.comments import definition as _comments # noqa: F401 +from app.capabilities.instagram.details import definition as _details # noqa: F401 +from app.capabilities.instagram.scrape import definition as _scrape # noqa: F401 diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 1a5b182b8..d15cfdc40 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -3,6 +3,7 @@ from fastapi import APIRouter, Depends # Import verb namespaces for their registration side effects before the door builds. import app.capabilities.google_maps import app.capabilities.google_search +import app.capabilities.instagram import app.capabilities.reddit import app.capabilities.web import app.capabilities.youtube # noqa: F401 From c5e3fd4006975075f603e3fa7d7581e0280e0291 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:01:55 +0530 Subject: [PATCH 16/72] feat(mcp): add Instagram scrape/comments/details tools --- .../mcp_server/features/scrapers/__init__.py | 12 +- .../features/scrapers/platforms/instagram.py | 194 ++++++++++++++++++ surfsense_mcp/mcp_server/selfcheck.py | 3 + surfsense_mcp/mcp_server/server.py | 3 +- 4 files changed, 209 insertions(+), 3 deletions(-) create mode 100644 surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index dfa2f3ab2..aade7a99d 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -13,9 +13,17 @@ from mcp.server.fastmcp import FastMCP from ...core.client import SurfSenseClient from ...core.workspace_context import WorkspaceContext from . import run_history -from .platforms import google_maps, google_search, reddit, web, youtube +from .platforms import google_maps, google_search, instagram, reddit, web, youtube -_REGISTRARS = (web, google_search, reddit, youtube, google_maps, run_history) +_REGISTRARS = ( + web, + google_search, + reddit, + youtube, + google_maps, + instagram, + run_history, +) def register( diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py new file mode 100644 index 000000000..39dfea71d --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py @@ -0,0 +1,194 @@ +"""Instagram scraper tools: posts/reels, comments, and details.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +ResultType = Literal["posts", "reels", "mentions"] +SearchType = Literal["hashtag", "profile", "place", "user"] +DetailSearchType = Literal["hashtag", "profile", "place"] + + +def register( + mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext +) -> None: + """Register the Instagram scrape, comments, and details tools.""" + + @mcp.tool( + name="surfsense_instagram_scrape", + title="Scrape Instagram posts or reels", + annotations=SCRAPE, + structured_output=False, + ) + async def instagram_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="Instagram URLs: a profile, post (/p/), reel " + "(/reel/), hashtag (/explore/tags/), or place " + "(/explore/locations/). Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to discover content for (hashtags as plain " + "text, no '#'). Provide search_queries OR urls." + ), + ] = None, + search_type: Annotated[ + SearchType, Field(description="What to discover from search_queries.") + ] = "hashtag", + result_type: Annotated[ + ResultType, + Field(description="Which feed to return. 'mentions' needs profile URLs."), + ] = "posts", + max_items: Annotated[ + int, Field(ge=1, description="Maximum items to return across sources.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape public Instagram posts or reels from URLs or search queries. + + Use this for Instagram content research: a creator's recent posts, a + hashtag or location feed, or discovering posts by keyword. For a post's + comment section use surfsense_instagram_comments; for profile/hashtag/ + place metadata use surfsense_instagram_details. Returns per-item caption, + likes, comments count, media URLs, and owner. + Example: urls=['https://www.instagram.com/natgeo/'], result_type='reels'. + """ + return await run_scraper( + client, + context, + platform="instagram", + verb="scrape", + payload={ + "urls": urls, + "search_queries": search_queries, + "search_type": search_type, + "result_type": result_type, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_instagram_comments", + title="Fetch Instagram comments", + annotations=SCRAPE, + structured_output=False, + ) + async def instagram_comments( + urls: Annotated[ + list[str], + Field( + min_length=1, + description="Instagram post or reel URLs, e.g. " + "['https://www.instagram.com/p/Cabc123/'].", + ), + ], + max_comments_per_post: Annotated[ + int, + Field(ge=1, le=50, description="Max comments per post (Instagram caps at 50)."), + ] = 10, + include_replies: Annotated[ + bool, Field(description="Include nested replies.") + ] = False, + newest_first: Annotated[ + bool, Field(description="Return newest comments first.") + ] = False, + max_items: Annotated[ + int, Field(ge=1, description="Max total comments across all posts.") + ] = 20, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch the comments (and optionally replies) on Instagram posts or reels. + + Use this when the user wants a post's discussion or audience reaction + rather than the post itself; get post URLs from surfsense_instagram_scrape + if you only have a topic or profile. Returns comment text, author, likes, + and replies. + Example: urls=['https://www.instagram.com/p/Cabc123/'], include_replies=True. + """ + return await run_scraper( + client, + context, + platform="instagram", + verb="comments", + payload={ + "urls": urls, + "max_comments_per_post": max_comments_per_post, + "include_replies": include_replies, + "newest_first": newest_first, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_instagram_details", + title="Fetch Instagram profile/hashtag/place details", + annotations=SCRAPE, + structured_output=False, + ) + async def instagram_details( + urls: Annotated[ + list[str] | None, + Field( + description="Profile, hashtag, or place URLs (or bare profile " + "IDs). Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Terms to discover profiles/hashtags/places for. " + "Provide search_queries OR urls." + ), + ] = None, + search_type: Annotated[ + DetailSearchType, + Field(description="What to discover from search_queries."), + ] = "hashtag", + max_items: Annotated[ + int, Field(ge=1, description="Max detail items to return.") + ] = 10, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch Instagram profile, hashtag, or place metadata. + + Use this for entity lookups: a profile's follower/post counts and bio, a + hashtag's post volume and top posts, or a place's coordinates and post + count. For a feed of posts use surfsense_instagram_scrape instead. Each + item carries a detailKind field marking whether it is a profile, hashtag, + or place. + Example: urls=['https://www.instagram.com/explore/tags/crossfit/']. + """ + return await run_scraper( + client, + context, + platform="instagram", + verb="details", + payload={ + "urls": urls, + "search_queries": search_queries, + "search_type": search_type, + "max_items": max_items, + }, + workspace=workspace, + response_format=response_format, + ) diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index 20224c173..f7b7a76c5 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -25,6 +25,9 @@ EXPECTED_TOOLS = { "surfsense_youtube_comments", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", + "surfsense_instagram_scrape", + "surfsense_instagram_comments", + "surfsense_instagram_details", "surfsense_list_scraper_runs", "surfsense_get_scraper_run", # knowledge-base management diff --git a/surfsense_mcp/mcp_server/server.py b/surfsense_mcp/mcp_server/server.py index 8ea9bc919..19cf966f7 100644 --- a/surfsense_mcp/mcp_server/server.py +++ b/surfsense_mcp/mcp_server/server.py @@ -36,7 +36,8 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: "SurfSense gives you live scrapers and a personal knowledge base. " "Prefer these tools over generic/built-in web search whenever the " "task involves Reddit (posts, comments, finding subreddits or " - "communities), YouTube (videos, transcripts, comments), Google " + "communities), YouTube (videos, transcripts, comments), Instagram " + "(posts, reels, comments, profile/hashtag/place details), Google " "Maps (places, reviews), Google Search results, or reading " "specific web pages. Scraper results are persisted as runs; if an " "inline result is truncated, fetch it in full with " From 98eab8c4b5bef0fc98cf5bf7b7feb5e243687a7f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:02:05 +0530 Subject: [PATCH 17/72] feat(chat): add Instagram builtin subagent --- .../subagents/builtins/instagram/__init__.py | 1 + .../subagents/builtins/instagram/agent.py | 43 ++++++++++++ .../builtins/instagram/description.md | 2 + .../builtins/instagram/system_prompt.md | 68 +++++++++++++++++++ .../builtins/instagram/tools/__init__.py | 0 .../builtins/instagram/tools/index.py | 29 ++++++++ .../multi_agent_chat/subagents/registry.py | 4 ++ 7 files changed, 147 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/agent.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/__init__.py new file mode 100644 index 000000000..5e3857c86 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/__init__.py @@ -0,0 +1 @@ +"""``instagram`` builtin subagent: structured Instagram posts, comments, and details.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/agent.py new file mode 100644 index 000000000..27a3e1bdb --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/agent.py @@ -0,0 +1,43 @@ +"""``instagram`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Pulls structured data from Instagram posts, reels, comments, and profiles." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md new file mode 100644 index 000000000..9efedc52a --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md @@ -0,0 +1,2 @@ +Instagram specialist: pulls structured data from Instagram — posts and reels (caption, likes, comments count, media URLs, owner, timestamp), a post's comments and replies, and profile/hashtag/place details (follower and post counts, bio, hashtag volume and top posts, place coordinates). Finds content by hashtag, profile, or place search, and compares fresh Instagram data against earlier findings in this chat. +Use whenever the task involves Instagram content or an instagram.com link. Triggers include "get this Instagram profile/post/reel", "find posts about X on Instagram", "how many followers/likes", "get the comments on this post", "what are people saying about this reel", "look up this hashtag/location", and comparisons against earlier Instagram results in this chat. Not for general web pages (use the web crawling specialist for non-Instagram URLs). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md new file mode 100644 index 000000000..7d5278afa --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md @@ -0,0 +1,68 @@ +You are the SurfSense Instagram sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from live Instagram data gathered with your verbs, comparing against earlier results already in this conversation when the task calls for it. + + + +- `instagram_scrape` +- `instagram_comments` +- `instagram_details` +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Known profile/post/reel/hashtag/place links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts, reels, or mentions). +- Finding content on a topic: call `instagram_scrape` with `search_queries` and the matching `search_type` (hashtag, profile, or place). +- Comments / sentiment on specific posts or reels: call `instagram_comments` with the post `urls`. +- Profile, hashtag, or place metadata (follower counts, bio, hashtag volume, coordinates): call `instagram_details`. +- Batch multiple URLs (or queries) into one call rather than many single-item calls. + +- Multi-post comment analysis: a batched comments result lists posts in order, so a truncated preview usually shows only the first post(s). Before summarizing, page the stored run (or `search_run` by post id) until you have read real comments for EVERY post in the batch — never infer one post's sentiment from another's, and never report a post as "limited data" while its comments sit unread in the run. +- Comparison requests: pull the current values, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new). + + + +- Use only tools in ``. +- An item whose `status` is not `success` returned no data — report it unavailable, never invent it. +- Anonymous Instagram access can be rate-limited or blocked; if a verb returns no data, report it unavailable and suggest a narrower retry rather than fabricating. +- Report only deltas you can point to in the evidence. Never fabricate facts, counts, quotes, or URLs. + + + +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Non-Instagram web pages belong to the web crawling specialist, not here. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable URL or search query — return `status=blocked` with the missing fields. +- Tool failure or access block: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the URLs you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: max 10 entries, each a single sentence stating one distinct fact or delta. Do not paste raw payloads. +- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once. + + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py new file mode 100644 index 000000000..1bfbabad9 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py @@ -0,0 +1,29 @@ +"""``instagram`` sub-agent tools: the three Instagram capability verbs.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.instagram.comments.definition import INSTAGRAM_COMMENTS +from app.capabilities.instagram.details.definition import INSTAGRAM_DETAILS +from app.capabilities.instagram.scrape.definition import INSTAGRAM_SCRAPE + +NAME = "instagram" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [INSTAGRAM_SCRAPE, INSTAGRAM_COMMENTS, INSTAGRAM_DETAILS] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 34895a514..7a9b5b1f7 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -21,6 +21,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.google_maps.agent impor from app.agents.chat.multi_agent_chat.subagents.builtins.google_search.agent import ( build_subagent as build_google_search_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.instagram.agent import ( + build_subagent as build_instagram_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent import ( build_subagent as build_knowledge_base_subagent, ) @@ -79,6 +82,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "google_drive": build_google_drive_subagent, "google_maps": build_google_maps_subagent, "google_search": build_google_search_subagent, + "instagram": build_instagram_subagent, "knowledge_base": build_knowledge_base_subagent, "mcp_discovery": build_mcp_discovery_subagent, "memory": build_memory_subagent, From 9da136070d638ce7df2befa81245bd957d564969 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:02:41 +0530 Subject: [PATCH 18/72] test(instagram): add platform-layer unit tests --- .../unit/platforms/instagram/__init__.py | 0 .../unit/platforms/instagram/test_budget.py | 97 ++++++ .../instagram/test_fetch_resilience.py | 308 ++++++++++++++++++ .../unit/platforms/instagram/test_parsers.py | 153 +++++++++ .../unit/platforms/instagram/test_skeleton.py | 106 ++++++ 5 files changed, 664 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/instagram/__init__.py create mode 100644 surfsense_backend/tests/unit/platforms/instagram/test_budget.py create mode 100644 surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py create mode 100644 surfsense_backend/tests/unit/platforms/instagram/test_parsers.py create mode 100644 surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py diff --git a/surfsense_backend/tests/unit/platforms/instagram/__init__.py b/surfsense_backend/tests/unit/platforms/instagram/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_budget.py b/surfsense_backend/tests/unit/platforms/instagram/test_budget.py new file mode 100644 index 000000000..ea500be40 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/test_budget.py @@ -0,0 +1,97 @@ +"""Offline budget tests: per-target caps, cross-target de-dup, and the limit guard. + +No network. ``fetch_json`` is stubbed with a synthetic profile payload and the +fan-out proxy holders are replaced with no-ops, so the orchestrator's paging and +de-dup policy is exercised deterministically. +""" + +from __future__ import annotations + +from contextlib import asynccontextmanager + +import pytest + +from app.proprietary.platforms.instagram import scraper +from app.proprietary.platforms.instagram.schemas import InstagramScrapeInput + + +class _NoopHolder: + async def close(self) -> None: + return None + + +@pytest.fixture +def _stub_proxy(monkeypatch): + async def _open(): + return _NoopHolder() + + @asynccontextmanager + async def _bind(_holder): + yield _holder + + monkeypatch.setattr(scraper, "open_proxy_holder", _open) + monkeypatch.setattr(scraper, "bind_proxy_holder", _bind) + + +def _profile_payload(n: int) -> dict: + return { + "data": { + "user": { + "id": "9", + "username": "acct", + "edge_owner_to_timeline_media": { + "count": n, + "edges": [ + {"node": {"id": str(i), "shortcode": f"S{i}"}} + for i in range(n) + ], + }, + } + } + } + + +async def test_per_target_cap_limits_media(_stub_proxy, monkeypatch): + async def _fetch(path, params=None): + return _profile_payload(50) + + monkeypatch.setattr(scraper, "fetch_json", _fetch) + model = InstagramScrapeInput( + resultsType="posts", + directUrls=["https://www.instagram.com/acct/"], + resultsLimit=5, + ) + items = [i async for i in scraper.iter_instagram(model)] + assert len(items) == 5 + + +async def test_cross_target_dedup_by_id(_stub_proxy, monkeypatch): + async def _fetch(path, params=None): + return _profile_payload(3) # both targets return ids 0,1,2 + + monkeypatch.setattr(scraper, "fetch_json", _fetch) + model = InstagramScrapeInput( + resultsType="posts", + directUrls=[ + "https://www.instagram.com/one/", + "https://www.instagram.com/two/", + ], + resultsLimit=10, + ) + items = [i async for i in scraper.iter_instagram(model)] + ids = sorted(i["id"] for i in items) + assert ids == ["0", "1", "2"] + + +async def test_scrape_instagram_honors_limit(_stub_proxy, monkeypatch): + async def _fetch(path, params=None): + return _profile_payload(50) + + monkeypatch.setattr(scraper, "fetch_json", _fetch) + model = InstagramScrapeInput( + resultsType="posts", + directUrls=["https://www.instagram.com/acct/"], + resultsLimit=100, + ) + items = await scraper.scrape_instagram(model, limit=7) + assert len(items) == 7 diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py new file mode 100644 index 000000000..eefae2fd9 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -0,0 +1,308 @@ +"""Offline resilience tests for the Instagram fetch seam and fan-out worker pool. + +No network. Fake sessions drive the ``csrftoken`` warm-up + rotate-on-block + +backoff paths deterministically (in live runs the first IP warms and returns +200s, so these branches rarely fire). Mirrors the reddit sibling's +``test_fetch_resilience.py`` shape, adapted to Instagram's cookie warm-up. +""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from app.proprietary.platforms.instagram import fetch, scraper +from app.proprietary.platforms.instagram.fetch import ( + InstagramAccessBlockedError, + _current_session, + fetch_json, +) + +_PAYLOAD = {"data": {"user": {"username": "natgeo"}}} + + +class _FakePage: + def __init__(self, status: int, *, cookies: dict | None = None, payload=None): + self.status = status + self.cookies = cookies or {} + self._payload = payload if payload is not None else _PAYLOAD + + def json(self): + return self._payload + + @property + def body(self) -> str: + return json.dumps(self._payload) + + +class _FakeSession: + """One 'IP': the warm-up GET mints csrftoken per flag; endpoint GETs return ``status``.""" + + def __init__(self, status: int = 200, *, csrftoken: bool = True, payload=None) -> None: + self.status = status + self.csrftoken = csrftoken + self.payload = payload + self.json_calls = 0 + self.warm_calls = 0 + + async def get(self, url, headers=None, cookies=None): + if url.rstrip("/") == "https://www.instagram.com": + self.warm_calls += 1 + ck = {"csrftoken": "x", "mid": "y"} if self.csrftoken else {} + return _FakePage(200, cookies=ck) + self.json_calls += 1 + return _FakePage(self.status, payload=self.payload) + + +class _FakeHolder: + """Holder whose ``rotate()`` advances to the next fake session (a new IP).""" + + def __init__(self, sessions: list[_FakeSession]) -> None: + self._sessions = sessions + self.session = sessions[0] + self.rotations = 0 + self.warmed = False + + async def rotate(self): + self.rotations += 1 + self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)] + self.warmed = False # cookies bind to the IP: re-warm on the fresh one + return self.session + + async def pace(self) -> None: + return None + + async def close(self) -> None: + return None + + +def _no_sleep(monkeypatch) -> None: + async def _noop(_seconds): + return None + + monkeypatch.setattr(fetch.asyncio, "sleep", _noop) + + +async def test_warms_then_returns_json(): + holder = _FakeHolder([_FakeSession(200, csrftoken=True)]) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/users/web_profile_info/", {"username": "natgeo"}) + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 0 + assert holder.session.warm_calls == 1 # warmed exactly once + + +async def test_rotates_when_warm_fails_then_succeeds(): + holder = _FakeHolder( + [_FakeSession(200, csrftoken=False), _FakeSession(200, csrftoken=True)] + ) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/users/web_profile_info/") + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 1 + + +async def test_raises_when_no_ip_can_warm(): + holder = _FakeHolder( + [_FakeSession(200, csrftoken=False) for _ in range(fetch._MAX_ROTATIONS + 1)] + ) + token = _current_session.set(holder) + try: + raised = False + try: + await fetch_json("api/v1/users/web_profile_info/") + except InstagramAccessBlockedError: + raised = True + finally: + _current_session.reset(token) + assert raised + assert holder.rotations == fetch._MAX_ROTATIONS + + +async def test_rotates_and_rewarms_on_403(): + holder = _FakeHolder([_FakeSession(403), _FakeSession(200)]) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/users/web_profile_info/") + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 1 + assert holder.session.warm_calls == 1 # re-warmed on the fresh IP + + +async def test_rotates_on_401_login_wall(): + holder = _FakeHolder([_FakeSession(401), _FakeSession(200)]) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/users/web_profile_info/") + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 1 + + +async def test_404_returns_none_without_rotating(): + holder = _FakeHolder([_FakeSession(404), _FakeSession(200)]) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/tags/web_info/") + finally: + _current_session.reset(token) + assert result is None + assert holder.rotations == 0 + + +async def test_429_backs_off_without_rotating(monkeypatch): + _no_sleep(monkeypatch) + session = _FakeSession(429) + + async def _get(url, headers=None, cookies=None): + if url.rstrip("/") == "https://www.instagram.com": + session.warm_calls += 1 + return _FakePage(200, cookies={"csrftoken": "x"}) + session.json_calls += 1 + return _FakePage(429 if session.json_calls == 1 else 200) + + session.get = _get # type: ignore[method-assign] + holder = _FakeHolder([session]) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/users/web_profile_info/") + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 0 + + +async def test_persistent_403_raises_blocked(monkeypatch): + _no_sleep(monkeypatch) + holder = _FakeHolder([_FakeSession(403) for _ in range(fetch._MAX_ROTATIONS + 1)]) + token = _current_session.set(holder) + try: + raised = False + try: + await fetch_json("api/v1/users/web_profile_info/") + except InstagramAccessBlockedError: + raised = True + finally: + _current_session.reset(token) + assert raised + assert holder.rotations == fetch._MAX_ROTATIONS + + +class _TrackingHolder: + """Fake fan-out session holder that records whether it was closed.""" + + def __init__(self) -> None: + self.closed = False + + async def close(self) -> None: + self.closed = True + + +async def test_fan_out_closes_all_sessions_on_early_stop(monkeypatch): + holders: list[_TrackingHolder] = [] + + async def _fake_open(): + h = _TrackingHolder() + holders.append(h) + return h + + @asynccontextmanager + async def _fake_bind(_holder): + yield _holder + + monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open) + monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind) + + async def _job(n: int) -> AsyncIterator[dict]: + for i in range(5): + yield {"job": n, "i": i} + + jobs = [_job(n) for n in range(20)] + gen = scraper.fan_out(jobs, concurrency=4) + collected = [] + async for item in gen: + collected.append(item) + if len(collected) >= 3: + break + await gen.aclose() + + assert len(collected) >= 3 + assert holders, "workers should have opened sessions" + assert all(h.closed for h in holders), "a worker leaked its proxy session" + + +async def test_fan_out_empty_jobs_is_noop(): + out = [x async for x in scraper.fan_out([])] + assert out == [] + + +def _profile_payload(username: str, n: int) -> dict: + # IDs namespaced per target so cross-target de-dup doesn't collapse them. + return { + "data": { + "user": { + "id": f"u_{username}", + "username": username, + "edge_owner_to_timeline_media": { + "count": n, + "edges": [{"node": {"id": f"{username}:{i}"}} for i in range(n)], + }, + } + } + } + + +async def test_scrape_instagram_closes_sessions_when_limit_stops_inflight_workers( + monkeypatch, +): + """Hitting ``limit`` must tear down the whole fan-out chain deterministically. + + Regression: closing the outer ``iter_instagram`` generator does NOT + synchronously close the inner ``fan_out`` it loops over — CPython defers that + to async-gen GC — so without an explicit ``aclosing`` the collector's early + ``break`` leaked every warm proxy session that was still mid-fetch. The + ``fan_out``-direct test misses this because instant jobs self-drain before + cancellation ever runs; here each fetch yields to the loop so workers are + genuinely in-flight when the limit trips. + """ + holders: list[_TrackingHolder] = [] + + async def _fake_open(): + h = _TrackingHolder() + holders.append(h) + return h + + @asynccontextmanager + async def _fake_bind(_holder): + yield _holder + + async def _fetch(path, params=None): + await asyncio.sleep(0) # yield control: keep sibling workers in-flight + username = (params or {}).get("username", "acct") + return _profile_payload(username, 5) + + monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open) + monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind) + monkeypatch.setattr(scraper, "fetch_json", _fetch) + + model = scraper.InstagramScrapeInput( + resultsType="posts", + directUrls=[f"https://www.instagram.com/acct{i}/" for i in range(50)], + resultsLimit=5, + ) + items = await scraper.scrape_instagram(model, limit=3) + + assert len(items) == 3 + assert holders, "workers should have opened sessions" + assert all(h.closed for h in holders), "early stop leaked a proxy session" diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py new file mode 100644 index 000000000..6f6c1411e --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -0,0 +1,153 @@ +"""Offline parser tests: raw web JSON -> flat item dicts. + +Synthetic nodes cover the GraphQL ``edge_*`` flattening the anonymous web +payloads use. A fixture-pinned test runs only when a captured fixture is present +(the live e2e script dumps trimmed, PII-anonymized fixtures), so the suite stays +green offline. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from app.proprietary.platforms.instagram.parsers import ( + parse_comment, + parse_hashtag, + parse_media, + parse_place, + parse_profile, +) + +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _edge(nodes: list[dict]) -> dict: + return {"edges": [{"node": n} for n in nodes]} + + +def test_parse_media_flattens_edges_and_extracts_tags(): + node = { + "id": "1", + "shortcode": "Cabc", + "__typename": "GraphImage", + "taken_at_timestamp": 1_600_000_000, + "edge_media_to_caption": _edge([{"text": "love #nasa shot @buzz"}]), + "edge_media_to_comment": {"count": 7}, + "edge_liked_by": {"count": 42}, + "owner": {"username": "natgeo", "id": "9"}, + } + item = parse_media(node) + assert item["shortCode"] == "Cabc" + assert item["type"] == "Image" + assert item["hashtags"] == ["nasa"] + assert item["mentions"] == ["buzz"] + assert item["commentsCount"] == 7 + assert item["likesCount"] == 42 + assert item["ownerUsername"] == "natgeo" + assert item["url"] == "https://www.instagram.com/p/Cabc/" + + +def test_parse_media_passes_through_hidden_like_count(): + # Instagram reports -1 when the creator hid likes; never coerce it away. + item = parse_media({"id": "1", "edge_liked_by": {"count": -1}}) + assert item["likesCount"] == -1 + + +def test_parse_media_marks_video_type(): + item = parse_media({"id": "1", "is_video": True, "video_view_count": 99}) + assert item["type"] == "Video" + assert item["videoViewCount"] == 99 + + +def test_parse_comment(): + node = { + "id": "c1", + "text": "nice", + "created_at": 1_600_000_000, + "shortcode": "Cabc", + "owner": {"username": "bob", "id": "5"}, + "edge_liked_by": {"count": 3}, + } + item = parse_comment(node, post_url="https://www.instagram.com/p/Cabc/") + assert item["id"] == "c1" + assert item["text"] == "nice" + assert item["ownerUsername"] == "bob" + assert item["likesCount"] == 3 + assert item["postUrl"] == "https://www.instagram.com/p/Cabc/" + + +def test_parse_profile_flattens_counts_and_latest_posts(): + user = { + "id": "9", + "username": "natgeo", + "full_name": "Nat Geo", + "edge_followed_by": {"count": 1000}, + "edge_follow": {"count": 50}, + "edge_owner_to_timeline_media": { + "count": 2, + "edges": [{"node": {"id": "p1", "shortcode": "A"}}], + }, + } + item = parse_profile(user) + assert item["detailKind"] == "profile" + assert item["username"] == "natgeo" + assert item["followersCount"] == 1000 + assert item["followsCount"] == 50 + assert item["postsCount"] == 2 + assert len(item["latestPosts"]) == 1 + + +def test_parse_hashtag(): + data = { + "data": { + "id": "h1", + "name": "crossfit", + "edge_hashtag_to_media": { + "count": 5, + "edges": [{"node": {"id": "m1", "shortcode": "A"}}], + }, + "edge_hashtag_to_top_posts": { + "edges": [{"node": {"id": "t1", "shortcode": "B"}}] + }, + } + } + item = parse_hashtag(data) + assert item["detailKind"] == "hashtag" + assert item["name"] == "crossfit" + assert item["postsCount"] == 5 + assert len(item["topPosts"]) == 1 + assert len(item["posts"]) == 1 + + +def test_parse_place(): + data = { + "location": { + "id": "7538318", + "name": "Copenhagen", + "slug": "copenhagen", + "edge_location_to_media": { + "count": 3, + "edges": [{"node": {"id": "m1", "shortcode": "A"}}], + }, + } + } + item = parse_place(data) + assert item["detailKind"] == "place" + assert item["name"] == "Copenhagen" + assert item["location_id"] == "7538318" + assert len(item["posts"]) == 1 + + +@pytest.mark.skipif( + not (_FIXTURES / "profile.json").exists(), + reason="captured fixture absent (run scripts/e2e_instagram_scraper.py to dump)", +) +def test_fixture_profile_maps(): + raw = json.loads((_FIXTURES / "profile.json").read_text()) + user = raw.get("data", {}).get("user", raw) + item = parse_profile(user) + assert item["detailKind"] == "profile" + assert item["username"] diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py b/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py new file mode 100644 index 000000000..59017c81c --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py @@ -0,0 +1,106 @@ +"""Offline skeleton tests: input surface parity + URL classification. + +No network. Locks the two invariants the reference-compatible surface promises — +no auth fields ever, and additive ``extra="allow"`` parity — plus the full +``url_resolver`` classification/normalization table (``_u/`` and profilecard +stripping, story→profile, ID-only locations, numeric post-ID flagging). +""" + +from __future__ import annotations + +from app.proprietary.platforms.instagram.schemas import ( + InstagramMediaItem, + InstagramScrapeInput, +) +from app.proprietary.platforms.instagram.url_resolver import resolve_url + + +def test_input_has_no_auth_fields(): + # Anonymous-only: the input surface must never expose a login/credential seam. + forbidden = { + "sessionid", + "username", + "password", + "cookies", + "authorization", + "proxyConfiguration", + "loginCredentials", + } + assert forbidden.isdisjoint(InstagramScrapeInput.model_fields) + + +def test_input_defaults(): + model = InstagramScrapeInput() + assert model.resultsType == "posts" + assert model.searchType == "hashtag" + assert model.directUrls == [] + assert model.addParentData is False + + +def test_input_allows_extra_inert_fields(): + # A reference field the acquisition layer doesn't source is accepted, not rejected. + model = InstagramScrapeInput(enhanceUserSearchWithFacebookPage="x") + assert model.model_dump().get("enhanceUserSearchWithFacebookPage") == "x" + + +def test_media_item_to_output_keeps_none_keys(): + out = InstagramMediaItem(id="123", shortCode="abc").to_output() + assert out["id"] == "123" + assert out["shortCode"] == "abc" + # Unsourced fields stay present as None / [] for additive parity. + assert out["likesCount"] is None + assert out["requestErrorMessages"] == [] + + +def test_resolve_profile(): + r = resolve_url("https://www.instagram.com/natgeo/") + assert r.kind == "profile" + assert r.value == "natgeo" + + +def test_resolve_bare_profile_id(): + r = resolve_url("natgeo") + assert r.kind == "profile" and r.value == "natgeo" + + +def test_resolve_post_and_reel(): + r = resolve_url("https://www.instagram.com/p/Cabc123/") + assert r.kind == "post" and r.value == "Cabc123" and r.numeric_post_id is False + r = resolve_url("https://www.instagram.com/reel/Cxyz/") + assert r.kind == "reel" and r.value == "Cxyz" + + +def test_resolve_hashtag(): + r = resolve_url("https://www.instagram.com/explore/tags/crossfit/") + assert r.kind == "hashtag" and r.value == "crossfit" + + +def test_resolve_place_with_slug_and_id_only(): + with_slug = resolve_url( + "https://www.instagram.com/explore/locations/7538318/copenhagen/" + ) + assert with_slug.kind == "place" and with_slug.value == "7538318" + assert with_slug.slug == "copenhagen" + id_only = resolve_url("https://www.instagram.com/explore/locations/7538318/") + assert id_only.kind == "place" and id_only.value == "7538318" + + +def test_resolve_strips_u_and_profilecard(): + stripped_u = resolve_url("https://www.instagram.com/_u/natgeo/") + assert stripped_u.kind == "profile" and stripped_u.value == "natgeo" + card = resolve_url("https://www.instagram.com/natgeo/profilecard/") + assert card.kind == "profile" and card.value == "natgeo" + + +def test_resolve_story_reduces_to_profile(): + r = resolve_url("https://www.instagram.com/stories/natgeo/12345/") + assert r.kind == "profile" and r.value == "natgeo" + + +def test_resolve_numeric_post_id_flagged(): + r = resolve_url("https://www.instagram.com/p/12345/") + assert r.kind == "post" and r.numeric_post_id is True + + +def test_resolve_rejects_non_instagram_host(): + assert resolve_url("https://example.com/natgeo/") is None From 2e66e714e2cd3818f3224232ef9b2502722d4c3f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:02:48 +0530 Subject: [PATCH 19/72] test(instagram): add capability unit tests --- .../unit/capabilities/instagram/__init__.py | 0 .../capabilities/instagram/test_executor.py | 104 ++++++++++++++++++ .../capabilities/instagram/test_registry.py | 35 ++++++ .../capabilities/instagram/test_schemas.py | 80 ++++++++++++++ 4 files changed, 219 insertions(+) create mode 100644 surfsense_backend/tests/unit/capabilities/instagram/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/instagram/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/instagram/test_registry.py create mode 100644 surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py diff --git a/surfsense_backend/tests/unit/capabilities/instagram/__init__.py b/surfsense_backend/tests/unit/capabilities/instagram/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py b/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py new file mode 100644 index 000000000..9087b0bcf --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py @@ -0,0 +1,104 @@ +"""Executor tests: lean verb input → ``InstagramScrapeInput`` mapping + wrapping. + +A fake scraper captures the actor input the executor built (no network), so the +snake_case→camelCase mapping and the ``InstagramAccessBlockedError`` → +``ForbiddenError`` translation are asserted deterministically. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.instagram.comments.executor import build_comments_executor +from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput +from app.capabilities.instagram.details.executor import build_details_executor +from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput +from app.capabilities.instagram.scrape.executor import build_scrape_executor +from app.capabilities.instagram.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.instagram import ( + InstagramAccessBlockedError, + InstagramScrapeInput, +) + +pytestmark = pytest.mark.unit + + +class _FakeScraper: + def __init__(self, items: list[dict]) -> None: + self.items = items + self.calls: list[tuple[InstagramScrapeInput, int | None]] = [] + + async def __call__(self, actor_input, *, limit=None): + self.calls.append((actor_input, limit)) + return self.items + + +async def test_scrape_maps_urls_and_wraps_items(): + fake = _FakeScraper([{"id": "1", "shortCode": "abc", "caption": "hi"}]) + execute = build_scrape_executor(fake) + out = await execute(ScrapeInput(urls=["https://www.instagram.com/natgeo/"])) + assert isinstance(out, ScrapeOutput) + assert out.items[0].shortCode == "abc" + actor_input, limit = fake.calls[0] + assert actor_input.resultsType == "posts" + assert actor_input.directUrls == ["https://www.instagram.com/natgeo/"] + assert actor_input.search == "" + assert limit == 10 # default max_items forwarded as the collector limit + + +async def test_scrape_joins_search_queries(): + fake = _FakeScraper([]) + execute = build_scrape_executor(fake) + await execute(ScrapeInput(search_queries=["fit", "gym"], search_type="hashtag")) + actor_input, _ = fake.calls[0] + assert actor_input.search == "fit,gym" + assert actor_input.searchType == "hashtag" + assert actor_input.directUrls == [] + + +async def test_scrape_access_blocked_maps_to_forbidden(): + async def _blocked(actor_input, *, limit=None): + raise InstagramAccessBlockedError("login wall") + + execute = build_scrape_executor(_blocked) + with pytest.raises(ForbiddenError): + await execute(ScrapeInput(urls=["x"])) + + +async def test_comments_maps_flags(): + fake = _FakeScraper([{"id": "c1", "text": "nice"}]) + execute = build_comments_executor(fake) + out = await execute( + CommentsInput( + urls=["https://www.instagram.com/p/Cabc/"], + newest_first=True, + include_replies=True, + max_comments_per_post=25, + ) + ) + assert isinstance(out, CommentsOutput) + assert out.items[0].text == "nice" + actor_input, _ = fake.calls[0] + assert actor_input.resultsType == "comments" + assert actor_input.isNewestComments is True + assert actor_input.includeNestedComments is True + assert actor_input.resultsLimit == 25 + + +async def test_details_maps_and_wraps_discriminated_items(): + fake = _FakeScraper( + [ + { + "detailKind": "profile", + "username": "natgeo", + "url": "https://www.instagram.com/natgeo/", + } + ] + ) + execute = build_details_executor(fake) + out = await execute(DetailsInput(urls=["https://www.instagram.com/natgeo/"])) + assert isinstance(out, DetailsOutput) + assert out.items[0].username == "natgeo" + actor_input, _ = fake.calls[0] + assert actor_input.resultsType == "details" diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py b/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py new file mode 100644 index 000000000..89257253a --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py @@ -0,0 +1,35 @@ +"""The instagram namespace registers its three verbs for the doors/agent to read. + +Unlike the stale reddit assertion (``billing_unit is None``), these assert the +real meters — the Capability definitions are the source of truth. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities import ( + instagram, # noqa: F401 — importing the namespace registers its verbs +) +from app.capabilities.core.store import get_capability +from app.capabilities.core.types import BillingUnit + +pytestmark = pytest.mark.unit + + +def test_instagram_scrape_registered_with_item_meter(): + cap = get_capability("instagram.scrape") + assert cap.name == "instagram.scrape" + assert cap.billing_unit is BillingUnit.INSTAGRAM_ITEM + + +def test_instagram_comments_registered_with_comment_meter(): + cap = get_capability("instagram.comments") + assert cap.name == "instagram.comments" + assert cap.billing_unit is BillingUnit.INSTAGRAM_COMMENT + + +def test_instagram_details_registered_with_item_meter(): + cap = get_capability("instagram.details") + assert cap.name == "instagram.details" + assert cap.billing_unit is BillingUnit.INSTAGRAM_ITEM diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py b/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py new file mode 100644 index 000000000..13efb8a48 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py @@ -0,0 +1,80 @@ +"""``instagram.*`` input guards: source exclusivity and bounded batches.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.instagram.comments.schemas import CommentsInput +from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput +from app.capabilities.instagram.scrape.schemas import ( + MAX_INSTAGRAM_ITEMS, + MAX_INSTAGRAM_SOURCES, + ScrapeInput, +) + +pytestmark = pytest.mark.unit + + +def test_scrape_rejects_no_source(): + with pytest.raises(ValidationError): + ScrapeInput() + + +def test_scrape_rejects_both_sources(): + with pytest.raises(ValidationError): + ScrapeInput(urls=["https://www.instagram.com/natgeo/"], search_queries=["fit"]) + + +def test_scrape_accepts_urls_only(): + payload = ScrapeInput(urls=["https://www.instagram.com/natgeo/"]) + assert payload.search_queries == [] + assert payload.estimated_units == payload.max_items + + +def test_scrape_bounds(): + with pytest.raises(ValidationError): + ScrapeInput( + urls=["https://www.instagram.com/x/"], + max_items=MAX_INSTAGRAM_ITEMS + 1, + ) + with pytest.raises(ValidationError): + ScrapeInput( + urls=[ + f"https://www.instagram.com/u{i}/" + for i in range(MAX_INSTAGRAM_SOURCES + 1) + ] + ) + + +def test_comments_requires_urls_and_caps_at_50(): + with pytest.raises(ValidationError): + CommentsInput(urls=[]) + with pytest.raises(ValidationError): + CommentsInput( + urls=["https://www.instagram.com/p/Cabc/"], max_comments_per_post=51 + ) + ok = CommentsInput( + urls=["https://www.instagram.com/p/Cabc/"], max_comments_per_post=50 + ) + assert ok.max_comments_per_post == 50 + + +def test_details_discriminated_union_selects_by_detail_kind(): + out = DetailsOutput( + items=[ + {"detailKind": "profile", "username": "natgeo"}, + {"detailKind": "hashtag", "name": "fit"}, + {"detailKind": "place", "name": "Copenhagen"}, + ] + ) + kinds = [type(i).__name__ for i in out.items] + assert kinds == ["InstagramProfile", "InstagramHashtag", "InstagramPlace"] + assert out.billable_units == 3 + + +def test_details_rejects_both_sources(): + with pytest.raises(ValidationError): + DetailsInput( + urls=["https://www.instagram.com/natgeo/"], search_queries=["x"] + ) From 458d223ef8bfbbe534ccd24c3801ac360ce12710 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:02:54 +0530 Subject: [PATCH 20/72] test(billing): cover Instagram item and comment billing --- .../tests/unit/capabilities/test_billing.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/surfsense_backend/tests/unit/capabilities/test_billing.py b/surfsense_backend/tests/unit/capabilities/test_billing.py index 36d9a2978..85187b5f5 100644 --- a/surfsense_backend/tests/unit/capabilities/test_billing.py +++ b/surfsense_backend/tests/unit/capabilities/test_billing.py @@ -427,3 +427,54 @@ async def test_platform_gate_disabled_is_noop(monkeypatch): ) session.execute.assert_not_called() + + +# =================================================================== +# Instagram per-item / per-comment billing +# =================================================================== + + +async def test_instagram_item_charges_owner_per_item( + monkeypatch, record_usage, _enable_platform_billing +): + monkeypatch.setattr(config, "INSTAGRAM_SCRAPE_MICROS_PER_ITEM", 3500) + session, user = _make_session(_OWNER, balance_micros=1_000_000) + + charged = await charge_capability( + _FakePlatformOutput(4), BillingUnit.INSTAGRAM_ITEM, _ctx(session) + ) + + assert charged == 4 * 3500 + assert user.credit_micros_balance == 1_000_000 - 4 * 3500 + kwargs = record_usage.await_args.kwargs + assert kwargs["usage_type"] == "instagram_item" + + +async def test_instagram_comment_charges_owner_per_comment( + monkeypatch, record_usage, _enable_platform_billing +): + monkeypatch.setattr(config, "INSTAGRAM_SCRAPE_MICROS_PER_COMMENT", 1500) + session, user = _make_session(_OWNER, balance_micros=1_000_000) + + charged = await charge_capability( + _FakePlatformOutput(6), BillingUnit.INSTAGRAM_COMMENT, _ctx(session) + ) + + assert charged == 6 * 1500 + assert user.credit_micros_balance == 1_000_000 - 6 * 1500 + kwargs = record_usage.await_args.kwargs + assert kwargs["usage_type"] == "instagram_comment" + + +async def test_instagram_gate_blocks_when_worst_case_exceeds_balance( + monkeypatch, _enable_platform_billing +): + monkeypatch.setattr(config, "INSTAGRAM_SCRAPE_MICROS_PER_ITEM", 3500) + session = _gate_session(_OWNER, balance_micros=5000) # affords 1 item, not 2 + + with pytest.raises(InsufficientCreditsError): + await gate_capability( + _FakePlatformInput(estimated_units=2), + BillingUnit.INSTAGRAM_ITEM, + _ctx(session), + ) From e151346fbed6a147c7096d3b3678a561566b894d Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:03:23 +0530 Subject: [PATCH 21/72] test(chat): include Instagram in subagent composition test --- .../unit/agents/multi_agent_chat/test_subagent_composition.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index c3ad04250..1d04f2ebf 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -32,6 +32,7 @@ _EXPECTED_SUBAGENTS = frozenset( "google_drive", "google_maps", "google_search", + "instagram", "knowledge_base", "mcp_discovery", "memory", From 7309ca5ca05e0a15d6c25e27634965948b948a27 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:03:29 +0530 Subject: [PATCH 22/72] test(instagram): add manual e2e scraper script --- .../scripts/e2e_instagram_scraper.py | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 surfsense_backend/scripts/e2e_instagram_scraper.py diff --git a/surfsense_backend/scripts/e2e_instagram_scraper.py b/surfsense_backend/scripts/e2e_instagram_scraper.py new file mode 100644 index 000000000..95625fd06 --- /dev/null +++ b/surfsense_backend/scripts/e2e_instagram_scraper.py @@ -0,0 +1,227 @@ +"""Manual functional e2e for the Instagram scraper (app/proprietary/platforms/instagram). + +Run from the backend directory: + cd surfsense_backend + uv run python scripts/e2e_instagram_scraper.py + # or: .venv/bin/python scripts/e2e_instagram_scraper.py + +This is NOT a pytest test (it needs live network + a residential/custom proxy). +It: + + Step 0 — go/no-go probe: open a proxy session, mint the anonymous + ``csrftoken``/``mid`` cookies, then fetch ``web_profile_info`` on the SAME + sticky IP and assert it returns a profile. If this fails the whole + approach is invalid — later steps are skipped. + Step 1 — scrape a profile's posts. + Step 2 — scrape a profile's reels. + Step 3 — fetch comments for a discovered post URL. + Step 4 — fetch profile / hashtag / place details. + Step 5 — run a discovery search. + Step 6 — dump trimmed, PII-anonymized raw fixtures into + tests/unit/platforms/instagram/fixtures/ for the offline parser tests. +""" + +import asyncio +import json +import sys +from pathlib import Path + +from dotenv import load_dotenv + +# --- bootstrap: load .env and put the backend root on sys.path before app.* --- +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +from app.proprietary.platforms.instagram import ( # noqa: E402 + InstagramScrapeInput, + scrape_instagram, +) +from app.proprietary.platforms.instagram.fetch import ( # noqa: E402 + fetch_json, + proxy_session, + warm_session, +) + +_PROFILE = "natgeo" +_HASHTAG = "https://www.instagram.com/explore/tags/travel/" +_PLACE = "https://www.instagram.com/explore/locations/213385402/new-york-new-york/" +_SEARCH_TERM = "coffee" + +_FIXTURE_DIR = ( + _BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures" +) + +# Fields to strip from dumped fixtures so we never commit PII / volatile tokens. +_PII_KEYS = frozenset( + {"profile_pic_url", "profile_pic_url_hd", "display_url", "video_url", "biography"} +) + + +def _hr(title: str) -> None: + print(f"\n{'=' * 70}\n{title}\n{'=' * 70}") + + +def _check(label: str, ok: bool, detail: str = "") -> bool: + print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}") + return ok + + +def _anonymize(obj): + """Recursively blank PII-ish string values so fixtures are safe to commit.""" + if isinstance(obj, dict): + return { + k: ("" if k in _PII_KEYS and v else _anonymize(v)) + for k, v in obj.items() + } + if isinstance(obj, list): + return [_anonymize(x) for x in obj] + return obj + + +async def step0_probe() -> bool: + _hr("STEP 0 — go/no-go: csrftoken warm-up + sticky web_profile_info") + async with proxy_session() as holder: + if holder.session is None: + return _check( + "proxy configured", False, "no proxy -> set PROXY_PROVIDER + creds" + ) + minted = await warm_session(holder.session) + holder.warmed = True # don't let fetch_json re-warm; we just warmed it + _check("csrftoken warm-up minted a session", minted) + data = await fetch_json( + "api/v1/users/web_profile_info/", {"username": _PROFILE} + ) + user = (data or {}).get("data", {}).get("user") if isinstance(data, dict) else None + print(f" web_profile_info({_PROFILE}) -> user={'yes' if user else 'no'}") + return _check("sticky web_profile_info", minted and bool(user)) + + +async def step1_posts() -> bool: + _hr("STEP 1 — profile posts") + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="posts", + directUrls=[f"https://www.instagram.com/{_PROFILE}/"], + resultsLimit=5, + ), + limit=5, + ) + for it in items[:5]: + print(f" - {it.get('shortCode')} | likes={it.get('likesCount')}") + return _check("profile returned posts", len(items) > 0, f"{len(items)} posts") + + +async def step2_reels() -> bool: + _hr("STEP 2 — profile reels") + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="reels", + directUrls=[f"https://www.instagram.com/{_PROFILE}/"], + resultsLimit=5, + ), + limit=5, + ) + print(f" {len(items)} reels for {_PROFILE}") + return _check("reels returned items", len(items) >= 0, f"{len(items)} reels") + + +async def step3_comments(post_url: str | None) -> bool: + _hr("STEP 3 — comments for a post") + if not post_url: + return _check("had a post URL", False, "step 1 found no post") + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="comments", directUrls=[post_url], resultsLimit=10 + ), + limit=10, + ) + print(f" {len(items)} comments for {post_url}") + return _check("comments returned", len(items) >= 0, f"{len(items)} comments") + + +async def step4_details() -> bool: + _hr("STEP 4 — profile / hashtag / place details") + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="details", + directUrls=[ + f"https://www.instagram.com/{_PROFILE}/", + _HASHTAG, + _PLACE, + ], + ), + limit=10, + ) + kinds = sorted({i.get("detailKind") for i in items}) + print(f" detail kinds={kinds}") + return _check("details returned", len(items) > 0, f"{len(items)} items {kinds}") + + +async def step5_search() -> bool: + _hr("STEP 5 — discovery search") + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="posts", + search=_SEARCH_TERM, + searchType="hashtag", + searchLimit=3, + resultsLimit=3, + ), + limit=9, + ) + print(f" {len(items)} items for search '{_SEARCH_TERM}'") + return _check("search returned results", len(items) >= 0, f"{len(items)} items") + + +async def step6_dump_fixtures() -> bool: + _hr("STEP 6 — dump trimmed, anonymized fixtures for offline tests") + profile = await fetch_json( + "api/v1/users/web_profile_info/", {"username": _PROFILE} + ) + _FIXTURE_DIR.mkdir(parents=True, exist_ok=True) + wrote = [] + if isinstance(profile, dict) and profile.get("data", {}).get("user"): + (_FIXTURE_DIR / "profile.json").write_text( + json.dumps(_anonymize(profile)), encoding="utf-8" + ) + wrote.append("profile.json") + return _check("dumped fixtures", bool(wrote), f"{wrote} -> {_FIXTURE_DIR}") + + +async def _first_post_url() -> str | None: + """Discover a live post URL from the target profile's first page.""" + items = await scrape_instagram( + InstagramScrapeInput( + resultsType="posts", + directUrls=[f"https://www.instagram.com/{_PROFILE}/"], + resultsLimit=1, + ), + limit=1, + ) + return items[0].get("url") if items else None + + +async def main() -> int: + results = [await step0_probe()] + if not results[-1]: + print("\ncookie probe failed — the approach is invalid on this IP/proxy.") + print("Aborting remaining steps.") + return 1 + results.append(await step1_posts()) + results.append(await step2_reels()) + post_url = await _first_post_url() + results.append(await step3_comments(post_url)) + results.append(await step4_details()) + results.append(await step5_search()) + results.append(await step6_dump_fixtures()) + _hr("SUMMARY") + print(f" {sum(results)}/{len(results)} steps passed") + return 0 if all(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) From d00030d97cac04b029e462950b5312fa1e2d0803 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:03:34 +0530 Subject: [PATCH 23/72] feat(web): add Instagram playground platform and brand icon --- surfsense_web/lib/playground/catalog.ts | 11 +++++++++++ surfsense_web/lib/playground/platform-icons.tsx | 1 + surfsense_web/public/connectors/instagram.svg | 1 + 3 files changed, 13 insertions(+) create mode 100644 surfsense_web/public/connectors/instagram.svg diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index 9b508228a..b1287e447 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -2,6 +2,7 @@ import type { ComponentType } from "react"; import { GoogleMapsIcon, GoogleSearchIcon, + InstagramIcon, RedditIcon, WebIcon, YouTubeIcon, @@ -48,6 +49,16 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ { name: "youtube.comments", verb: "comments", label: "Comments" }, ], }, + { + id: "instagram", + label: "Instagram", + icon: InstagramIcon, + verbs: [ + { name: "instagram.scrape", verb: "scrape", label: "Scrape" }, + { name: "instagram.comments", verb: "comments", label: "Comments" }, + { name: "instagram.details", verb: "details", label: "Details" }, + ], + }, { id: "google_maps", label: "Google Maps", diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx index e7a443c2f..460b7f7a3 100644 --- a/surfsense_web/lib/playground/platform-icons.tsx +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -24,6 +24,7 @@ function brandIcon(src: string, alt: string) { export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit"); export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube"); +export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram"); export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps"); export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search"); export const WebIcon = brandIcon("/connectors/web.svg", "Web"); diff --git a/surfsense_web/public/connectors/instagram.svg b/surfsense_web/public/connectors/instagram.svg new file mode 100644 index 000000000..81233a38e --- /dev/null +++ b/surfsense_web/public/connectors/instagram.svg @@ -0,0 +1 @@ + From fa2ac406f363d9ea966668f63fde6352bba0f2d2 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:07:19 +0530 Subject: [PATCH 24/72] feat(web): add Instagram connector marketing page --- .../lib/connectors-marketing/index.ts | 2 + .../lib/connectors-marketing/instagram.tsx | 291 ++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 surfsense_web/lib/connectors-marketing/instagram.tsx diff --git a/surfsense_web/lib/connectors-marketing/index.ts b/surfsense_web/lib/connectors-marketing/index.ts index 901670f8a..429fe9b59 100644 --- a/surfsense_web/lib/connectors-marketing/index.ts +++ b/surfsense_web/lib/connectors-marketing/index.ts @@ -1,5 +1,6 @@ import { googleMaps } from "./google-maps"; import { googleSearch } from "./google-search"; +import { instagram } from "./instagram"; import { reddit } from "./reddit"; import type { ConnectorPageContent } from "./types"; import { webCrawl } from "./web-crawl"; @@ -11,6 +12,7 @@ export type { ConnectorPageContent } from "./types"; const CONNECTOR_LIST: ConnectorPageContent[] = [ reddit, youtube, + instagram, googleMaps, googleSearch, webCrawl, diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx new file mode 100644 index 000000000..33b963eb8 --- /dev/null +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -0,0 +1,291 @@ +import { IconBrandInstagram } from "@tabler/icons-react"; +import type { ConnectorPageContent } from "./types"; + +export const instagram: ConnectorPageContent = { + slug: "instagram", + name: "Instagram", + icon: IconBrandInstagram, + + metaTitle: "Instagram Scraper API for Social Listening | SurfSense", + metaDescription: + "Scrape public Instagram posts, reels, comments, and profiles at scale with the SurfSense Instagram Scraper API. No login, no official API, plus a free tier. Start now.", + keywords: [ + "instagram scraper", + "instagram scraper api", + "instagram api", + "instagram api alternative", + "scrape instagram", + "instagram graph api alternative", + "instagram comment scraper", + "instagram profile scraper", + "instagram hashtag scraper", + "instagram data api", + "instagram mcp server", + "instagram sentiment analysis", + "social listening", + ], + + h1: "Instagram Scraper API for Social Listening and Creator Research", + heroLede: + "The SurfSense Instagram API extracts public posts, reels, comments, and profile, hashtag, and place details without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post and what their audiences say, so you spot trends and sentiment first.", + + transcript: { + prompt: "Pull recent reels from @competitor and tell me what the comments think", + toolCall: + 'instagram.scrape({ urls: ["instagram.com/competitor/"],\n result_type: "reels", max_items: 20 })', + rows: [ + { + primary: "Behind the scenes of our new launch", + secondary: "@competitor · 84.2k likes · 1,203 comments", + tag: "top reel", + }, + { + primary: "Comments skew positive on price, negative on shipping", + secondary: "1,203 comments · 0.71 positive", + tag: "sentiment", + }, + { + primary: "3 creators tagged asking for a collab", + secondary: "@a · @b · @c · buying intent", + tag: "lead signal", + }, + ], + resultSummary: "20 reels · 4,910 comments · surfaced in 2.4s", + }, + + extractIntro: + "Every call returns structured items keyed by type. Point the API at a profile, post, reel, hashtag, or place URL, or discover content with a search query.", + extractFields: [ + { + label: "Posts & Reels", + description: + "Caption, hashtags, mentions, like and comment counts, media URLs, dimensions, and timestamp.", + }, + { + label: "Comments", + description: "Comment text, author, like and reply counts, and nested replies for any post.", + }, + { + label: "Profiles", + description: + "Follower, following, and post counts, bio, external URL, verified and business flags.", + }, + { + label: "Hashtags", + description: "Post volume, top posts, and recent posts for any /explore/tags/ hashtag.", + }, + { + label: "Places", + description: "Name, coordinates, address, and recent posts for any /explore/locations/ place.", + }, + { + label: "Owner & Media", + description: + "Owner username and id on every item, plus image and video URLs, alt text, and view counts.", + }, + ], + + useCasesHeading: "What teams do with the Instagram API", + useCases: [ + { + title: "Creator and competitor monitoring", + description: + "Track what your competitors and target creators post, and how their audiences react. Feed the stream to an agent that flags viral formats, launches, and shifts in engagement the moment they land.", + }, + { + title: "Audience sentiment and comment mining", + description: + "Pull full comment threads on a post or reel and score them for sentiment, so you can measure how a launch, a collab, or a campaign actually resonated with real followers.", + }, + { + title: "Hashtag and trend research", + description: + "Map the volume and top content behind any hashtag before you spend on a campaign. Turn trend research into a content calendar your team can act on.", + }, + { + title: "Influencer vetting and outreach", + description: + "Verify a creator's follower count, post cadence, and real engagement from public data before you pay for a partnership, and surface who is already mentioning your brand.", + }, + ], + + comparison: { + heading: "An Instagram API alternative built for agents", + intro: + "The official Instagram Graph API requires a Business account, app review, and access tokens, and it can't read arbitrary public profiles. Here is how SurfSense compares.", + columnLabel: "Instagram Graph API", + rows: [ + { + feature: "Access", + official: "Business/Creator account + app review + tokens", + surfsense: "Public data only, no login or account required", + }, + { + feature: "Coverage", + official: "Mostly your own or connected accounts", + surfsense: "Any public profile, post, hashtag, or place", + }, + { + feature: "Comments", + official: "Limited to accounts you manage", + surfsense: "Public comments and replies on any post, up to 50 per post", + }, + { + feature: "Setup", + official: "Register an app, pass review, manage tokens", + surfsense: "One API key, or add the MCP server to your agent", + }, + { + feature: "Agent-ready", + official: "No; you build the harness yourself", + surfsense: "MCP server exposes instagram.scrape as a native tool", + }, + ], + }, + + api: { + platform: "instagram", + verb: "scrape", + mcpTool: "instagram.scrape", + requestBody: { + urls: ["instagram.com/natgeo/"], + result_type: "reels", + max_items: 20, + }, + }, + + schema: { + requestNote: + "Provide exactly one source: urls OR search_queries (never both). Up to 20 sources per call.", + request: [ + { + name: "urls", + type: "string[]", + defaultValue: "[]", + description: + "Instagram URLs or bare profile IDs: profile, post (/p/), reel (/reel/), hashtag (/explore/tags/), or place (/explore/locations/). Max 20.", + }, + { + name: "search_queries", + type: "string[]", + defaultValue: "[]", + description: + "Discovery keywords (hashtags as plaintext, no '#'). Provide these OR urls, not both. Max 20.", + }, + { + name: "search_type", + type: "string", + defaultValue: '"hashtag"', + description: + "What to discover from search_queries: hashtag, profile, place, or user. Only used with search_queries.", + }, + { + name: "result_type", + type: "string", + defaultValue: '"posts"', + description: "Which feed to return: posts, reels, or mentions. 'mentions' requires profile URLs.", + }, + { + name: "newer_than", + type: "string", + description: + "Only return posts newer than this: YYYY-MM-DD, ISO timestamp, or relative ('1 day', '2 months'), UTC.", + }, + { + name: "skip_pinned_posts", + type: "boolean", + defaultValue: "false", + description: "Exclude pinned posts in posts mode.", + }, + { + name: "max_per_target", + type: "integer", + defaultValue: "10", + description: "Max results per URL or per discovered target.", + }, + { + name: "max_items", + type: "integer", + defaultValue: "10", + description: "Max total items to return across all sources. 1 to 100.", + }, + { + name: "add_parent_data", + type: "boolean", + defaultValue: "false", + description: "Attach a dataSource block to each feed item describing its source.", + }, + ], + responseNote: + "The response is { items: [...] } with one flat media item per result. Fields the anonymous endpoints do not expose are null. One returned item is one billable unit.", + response: [ + { + name: "id / shortCode / url", + type: "string", + description: "Identity and provenance: the media id, shortcode, and canonical post URL.", + }, + { + name: "type", + type: "string", + description: "Media type: Image, Video, or Sidecar (carousel).", + }, + { + name: "caption / hashtags / mentions", + type: "string / string[]", + description: "Caption text plus the hashtags and @-mentions parsed from it.", + }, + { + name: "likesCount / commentsCount", + type: "integer", + description: "Engagement counts. likesCount is -1 when the creator hides likes.", + }, + { + name: "displayUrl / videoUrl / videoViewCount", + type: "string / integer", + description: "Media URLs and, for videos, the public view count.", + }, + { + name: "ownerUsername / ownerId / ownerFullName", + type: "string", + description: "The account that posted the item.", + }, + { + name: "timestamp", + type: "string", + description: "ISO timestamp for when the post was published.", + }, + ], + }, + + faq: [ + { + question: "Is scraping Instagram legal?", + answer: + "SurfSense reads only public Instagram data, the same posts, reels, and comments any logged-out visitor can see. It never logs in and cannot access private accounts or stories. As always, review Instagram's terms and your own compliance needs before you run at scale.", + }, + { + question: "Do I need an Instagram account or the Graph API?", + answer: + "No. This is an independent alternative to the Instagram Graph API, not a wrapper. You do not create a Business account, pass app review, or manage access tokens. You call the SurfSense API with one key, or add the MCP server to your agent, and get structured data back.", + }, + { + question: "What are the rate limits?", + answer: + "Each call caps at 100 returned items across all sources, with up to 20 URLs or search queries per request, and up to 50 comments per post. SurfSense manages the underlying request budget and proxy rotation for you, so you scale reads without managing tokens.", + }, + { + question: "Can I get comments and replies?", + answer: + "Yes. The instagram.comments verb returns public comments on any post or reel, with author, like and reply counts, and optionally the nested replies. Get post URLs first from instagram.scrape if you only have a topic or a profile.", + }, + ], + + related: [ + { label: "Reddit API", href: "/reddit" }, + { label: "YouTube API", href: "/youtube" }, + { label: "Google Maps API", href: "/google-maps" }, + { label: "SERP API", href: "/google-search" }, + { label: "SurfSense MCP Server", href: "/mcp-server" }, + { label: "Read the docs", href: "/docs" }, + ], +}; From 4e86710b26646bfa64bba06542c25964459a3d5b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:07:34 +0530 Subject: [PATCH 25/72] feat(web): cross-link Instagram from sibling connectors --- surfsense_web/lib/connectors-marketing/google-maps.tsx | 1 + surfsense_web/lib/connectors-marketing/google-search.tsx | 1 + surfsense_web/lib/connectors-marketing/reddit.tsx | 1 + surfsense_web/lib/connectors-marketing/web-crawl.tsx | 1 + surfsense_web/lib/connectors-marketing/youtube.tsx | 1 + 5 files changed, 5 insertions(+) diff --git a/surfsense_web/lib/connectors-marketing/google-maps.tsx b/surfsense_web/lib/connectors-marketing/google-maps.tsx index 97103ddac..03b1d247b 100644 --- a/surfsense_web/lib/connectors-marketing/google-maps.tsx +++ b/surfsense_web/lib/connectors-marketing/google-maps.tsx @@ -303,6 +303,7 @@ export const googleMaps: ConnectorPageContent = { related: [ { label: "Reddit API", href: "/reddit" }, { label: "YouTube API", href: "/youtube" }, + { label: "Instagram API", href: "/instagram" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, diff --git a/surfsense_web/lib/connectors-marketing/google-search.tsx b/surfsense_web/lib/connectors-marketing/google-search.tsx index 8cab4cd82..7f436f9b0 100644 --- a/surfsense_web/lib/connectors-marketing/google-search.tsx +++ b/surfsense_web/lib/connectors-marketing/google-search.tsx @@ -260,6 +260,7 @@ export const googleSearch: ConnectorPageContent = { { label: "Web Crawl API", href: "/web-crawl" }, { label: "Google Maps API", href: "/google-maps" }, { label: "Reddit API", href: "/reddit" }, + { label: "Instagram API", href: "/instagram" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, { label: "Read the docs", href: "/docs" }, ], diff --git a/surfsense_web/lib/connectors-marketing/reddit.tsx b/surfsense_web/lib/connectors-marketing/reddit.tsx index db85a969c..93a8618ef 100644 --- a/surfsense_web/lib/connectors-marketing/reddit.tsx +++ b/surfsense_web/lib/connectors-marketing/reddit.tsx @@ -310,6 +310,7 @@ export const reddit: ConnectorPageContent = { related: [ { label: "YouTube API", href: "/youtube" }, + { label: "Instagram API", href: "/instagram" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, diff --git a/surfsense_web/lib/connectors-marketing/web-crawl.tsx b/surfsense_web/lib/connectors-marketing/web-crawl.tsx index fe25a85fe..1cb4d5dac 100644 --- a/surfsense_web/lib/connectors-marketing/web-crawl.tsx +++ b/surfsense_web/lib/connectors-marketing/web-crawl.tsx @@ -282,6 +282,7 @@ export const webCrawl: ConnectorPageContent = { { label: "SERP API", href: "/google-search" }, { label: "Google Maps API", href: "/google-maps" }, { label: "Reddit API", href: "/reddit" }, + { label: "Instagram API", href: "/instagram" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, { label: "Read the docs", href: "/docs" }, ], diff --git a/surfsense_web/lib/connectors-marketing/youtube.tsx b/surfsense_web/lib/connectors-marketing/youtube.tsx index 1850cbe86..f5f3215ef 100644 --- a/surfsense_web/lib/connectors-marketing/youtube.tsx +++ b/surfsense_web/lib/connectors-marketing/youtube.tsx @@ -270,6 +270,7 @@ export const youtube: ConnectorPageContent = { related: [ { label: "Reddit API", href: "/reddit" }, + { label: "Instagram API", href: "/instagram" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, From f5aad487ac549e047246cd8fa72396af4f3fb3b4 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:07:42 +0530 Subject: [PATCH 26/72] docs(web): document Instagram native connector --- .../content/docs/connectors/index.mdx | 2 +- .../content/docs/connectors/native/index.mdx | 7 +- .../docs/connectors/native/instagram.mdx | 77 +++++++++++++++++++ .../content/docs/connectors/native/meta.json | 2 +- 4 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 surfsense_web/content/docs/connectors/native/instagram.mdx diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx index 1ee9209e6..747d278e2 100644 --- a/surfsense_web/content/docs/connectors/index.mdx +++ b/surfsense_web/content/docs/connectors/index.mdx @@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li } title="Native Connectors" - description="SurfSense's built-in scraper APIs: Reddit, YouTube, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" + description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" href="/docs/connectors/native" /> + Date: Thu, 9 Jul 2026 16:07:50 +0530 Subject: [PATCH 27/72] docs(web): add Instagram to MCP server tool docs --- surfsense_web/app/(home)/mcp-server/page.tsx | 14 ++++++++++---- surfsense_web/content/docs/how-to/mcp-server.mdx | 8 ++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index 5e4961b1e..a5557e11e 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types"; const canonicalUrl = "https://www.surfsense.com/mcp-server"; const metaDescription = - "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; + "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; export const metadata: Metadata = { title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools", @@ -93,6 +93,9 @@ const TOOL_GROUPS = [ "surfsense_reddit_scrape", "surfsense_youtube_scrape", "surfsense_youtube_comments", + "surfsense_instagram_scrape", + "surfsense_instagram_comments", + "surfsense_instagram_details", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_google_search", @@ -127,7 +130,7 @@ const FAQ: FaqItem[] = [ { question: "What is the SurfSense MCP server?", answer: - "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", + "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", }, { question: "Which MCP clients does it work with?", @@ -215,8 +218,8 @@ export default function McpServerPage() {

The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform: - scrape Reddit, YouTube, Google Maps, Google Search, and the open web, and search, - read, and write your knowledge base. One API key, typed tools, pay as you go. + scrape Reddit, YouTube, Instagram, Google Maps, Google Search, and the open web, and + search, read, and write your knowledge base. One API key, typed tools, pay as you go.

+ diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index f5f419219..632dc5a4d 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; # SurfSense MCP Server -The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 18 native, typed tools: every scraper (Reddit, YouTube, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. +The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 21 native, typed tools: every scraper (Reddit, YouTube, Instagram, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted. @@ -133,7 +133,7 @@ Add to `~/.cursor/mcp.json` (global — keeps the key out of your repo) or a pro } ``` -Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 18 tools should appear with a green dot. +Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 21 tools should appear with a green dot. @@ -257,14 +257,14 @@ For self-host (stdio), all settings are environment variables passed by the clie - **401 errors** — the API key is wrong or expired; create a new one. - **403 errors** — API access is disabled for the workspace; toggle **API key access** on under **API Playground → API Keys**. - **"Could not reach SurfSense"** — the backend isn't running or `SURFSENSE_BASE_URL` is wrong. -- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 18 tools register without needing a backend. +- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 21 tools register without needing a backend. ## Tools reference | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_comments`, `surfsense_instagram_details`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. From d800ca33a5198715d53ce16c8040d35d392c1280 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 18:29:17 +0530 Subject: [PATCH 28/72] feat(instagram): handle soft login wall in fetch logic Added detection for Instagram's soft login wall, which returns a 200 status with login HTML. Implemented a new function to identify login redirects and adjusted the fetch logic to treat these cases similarly to 401/403 responses. --- .../proprietary/platforms/instagram/fetch.py | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py index 82b0bcc14..e734a0dc4 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -101,6 +101,13 @@ _WARM_URL = "https://www.instagram.com/" _BASE = "https://www.instagram.com" _CSRF_COOKIE = "csrftoken" +# Soft login wall: instead of a 401/403, IG answers an api/v1/* request with a +# 302 to /accounts/login/ that the impersonated client follows to a 200 login +# page. The status is 200 but the body is login HTML, so this evades the +# status-code rotate check — detect it via the response's final URL and treat +# it exactly like a 403. +_LOGIN_PATH = "/accounts/login" + def now_iso() -> str: """UTC timestamp in the millisecond ISO shape used by scraper output.""" @@ -136,6 +143,17 @@ def _parse_json(page: Any) -> Any | None: return None +def _is_login_redirect(page: Any) -> bool: + """True if IG redirected this request to the anonymous login wall. + + A soft block: the final URL lands on ``/accounts/login/`` (served 200), so + the status check never fires. Best-effort — returns ``False`` when the + response exposes no URL. + """ + final = getattr(page, "url", None) + return isinstance(final, str) and _LOGIN_PATH in final + + def _build_url(path: str, params: dict[str, Any] | None) -> str: """Absolute URL for an instagram.com path (accepts already-absolute URLs).""" base = path if path.startswith("http") else f"{_BASE}/{path.strip('/')}/" @@ -320,8 +338,11 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N await holder.pace() page = await _get_page(session, url) status = page.status + # A 302 -> /accounts/login/ soft block presents as 200 login HTML; + # fold it into the same rotate path as a 401/403. + blocked = status in _ROTATE_STATUSES or _is_login_redirect(page) - if status == 200: + if status == 200 and not blocked: return _parse_json(page) if status == 404: return None @@ -333,13 +354,14 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N ) await asyncio.sleep(delay + random.uniform(0, 1)) continue - if status in _ROTATE_STATUSES and attempt < _MAX_ROTATIONS: + if blocked and attempt < _MAX_ROTATIONS: attempt += 1 await holder.rotate() continue - if status in _ROTATE_STATUSES: + if blocked: raise InstagramAccessBlockedError( - f"Instagram refused {path} on {attempt} rotated IPs ({status})" + f"Instagram refused {path} on {attempt} rotated IPs " + f"(status={status}, login_wall={_is_login_redirect(page)})" ) logger.warning("[instagram] GET %s returned %s", path, status) return None From 6384409b1d97bc3013d5726dab7156093ea4082f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 18:29:29 +0530 Subject: [PATCH 29/72] test(instagram): enhance fetch resilience tests for login redirects Added tests to handle scenarios where a 200 status response is redirected to a login page, ensuring proper rotation of IPs and raising errors when persistent login redirects occur. Updated the _FakeSession class to support login wall detection. --- .../instagram/test_fetch_resilience.py | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py index eefae2fd9..037268db2 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -24,9 +24,12 @@ _PAYLOAD = {"data": {"user": {"username": "natgeo"}}} class _FakePage: - def __init__(self, status: int, *, cookies: dict | None = None, payload=None): + def __init__( + self, status: int, *, cookies: dict | None = None, payload=None, url=None + ): self.status = status self.cookies = cookies or {} + self.url = url self._payload = payload if payload is not None else _PAYLOAD def json(self): @@ -40,10 +43,18 @@ class _FakePage: class _FakeSession: """One 'IP': the warm-up GET mints csrftoken per flag; endpoint GETs return ``status``.""" - def __init__(self, status: int = 200, *, csrftoken: bool = True, payload=None) -> None: + def __init__( + self, + status: int = 200, + *, + csrftoken: bool = True, + payload=None, + login_wall: bool = False, + ) -> None: self.status = status self.csrftoken = csrftoken self.payload = payload + self.login_wall = login_wall self.json_calls = 0 self.warm_calls = 0 @@ -53,7 +64,9 @@ class _FakeSession: ck = {"csrftoken": "x", "mid": "y"} if self.csrftoken else {} return _FakePage(200, cookies=ck) self.json_calls += 1 - return _FakePage(self.status, payload=self.payload) + # A soft login wall: 200, but the final URL is the login page. + final = "https://www.instagram.com/accounts/login/" if self.login_wall else url + return _FakePage(self.status, payload=self.payload, url=final) class _FakeHolder: @@ -150,6 +163,41 @@ async def test_rotates_on_401_login_wall(): assert holder.rotations == 1 +async def test_rotates_on_login_redirect_then_succeeds(): + # 200 status but redirected to /accounts/login/: a soft block that must + # rotate to a fresh IP, not be mistaken for an empty result. + holder = _FakeHolder( + [_FakeSession(200, login_wall=True), _FakeSession(200)] + ) + token = _current_session.set(holder) + try: + result = await fetch_json("api/v1/tags/web_info/", {"tag_name": "travel"}) + finally: + _current_session.reset(token) + assert result == _PAYLOAD + assert holder.rotations == 1 + + +async def test_persistent_login_redirect_raises_blocked(): + holder = _FakeHolder( + [ + _FakeSession(200, login_wall=True) + for _ in range(fetch._MAX_ROTATIONS + 1) + ] + ) + token = _current_session.set(holder) + try: + raised = False + try: + await fetch_json("api/v1/tags/web_info/", {"tag_name": "travel"}) + except InstagramAccessBlockedError: + raised = True + finally: + _current_session.reset(token) + assert raised + assert holder.rotations == fetch._MAX_ROTATIONS + + async def test_404_returns_none_without_rotating(): holder = _FakeHolder([_FakeSession(404), _FakeSession(200)]) token = _current_session.set(holder) From 5551ebbda74ad1d8ae7f712b03a8448745536524 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:08:00 +0530 Subject: [PATCH 30/72] chore(instagram): modify instagram svg --- surfsense_web/public/connectors/instagram.svg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/surfsense_web/public/connectors/instagram.svg b/surfsense_web/public/connectors/instagram.svg index 81233a38e..edba1b251 100644 --- a/surfsense_web/public/connectors/instagram.svg +++ b/surfsense_web/public/connectors/instagram.svg @@ -1 +1,2 @@ - + + \ No newline at end of file From e8f8eeab270b65c5b51fadc548df6065f8a96942 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:08:18 +0530 Subject: [PATCH 31/72] fix(instagram): improve error handling for Instagram access blocks in scraper Updated the fan_out function to handle InstagramAccessBlockedError more gracefully. Instead of raising the error directly, it now puts the error into the results queue to prevent deadlocks. This change ensures that the consumer can handle access block scenarios without interrupting the processing of other jobs. --- .../platforms/instagram/scraper.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index 46569f4dd..34f6658a3 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -132,7 +132,13 @@ async def fan_out( job_queue: asyncio.Queue[AsyncIterator[dict[str, Any]]] = asyncio.Queue() for job in jobs: job_queue.put_nowait(job) - results: asyncio.Queue[list[dict[str, Any]]] = asyncio.Queue() + # A batch of items on success, or a hard-block exception to re-raise on the + # consumer side. The consumer reads exactly one entry per job, so a worker + # MUST put something for every job it pulls — raising instead would strand + # the error on a dead task and deadlock the consumer on ``results.get()``. + results: asyncio.Queue[list[dict[str, Any]] | InstagramAccessBlockedError] = ( + asyncio.Queue() + ) async def worker() -> None: holder = None @@ -153,8 +159,11 @@ async def fan_out( items = [item async for item in job] else: items = [item async for item in job] - except InstagramAccessBlockedError: - raise # a hard login wall must abort the batch, not be swallowed + except InstagramAccessBlockedError as e: + # A hard login wall aborts the batch. Hand it to the consumer + # via the queue (not ``raise``) so it never deadlocks waiting. + await results.put(e) + return except Exception as e: # one bad target must not kill the run logger.warning("[instagram] fan-out job failed: %s", e) await results.put(items) @@ -165,7 +174,10 @@ async def fan_out( tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))] try: for _ in range(len(jobs)): - for item in await results.get(): + batch = await results.get() + if isinstance(batch, InstagramAccessBlockedError): + raise batch + for item in batch: yield item finally: for task in tasks: From 4af0b9dbbd99b6982ed364ed2eeef6d28e2df7ba Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:09:06 +0530 Subject: [PATCH 32/72] test(instagram): add regression test for InstagramAccessBlockedError handling Introduced a new test to ensure that InstagramAccessBlockedError is properly propagated without causing deadlocks in the scraper's fan_out function. This regression test verifies that the error surfaces correctly when a blocked job is encountered, enhancing the resilience of the fetch process. --- .../instagram/test_fetch_resilience.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py index 037268db2..0ded704f2 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -295,6 +295,34 @@ async def test_fan_out_empty_jobs_is_noop(): assert out == [] +async def test_fan_out_propagates_blocked_without_deadlock(monkeypatch): + # Regression: a worker that raises InstagramAccessBlockedError used to strand + # the exception on its task and deadlock the consumer on results.get(). It + # must surface as InstagramAccessBlockedError, not hang. + async def _fake_open(): + return _TrackingHolder() + + @asynccontextmanager + async def _fake_bind(_holder): + yield _holder + + monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open) + monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind) + + async def _blocked_job() -> AsyncIterator[dict]: + raise InstagramAccessBlockedError("login wall") + yield {} # unreachable; makes this an async generator + + raised = False + try: + async with asyncio.timeout(5): # fail fast if the deadlock regresses + async for _ in scraper.fan_out([_blocked_job()], concurrency=1): + pass + except InstagramAccessBlockedError: + raised = True + assert raised, "hard block must propagate, not deadlock" + + def _profile_payload(username: str, n: int) -> dict: # IDs namespaced per target so cross-target de-dup doesn't collapse them. return { From be26f00d20a3f18e62ca7f36ccaf8c14ca753631 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:56:29 +0530 Subject: [PATCH 33/72] refactor(instagram): enhance fetch logic for login wall detection Updated the fetch_json function to raise InstagramAccessBlockedError immediately upon detecting a login redirect (302 -> /accounts/login/). This change prevents unnecessary IP rotations when encountering endpoint-level access blocks, improving the efficiency of the scraper's handling of Instagram's login wall. --- .../proprietary/platforms/instagram/fetch.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py index e734a0dc4..3e9955943 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -338,11 +338,19 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N await holder.pace() page = await _get_page(session, url) status = page.status - # A 302 -> /accounts/login/ soft block presents as 200 login HTML; - # fold it into the same rotate path as a 401/403. - blocked = status in _ROTATE_STATUSES or _is_login_redirect(page) - if status == 200 and not blocked: + # Endpoint-level login wall (302 -> /accounts/login/, served as 200 + # login HTML): fail fast, do NOT rotate. Unlike the per-IP 401/403 + # below — which recovers on a fresh exit IP, so it still rotates — + # every rotated IP hits this same wall (observed live), so rotating + # only burns the pool and re-warms for an unwinnable block. Raising + # (vs returning None) keeps a blocked target distinguishable from an + # empty one; fan_out swallows it per-target for partial results. + if _is_login_redirect(page): + raise InstagramAccessBlockedError( + f"Instagram login wall on {path} (endpoint requires auth)" + ) + if status == 200: return _parse_json(page) if status == 404: return None @@ -354,14 +362,13 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N ) await asyncio.sleep(delay + random.uniform(0, 1)) continue - if blocked and attempt < _MAX_ROTATIONS: + if status in _ROTATE_STATUSES and attempt < _MAX_ROTATIONS: attempt += 1 await holder.rotate() continue - if blocked: + if status in _ROTATE_STATUSES: raise InstagramAccessBlockedError( - f"Instagram refused {path} on {attempt} rotated IPs " - f"(status={status}, login_wall={_is_login_redirect(page)})" + f"Instagram refused {path} on {attempt} rotated IPs ({status})" ) logger.warning("[instagram] GET %s returned %s", path, status) return None From a0b388a5c5cdda81b594d0b2caadb0daa26398dc Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:56:41 +0530 Subject: [PATCH 34/72] refactor(instagram): enhance batch processing for access block handling Updated the fan_out function to allow partial results when encountering blocked targets. Instead of aborting the entire batch on a hard login wall, the function now tracks blocked statuses and raises InstagramAccessBlockedError only if all targets are blocked. This change improves the scraper's resilience and efficiency in handling Instagram's access restrictions. --- .../platforms/instagram/scraper.py | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index 34f6658a3..7d25e8c83 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -124,23 +124,24 @@ async def fan_out( Each worker opens ONE proxy session and reuses it across the sequential jobs it pulls, so only the first job per worker pays the proxy handshake + the - cookie warm-up. A bad job yields nothing rather than aborting the batch; - workers are cancelled and their sessions closed if the consumer stops early. + cookie warm-up. Partial results (matches the reddit sibling): one blocked or + failed target yields nothing rather than aborting the batch — Instagram is + an aggregation, not an atomic transaction, so 4/5 good targets beat 0/5. But + if EVERY target was refused (zero items AND a hard block seen), the whole run + couldn't reach anonymous data, so we surface ``InstagramAccessBlockedError`` + (-> 403) instead of a misleading empty success. Workers are cancelled and + their sessions closed if the consumer stops early. """ if not jobs: return job_queue: asyncio.Queue[AsyncIterator[dict[str, Any]]] = asyncio.Queue() for job in jobs: job_queue.put_nowait(job) - # A batch of items on success, or a hard-block exception to re-raise on the - # consumer side. The consumer reads exactly one entry per job, so a worker - # MUST put something for every job it pulls — raising instead would strand - # the error on a dead task and deadlock the consumer on ``results.get()``. - results: asyncio.Queue[list[dict[str, Any]] | InstagramAccessBlockedError] = ( - asyncio.Queue() - ) + results: asyncio.Queue[list[dict[str, Any]]] = asyncio.Queue() + blocked = False # set if any target hit a hard login/auth wall async def worker() -> None: + nonlocal blocked holder = None try: holder = await open_proxy_holder() @@ -160,10 +161,10 @@ async def fan_out( else: items = [item async for item in job] except InstagramAccessBlockedError as e: - # A hard login wall aborts the batch. Hand it to the consumer - # via the queue (not ``raise``) so it never deadlocks waiting. - await results.put(e) - return + # Partial results: a blocked target must not kill the batch. + # Record it so a fully-blocked run can still surface the 403. + blocked = True + logger.warning("[instagram] target blocked: %s", e) except Exception as e: # one bad target must not kill the run logger.warning("[instagram] fan-out job failed: %s", e) await results.put(items) @@ -172,18 +173,24 @@ async def fan_out( await holder.close() tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))] + emitted = 0 try: for _ in range(len(jobs)): - batch = await results.get() - if isinstance(batch, InstagramAccessBlockedError): - raise batch - for item in batch: + for item in await results.get(): + emitted += 1 yield item finally: for task in tasks: if not task.done(): task.cancel() await asyncio.gather(*tasks, return_exceptions=True) + # Reached only on natural exhaustion (an early-stop close raises GeneratorExit + # inside the loop and skips this). Nothing came back AND a wall was hit -> + # the run was fully refused, so fail loud rather than return empty. + if emitted == 0 and blocked: + raise InstagramAccessBlockedError( + "Instagram refused anonymous access to every target" + ) def _emit(partial: dict[str, Any], *, input_url: str | None) -> dict[str, Any]: From 0309e1d28ac8a71382113ddfe0a9c5c65122475a Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:56:55 +0530 Subject: [PATCH 35/72] refactor(instagram): streamline login redirect handling in tests Revised tests for handling login redirects to improve clarity and efficiency. The test for immediate failure on endpoint-level walls was updated to prevent unnecessary IP rotations. Additionally, new tests were added to ensure that partial results are returned when encountering blocked targets, enhancing the resilience of the scraper's batch processing. --- .../instagram/test_fetch_resilience.py | 69 ++++++++++--------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py index 0ded704f2..29d92fad8 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -163,28 +163,11 @@ async def test_rotates_on_401_login_wall(): assert holder.rotations == 1 -async def test_rotates_on_login_redirect_then_succeeds(): - # 200 status but redirected to /accounts/login/: a soft block that must - # rotate to a fresh IP, not be mistaken for an empty result. - holder = _FakeHolder( - [_FakeSession(200, login_wall=True), _FakeSession(200)] - ) - token = _current_session.set(holder) - try: - result = await fetch_json("api/v1/tags/web_info/", {"tag_name": "travel"}) - finally: - _current_session.reset(token) - assert result == _PAYLOAD - assert holder.rotations == 1 - - -async def test_persistent_login_redirect_raises_blocked(): - holder = _FakeHolder( - [ - _FakeSession(200, login_wall=True) - for _ in range(fetch._MAX_ROTATIONS + 1) - ] - ) +async def test_login_redirect_fails_fast_without_rotating(): + # A 302 -> /accounts/login/ (served 200) is an endpoint-level wall: rotating + # never clears it, so we raise immediately instead of burning IP rotations. + # A second healthy session is present to prove we do NOT fall through to it. + holder = _FakeHolder([_FakeSession(200, login_wall=True), _FakeSession(200)]) token = _current_session.set(holder) try: raised = False @@ -195,7 +178,7 @@ async def test_persistent_login_redirect_raises_blocked(): finally: _current_session.reset(token) assert raised - assert holder.rotations == fetch._MAX_ROTATIONS + assert holder.rotations == 0 # fail fast: no rotation burned async def test_404_returns_none_without_rotating(): @@ -295,10 +278,7 @@ async def test_fan_out_empty_jobs_is_noop(): assert out == [] -async def test_fan_out_propagates_blocked_without_deadlock(monkeypatch): - # Regression: a worker that raises InstagramAccessBlockedError used to strand - # the exception on its task and deadlock the consumer on results.get(). It - # must surface as InstagramAccessBlockedError, not hang. +async def _install_fake_holders(monkeypatch) -> None: async def _fake_open(): return _TrackingHolder() @@ -309,9 +289,17 @@ async def test_fan_out_propagates_blocked_without_deadlock(monkeypatch): monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open) monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind) - async def _blocked_job() -> AsyncIterator[dict]: - raise InstagramAccessBlockedError("login wall") - yield {} # unreachable; makes this an async generator + +async def _blocked_job() -> AsyncIterator[dict]: + raise InstagramAccessBlockedError("login wall") + yield {} # unreachable; makes this an async generator + + +async def test_fan_out_all_blocked_raises_without_deadlock(monkeypatch): + # Regression: a blocked worker used to strand its exception on a dead task + # and deadlock the consumer on results.get(). When EVERY target is blocked + # (zero items), it must surface InstagramAccessBlockedError, not hang. + await _install_fake_holders(monkeypatch) raised = False try: @@ -320,7 +308,26 @@ async def test_fan_out_propagates_blocked_without_deadlock(monkeypatch): pass except InstagramAccessBlockedError: raised = True - assert raised, "hard block must propagate, not deadlock" + assert raised, "fully-blocked run must surface the 403, not deadlock" + + +async def test_fan_out_partial_results_survive_one_blocked_target(monkeypatch): + # Q2: one blocked target must NOT abort the batch — the good target's items + # come through and no exception is raised (aggregation, not a transaction). + await _install_fake_holders(monkeypatch) + + async def _good_job() -> AsyncIterator[dict]: + for i in range(3): + yield {"id": f"good:{i}"} + + async with asyncio.timeout(5): + items = [ + item + async for item in scraper.fan_out( + [_blocked_job(), _good_job()], concurrency=2 + ) + ] + assert [it["id"] for it in items] == ["good:0", "good:1", "good:2"] def _profile_payload(username: str, n: int) -> dict: From f66ab730445629a6f5cf3ce558db1448e7691fd6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:22:46 +0530 Subject: [PATCH 36/72] refactor(instagram): improve handling of login-gated endpoints in fetch logic Enhanced the fetch_json function to immediately raise InstagramAccessBlockedError for login-gated endpoints, preventing unnecessary IP rotations. Introduced a new constant for authentication-walled paths to streamline the detection of access blocks, improving the scraper's efficiency in handling Instagram's restrictions. --- .../proprietary/platforms/instagram/fetch.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py index 3e9955943..73320eddb 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -75,6 +75,12 @@ _MAX_ROTATIONS = 3 _MAX_BACKOFFS = 4 _BACKOFF_BASE_S = 5.0 +# Endpoints Instagram serves only to logged-in clients (confirmed live). A bare +# 401/403 here is an endpoint auth wall, not a per-IP block, so every rotated IP +# hits the same wall — fail fast instead of burning the pool, exactly like the +# /accounts/login/ redirect branch. Content endpoints (profiles) still rotate. +_AUTH_WALLED_PATHS = ("web/search/topsearch/", "api/v1/tags/web_info/") + # Instagram 429s hard on bursts. Pace each sticky session so a fast IP can't # burst past the per-IP threshold. ponytail: 1.5s is tuned to residential exits; # a pool with a stricter per-IP cap may need it raised — watch for 429 log spam. @@ -362,11 +368,18 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N ) await asyncio.sleep(delay + random.uniform(0, 1)) continue - if status in _ROTATE_STATUSES and attempt < _MAX_ROTATIONS: - attempt += 1 - await holder.rotate() - continue if status in _ROTATE_STATUSES: + # Bare 401/403 on a login-gated endpoint: rotating never clears an + # endpoint auth wall, so fail fast (mirrors the login-redirect + # branch above). Other endpoints rotate — a per-IP 401 recovers. + if any(p in path for p in _AUTH_WALLED_PATHS): + raise InstagramAccessBlockedError( + f"Instagram login wall on {path} (endpoint requires auth)" + ) + if attempt < _MAX_ROTATIONS: + attempt += 1 + await holder.rotate() + continue raise InstagramAccessBlockedError( f"Instagram refused {path} on {attempt} rotated IPs ({status})" ) From 82f1d0b4e5b7ac3f403ecdba22e9d8764f2eddd1 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:23:09 +0530 Subject: [PATCH 37/72] refactor(instagram): improve anonymous scraping logic and error messaging Enhanced the Instagram scraper to clarify the requirements for accessing user profiles and hashtags. Updated the error message for blocked access to provide detailed guidance on necessary credentials. Introduced a regex for validating Instagram usernames and refined the discovery function to handle profile queries directly, improving user experience and error handling in anonymous mode. --- .../capabilities/instagram/scrape/executor.py | 5 +- .../platforms/instagram/scraper.py | 64 +++++++------------ 2 files changed, 27 insertions(+), 42 deletions(-) diff --git a/surfsense_backend/app/capabilities/instagram/scrape/executor.py b/surfsense_backend/app/capabilities/instagram/scrape/executor.py index a76722dcd..c65e2ccb7 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/executor.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/executor.py @@ -43,7 +43,10 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: except InstagramAccessBlockedError as exc: # Anonymous-only scraper; a hard block can't be retried with creds. raise ForbiddenError( - f"Instagram refused anonymous access: {exc}", + "Instagram requires a login for this request and SurfSense scrapes " + "anonymously. Provide a profile URL or handle via directUrls; " + "keyword/hashtag search needs an account and is unavailable. " + f"Details: {exc}", code="INSTAGRAM_ACCESS_BLOCKED", ) from exc emit_progress( diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index 7d25e8c83..f748d5026 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -26,6 +26,7 @@ from __future__ import annotations import asyncio import logging +import re from collections.abc import AsyncIterator from contextlib import aclosing from datetime import UTC, datetime, timedelta @@ -70,6 +71,10 @@ _HASHTAG_PATH = "api/v1/tags/web_info/" _LOCATION_PATH = "api/v1/locations/web_info/" _SEARCH_PATH = "web/search/topsearch/" +# Instagram usernames: 1-30 chars of letters/digits/period/underscore. Used to +# treat a profile/user discovery query as a direct (anonymous) handle lookup. +_HANDLE_RE = re.compile(r"[A-Za-z0-9._]{1,30}\Z") + def _parse_newer_than(value: str | None) -> datetime | None: """Parse ``onlyPostsNewerThan`` (ISO, YYYY-MM-DD, or relative) to UTC. @@ -356,48 +361,25 @@ async def _details_flow( async def _discover( query: str, *, search_type: str, limit: int ) -> list[ResolvedUrl]: - """Resolve a discovery query into target URLs via topsearch.""" - data = await fetch_json(_SEARCH_PATH, {"query": query, "context": "blended"}) - if not isinstance(data, dict): - return [] - out: list[ResolvedUrl] = [] + """Resolve a discovery query into targets - anonymously. + + Instagram's keyword search (``topsearch``) is login-walled, so we never call + it. A profile/user query is resolved as a direct handle lookup against the + anonymous profile endpoint ("messi" -> instagram.com/messi/). Hashtag/place + keyword discovery has NO anonymous endpoint (topsearch and the tag/location + feeds all require a session), so we surface a clear block instead of a + misleading empty success. + """ if search_type in ("profile", "user"): - for entry in data.get("users", []): - user = entry.get("user", {}) if isinstance(entry, dict) else {} - name = user.get("username") - if not name: - continue - out.append( - ResolvedUrl("profile", name, f"https://www.instagram.com/{name}/") - ) - elif search_type == "hashtag": - for entry in data.get("hashtags", []): - tag = entry.get("hashtag", {}) if isinstance(entry, dict) else {} - name = tag.get("name") - if not name: - continue - out.append( - ResolvedUrl( - "hashtag", - name, - f"https://www.instagram.com/explore/tags/{name}/", - ) - ) - elif search_type == "place": - for entry in data.get("places", []): - place = entry.get("place", {}) if isinstance(entry, dict) else {} - loc = place.get("location", {}) if isinstance(place, dict) else {} - pk = loc.get("pk") or loc.get("id") - if not pk: - continue - out.append( - ResolvedUrl( - "place", - str(pk), - f"https://www.instagram.com/explore/locations/{pk}/", - ) - ) - return out[:limit] + handle = query.strip().lstrip("@") + if _HANDLE_RE.match(handle): + url = f"https://www.instagram.com/{handle}/" + return [ResolvedUrl("profile", handle, url)][:limit] + return [] # not a handle, and no anonymous fuzzy search -> nothing to do + raise InstagramAccessBlockedError( + f"Instagram {search_type} search requires login and is unavailable in " + "anonymous mode - pass a profile URL or handle via directUrls instead" + ) def _resolve_inputs(input_model: InstagramScrapeInput) -> list[ResolvedUrl]: From 2c251f9e0780464b783b083a574faa235f82fdfc Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:26:18 +0530 Subject: [PATCH 38/72] test(instagram): add tests for anonymous profile and hashtag discovery Introduced new tests to validate the behavior of the Instagram scraper when discovering anonymous profiles and handling hashtag searches. The tests ensure that profile lookups succeed without requiring authentication, while hashtag searches correctly raise an error when access is blocked, enhancing the robustness of the scraper's functionality. --- .../instagram/test_fetch_resilience.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py index 29d92fad8..e142a33f7 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -389,3 +389,22 @@ async def test_scrape_instagram_closes_sessions_when_limit_stops_inflight_worker assert len(items) == 3 assert holders, "workers should have opened sessions" assert all(h.closed for h in holders), "early stop leaked a proxy session" + + +async def test_discover_profile_is_anonymous_handle_lookup(): + # keyword search (topsearch) is login-walled, so a profile/user query resolves + # as a DIRECT handle lookup against the anonymous profile endpoint — no network + # here, just the URL resolution, so no session/monkeypatch needed. + targets = await scraper._discover("messi", search_type="user", limit=10) + assert [(t.kind, t.value) for t in targets] == [("profile", "messi")] + + +async def test_discover_hashtag_search_blocks_anonymously(): + # hashtag/place keyword discovery has no anonymous endpoint at all, so it must + # fail loud (clear message) rather than return a misleading empty success. + raised = False + try: + await scraper._discover("travel", search_type="hashtag", limit=10) + except InstagramAccessBlockedError: + raised = True + assert raised From 2a6715e2a362ceda5f0baebd7c5a5d4963f65633 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:23:53 +0530 Subject: [PATCH 39/72] feat(homepage): update hero section description and add Instagram use case Enhanced the hero section by including Instagram in the list of platforms monitored by SurfSense. Additionally, introduced a new use case for "Social sentiment mining" that details the capabilities of pulling public posts and analyzing audience reactions on Instagram. --- surfsense_web/components/homepage/hero-section.tsx | 2 +- surfsense_web/components/homepage/use-cases.tsx | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index 89d1e46f5..ddcddf9cc 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -719,7 +719,7 @@ export function HeroSection() { > SurfSense is an open-source competitive intelligence platform. Your AI agents monitor competitors, track rankings, and listen to your market with live data from platforms - like Reddit, YouTube, Google Maps, Google Search, and the open web. + like Reddit, Instagram, YouTube, Google Maps, Google Search, and the open web.

diff --git a/surfsense_web/components/homepage/use-cases.tsx b/surfsense_web/components/homepage/use-cases.tsx index 44609351d..6163f7481 100644 --- a/surfsense_web/components/homepage/use-cases.tsx +++ b/surfsense_web/components/homepage/use-cases.tsx @@ -28,6 +28,14 @@ const USE_CASES: { anchor: "Reddit API", art: "brand", }, + { + title: "Social sentiment mining", + description: + "Pull public posts, reels, and full comment threads from any creator or competitor, then score how audiences actually react to launches and campaigns.", + href: "/instagram", + anchor: "Instagram API", + art: "chat", + }, { title: "B2B lead generation", description: From abb2ea2d3b45b673ef5facdf6a3f4ebc4a5d1054 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:24:06 +0530 Subject: [PATCH 40/72] feat(auth-utils): add Instagram to public route prefixes for monitoring --- surfsense_web/lib/auth-utils.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts index 979b09605..8c8d919c5 100644 --- a/surfsense_web/lib/auth-utils.ts +++ b/surfsense_web/lib/auth-utils.ts @@ -39,6 +39,7 @@ const PUBLIC_ROUTE_PREFIXES = [ "/mcp-server", "/external-mcp-connectors", "/reddit", + "/instagram", "/youtube", "/google-maps", "/google-search", From 8afa4c6fc6a6caec0f32f3bca6add6c696aa1b1b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:20:39 +0530 Subject: [PATCH 41/72] refactor(instagram): remove unused comments capability --- .../app/capabilities/instagram/__init__.py | 1 - .../instagram/comments/__init__.py | 3 - .../instagram/comments/definition.py | 22 ------- .../instagram/comments/executor.py | 53 ---------------- .../instagram/comments/schemas.py | 61 ------------------- 5 files changed, 140 deletions(-) delete mode 100644 surfsense_backend/app/capabilities/instagram/comments/__init__.py delete mode 100644 surfsense_backend/app/capabilities/instagram/comments/definition.py delete mode 100644 surfsense_backend/app/capabilities/instagram/comments/executor.py delete mode 100644 surfsense_backend/app/capabilities/instagram/comments/schemas.py diff --git a/surfsense_backend/app/capabilities/instagram/__init__.py b/surfsense_backend/app/capabilities/instagram/__init__.py index 4004f86c7..17866e4b5 100644 --- a/surfsense_backend/app/capabilities/instagram/__init__.py +++ b/surfsense_backend/app/capabilities/instagram/__init__.py @@ -2,6 +2,5 @@ from __future__ import annotations -from app.capabilities.instagram.comments import definition as _comments # noqa: F401 from app.capabilities.instagram.details import definition as _details # noqa: F401 from app.capabilities.instagram.scrape import definition as _scrape # noqa: F401 diff --git a/surfsense_backend/app/capabilities/instagram/comments/__init__.py b/surfsense_backend/app/capabilities/instagram/comments/__init__.py deleted file mode 100644 index 84899d758..000000000 --- a/surfsense_backend/app/capabilities/instagram/comments/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""``instagram.comments`` verb: post/reel URLs → comments (and replies).""" - -from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/instagram/comments/definition.py b/surfsense_backend/app/capabilities/instagram/comments/definition.py deleted file mode 100644 index 7794228ac..000000000 --- a/surfsense_backend/app/capabilities/instagram/comments/definition.py +++ /dev/null @@ -1,22 +0,0 @@ -"""``instagram.comments`` capability registration (billed per comment; see config -``INSTAGRAM_SCRAPE_MICROS_PER_COMMENT``).""" - -from __future__ import annotations - -from app.capabilities.core import BillingUnit, Capability, register_capability -from app.capabilities.instagram.comments.executor import build_comments_executor -from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput - -INSTAGRAM_COMMENTS = Capability( - name="instagram.comments", - description=( - "Fetch comments (and optionally replies) for Instagram post or reel URLs." - ), - input_schema=CommentsInput, - output_schema=CommentsOutput, - executor=build_comments_executor(), - billing_unit=BillingUnit.INSTAGRAM_COMMENT, - docs_url="/docs/connectors/native/instagram", -) - -register_capability(INSTAGRAM_COMMENTS) diff --git a/surfsense_backend/app/capabilities/instagram/comments/executor.py b/surfsense_backend/app/capabilities/instagram/comments/executor.py deleted file mode 100644 index 260ee06c9..000000000 --- a/surfsense_backend/app/capabilities/instagram/comments/executor.py +++ /dev/null @@ -1,53 +0,0 @@ -"""``instagram.comments`` executor: verb input → scraper → comment items.""" - -from __future__ import annotations - -from collections.abc import Awaitable, Callable - -from app.capabilities.core import Executor -from app.capabilities.core.progress import emit_progress -from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput -from app.exceptions import ForbiddenError -from app.proprietary.platforms.instagram import ( - InstagramAccessBlockedError, - InstagramScrapeInput, - scrape_instagram, -) - -ScrapeFn = Callable[..., Awaitable[list[dict]]] - - -def build_comments_executor(scrape_fn: ScrapeFn | None = None) -> Executor: - """Bind the executor to a scraper fn (defaults to the proprietary actor).""" - scrape_fn = scrape_fn or scrape_instagram - - async def execute(payload: CommentsInput) -> CommentsOutput: - actor_input = InstagramScrapeInput( - resultsType="comments", - directUrls=payload.urls, - resultsLimit=payload.max_comments_per_post, - isNewestComments=payload.newest_first, - includeNestedComments=payload.include_replies, - ) - emit_progress( - "starting", - "Fetching Instagram comments", - total=payload.max_items, - unit="comment", - ) - try: - items = await scrape_fn(actor_input, limit=payload.max_items) - except InstagramAccessBlockedError as exc: - raise ForbiddenError( - f"Instagram refused anonymous access: {exc}", - code="INSTAGRAM_ACCESS_BLOCKED", - ) from exc - emit_progress( - "done", - f"Scraped {len(items)} comment(s)", - current=len(items), - unit="comment", - ) - return CommentsOutput(items=items) - - return execute diff --git a/surfsense_backend/app/capabilities/instagram/comments/schemas.py b/surfsense_backend/app/capabilities/instagram/comments/schemas.py deleted file mode 100644 index fadb83215..000000000 --- a/surfsense_backend/app/capabilities/instagram/comments/schemas.py +++ /dev/null @@ -1,61 +0,0 @@ -"""``instagram.comments`` I/O contracts. - -A lean surface over ``InstagramScrapeInput`` (``resultsType="comments"``). The -scraper's ``InstagramComment`` is reused verbatim as the output element. -""" - -from __future__ import annotations - -from pydantic import BaseModel, Field - -from app.capabilities.instagram.scrape.schemas import ( - MAX_INSTAGRAM_ITEMS, - MAX_INSTAGRAM_SOURCES, -) -from app.proprietary.platforms.instagram import InstagramComment - -MAX_COMMENTS_PER_POST = 50 -"""Anonymous web media pages surface at most ~50 comments per post.""" - - -class CommentsInput(BaseModel): - urls: list[str] = Field( - min_length=1, - max_length=MAX_INSTAGRAM_SOURCES, - description="Post or reel URLs to fetch comments for (shortCode or numeric-ID forms).", - ) - newest_first: bool = Field( - default=False, - description="Return newest comments first.", - ) - include_replies: bool = Field( - default=False, - description="Include nested replies; each reply is a separate billable item.", - ) - max_comments_per_post: int = Field( - default=10, - ge=1, - le=MAX_COMMENTS_PER_POST, - description="Max comments per post (Instagram caps at 50).", - ) - max_items: int = Field( - default=10, - ge=1, - le=MAX_INSTAGRAM_ITEMS, - description="Max total comments to return across all posts.", - ) - - @property - def estimated_units(self) -> int: - return self.max_items - - -class CommentsOutput(BaseModel): - items: list[InstagramComment] = Field( - default_factory=list, - description="One item per comment (or reply), in emission order.", - ) - - @property - def billable_units(self) -> int: - return len(self.items) From de1990f9f6dc0cf2aaae3bfb8baa05fd46af4585 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:20:46 +0530 Subject: [PATCH 42/72] refactor(instagram): simplify scrape and details capability schemas --- .../capabilities/instagram/details/schemas.py | 31 +++++++------------ .../instagram/scrape/definition.py | 4 +-- .../capabilities/instagram/scrape/schemas.py | 14 ++++----- 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/surfsense_backend/app/capabilities/instagram/details/schemas.py b/surfsense_backend/app/capabilities/instagram/details/schemas.py index 31972b095..1372f881f 100644 --- a/surfsense_backend/app/capabilities/instagram/details/schemas.py +++ b/surfsense_backend/app/capabilities/instagram/details/schemas.py @@ -1,13 +1,14 @@ """``instagram.details`` I/O contracts. A lean surface over ``InstagramScrapeInput`` (``resultsType="details"``). Each -output item is a profile / hashtag / place, discriminated by the synthesized -``detailKind`` field (a SurfSense addition; every other field mirrors the actor). +output item is a profile (``detailKind="profile"``, a SurfSense addition; every +other field mirrors the actor). Hashtag/place details are login-walled and +therefore unsupported. """ from __future__ import annotations -from typing import Annotated, Literal +from typing import Literal from pydantic import BaseModel, Field, model_validator @@ -15,16 +16,9 @@ from app.capabilities.instagram.scrape.schemas import ( MAX_INSTAGRAM_ITEMS, MAX_INSTAGRAM_SOURCES, ) -from app.proprietary.platforms.instagram import ( - InstagramHashtag, - InstagramPlace, - InstagramProfile, -) +from app.proprietary.platforms.instagram import InstagramProfile -InstagramDetailItem = Annotated[ - InstagramProfile | InstagramHashtag | InstagramPlace, - Field(discriminator="detailKind"), -] +InstagramDetailItem = InstagramProfile class DetailsInput(BaseModel): @@ -32,18 +26,17 @@ class DetailsInput(BaseModel): default_factory=list, max_length=MAX_INSTAGRAM_SOURCES, description=( - "Profile / hashtag / place URLs (or bare profile IDs). The URL type " - "determines the detail kind. Provide these OR search_queries." + "Profile URLs or bare profile IDs. Provide these OR search_queries." ), ) search_queries: list[str] = Field( default_factory=list, max_length=MAX_INSTAGRAM_SOURCES, - description="Discovery keywords. Provide these OR urls (never both).", + description="Discovery keywords resolved to profiles. Provide these OR urls.", ) - search_type: Literal["hashtag", "profile", "place"] = Field( - default="hashtag", - description="What to discover from search_queries (no 'user' — use instagram.scrape).", + search_type: Literal["profile", "user"] = Field( + default="profile", + description="Discovery kind (profile-only; hashtag/place are login-walled).", ) search_limit: int = Field( default=10, @@ -78,7 +71,7 @@ class DetailsInput(BaseModel): class DetailsOutput(BaseModel): items: list[InstagramDetailItem] = Field( default_factory=list, - description="One item per profile/hashtag/place, keyed by detailKind.", + description="One profile detail item per resolved profile.", ) @property diff --git a/surfsense_backend/app/capabilities/instagram/scrape/definition.py b/surfsense_backend/app/capabilities/instagram/scrape/definition.py index e84ca4938..7b5d9769f 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/definition.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/definition.py @@ -10,8 +10,8 @@ from app.capabilities.instagram.scrape.schemas import ScrapeInput, ScrapeOutput INSTAGRAM_SCRAPE = Capability( name="instagram.scrape", description=( - "Scrape public Instagram posts, reels, or mentions from " - "profile/post/hashtag/place URLs, or discover content via search queries." + "Scrape public Instagram posts, reels, or mentions from profile/post/" + "reel URLs, or discover public profiles via search queries." ), input_schema=ScrapeInput, output_schema=ScrapeOutput, diff --git a/surfsense_backend/app/capabilities/instagram/scrape/schemas.py b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py index 0e5646dc7..03e03e55d 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py @@ -26,8 +26,8 @@ class ScrapeInput(BaseModel): default_factory=list, max_length=MAX_INSTAGRAM_SOURCES, description=( - "Instagram URLs or bare profile IDs: profile, post (/p/), reel " - "(/reel/), hashtag (/explore/tags/), or place (/explore/locations/). " + "Instagram URLs or bare profile IDs: profile, post (/p/), or reel " + "(/reel/). Hashtag/place URLs are unsupported (login-walled). " "Provide these OR search_queries (never both)." ), ) @@ -35,13 +35,13 @@ class ScrapeInput(BaseModel): default_factory=list, max_length=MAX_INSTAGRAM_SOURCES, description=( - "Discovery keywords (hashtags as plaintext, no '#'). Provide these " - "OR urls (never both)." + "Discovery keywords resolved to profiles via Google (IG's keyword " + "search is login-walled). Provide these OR urls (never both)." ), ) - search_type: Literal["hashtag", "profile", "place", "user"] = Field( - default="hashtag", - description="What to discover from search_queries. Only used with search_queries.", + search_type: Literal["profile", "user"] = Field( + default="profile", + description="Discovery kind (profile-only; hashtag/place are login-walled).", ) result_type: Literal["posts", "reels", "mentions"] = Field( default="posts", From 4813dc96e3433cb0e44b71af789e8db205dce5b6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:20:51 +0530 Subject: [PATCH 43/72] refactor(instagram): streamline scraper, fetch, and parsing pipeline --- .../platforms/instagram/__init__.py | 6 - .../proprietary/platforms/instagram/fetch.py | 69 ++++-- .../platforms/instagram/parsers.py | 210 ++++++++++++----- .../platforms/instagram/schemas.py | 63 +---- .../platforms/instagram/scraper.py | 221 +++++++----------- .../platforms/instagram/url_resolver.py | 25 +- 6 files changed, 304 insertions(+), 290 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/__init__.py b/surfsense_backend/app/proprietary/platforms/instagram/__init__.py index e3a2a122a..2302ca9fc 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/__init__.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/__init__.py @@ -2,10 +2,7 @@ from .fetch import InstagramAccessBlockedError from .schemas import ( - InstagramComment, - InstagramHashtag, InstagramMediaItem, - InstagramPlace, InstagramProfile, InstagramScrapeInput, ) @@ -13,10 +10,7 @@ from .scraper import iter_instagram, scrape_instagram __all__ = [ "InstagramAccessBlockedError", - "InstagramComment", - "InstagramHashtag", "InstagramMediaItem", - "InstagramPlace", "InstagramProfile", "InstagramScrapeInput", "iter_instagram", diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py index 73320eddb..e2b7c5818 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -32,6 +32,7 @@ import json import logging import random import time +from collections.abc import Callable from contextlib import asynccontextmanager, suppress from contextvars import ContextVar from datetime import UTC, datetime @@ -75,12 +76,6 @@ _MAX_ROTATIONS = 3 _MAX_BACKOFFS = 4 _BACKOFF_BASE_S = 5.0 -# Endpoints Instagram serves only to logged-in clients (confirmed live). A bare -# 401/403 here is an endpoint auth wall, not a per-IP block, so every rotated IP -# hits the same wall — fail fast instead of burning the pool, exactly like the -# /accounts/login/ redirect branch. Content endpoints (profiles) still rotate. -_AUTH_WALLED_PATHS = ("web/search/topsearch/", "api/v1/tags/web_info/") - # Instagram 429s hard on bursts. Pace each sticky session so a fast IP can't # burst past the per-IP threshold. ponytail: 1.5s is tuned to residential exits; # a pool with a stricter per-IP cap may need it raised — watch for 429 log spam. @@ -149,6 +144,20 @@ def _parse_json(page: Any) -> Any | None: return None +def _page_text(page: Any) -> str | None: + """Best-effort HTML/text body of a scrapling response, or ``None``.""" + for attr in ("text", "body", "content"): + val = getattr(page, attr, None) + if callable(val): + with suppress(Exception): + val = val() + if isinstance(val, bytes): + val = val.decode("utf-8", "replace") + if isinstance(val, str) and val.strip(): + return val + return None + + def _is_login_redirect(page: Any) -> bool: """True if IG redirected this request to the anonymous login wall. @@ -308,20 +317,44 @@ async def resolve_redirect(url: str) -> str | None: async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | None: - """GET an Instagram web endpoint through a warmed HTTP session. + """GET an Instagram web endpoint through a warmed session; parse JSON. Returns parsed JSON (dict or list), or ``None`` on 404 / non-block failure. - Warms cookies once per session; rotates the residential IP and re-warms on - 401/403; backs off on 429. Raises :class:`InstagramAccessBlockedError` only - when every rotated IP refuses anonymous access (the login-wall branch, which - we cannot satisfy). + See :func:`_fetch` for the warm/rotate/backoff resilience contract. + """ + return await _fetch(path, params, _parse_json) + + +async def fetch_html(path: str, params: dict[str, Any] | None = None) -> str | None: + """GET an Instagram web page through a warmed session; return its HTML text. + + Same warm/rotate/backoff resilience as :func:`fetch_json` (a login-wall + redirect still raises :class:`InstagramAccessBlockedError`), but hands back + the raw HTML body for the pages that embed their data in the document + (``/p//`` og-meta / ld+json) instead of a JSON XHR endpoint. + """ + return await _fetch(path, params, _page_text) + + +async def _fetch( + path: str, + params: dict[str, Any] | None, + extract: Callable[[Any], Any | None], +) -> Any | None: + """GET an Instagram web endpoint through a warmed HTTP session. + + Applies ``extract`` to the 200 response (JSON parse or HTML text); returns + ``None`` on 404 / non-block failure. Warms cookies once per session; rotates + the residential IP and re-warms on 401/403; backs off on 429. Raises + :class:`InstagramAccessBlockedError` only when every rotated IP refuses + anonymous access (the login-wall branch, which we cannot satisfy). """ holder = _current_session.get() if holder is None: # No bound session (e.g. a direct call outside fan_out): open a # short-lived warmed session for this one fetch, then tear it down. async with proxy_session(): - return await fetch_json(path, params) + return await _fetch(path, params, extract) url = _build_url(path, params) attempt = 0 @@ -357,7 +390,7 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N f"Instagram login wall on {path} (endpoint requires auth)" ) if status == 200: - return _parse_json(page) + return extract(page) if status == 404: return None if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS: @@ -369,13 +402,9 @@ async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | N await asyncio.sleep(delay + random.uniform(0, 1)) continue if status in _ROTATE_STATUSES: - # Bare 401/403 on a login-gated endpoint: rotating never clears an - # endpoint auth wall, so fail fast (mirrors the login-redirect - # branch above). Other endpoints rotate — a per-IP 401 recovers. - if any(p in path for p in _AUTH_WALLED_PATHS): - raise InstagramAccessBlockedError( - f"Instagram login wall on {path} (endpoint requires auth)" - ) + # Bare 401/403: a per-IP block that a fresh exit IP recovers, so + # rotate and re-warm. (The endpoint-level auth wall is caught by + # the login-redirect branch above and fails fast without rotating.) if attempt < _MAX_ROTATIONS: attempt += 1 await holder.rotate() diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py index 96d66a2c5..55884537d 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -15,6 +15,7 @@ parity is additive. from __future__ import annotations +import json import re from datetime import UTC, datetime from typing import Any @@ -136,28 +137,6 @@ def parse_media(node: dict[str, Any]) -> dict[str, Any]: } -def parse_comment(node: dict[str, Any], *, post_url: str | None) -> dict[str, Any]: - """Map a raw comment node to a flat comment item dict.""" - owner = node.get("owner") if isinstance(node.get("owner"), dict) else {} - code = _shortcode(node) - return { - "id": node.get("id"), - "postUrl": post_url, - "commentUrl": f"{_BASE}/p/{code}/c/{node.get('id')}/" if code else None, - "text": node.get("text"), - "ownerUsername": owner.get("username"), - "ownerProfilePicUrl": owner.get("profile_pic_url"), - "timestamp": _utc_from_sec(node.get("created_at")), - "repliesCount": _edge_count(node, "edge_threaded_comments") - or _int(node.get("child_comment_count")), - "likesCount": _edge_count(node, "edge_liked_by") - or _int(node.get("comment_like_count")), - "owner": {"id": owner.get("id"), "username": owner.get("username")} - if owner - else None, - } - - def parse_profile(user: dict[str, Any]) -> dict[str, Any]: """Map a raw ``web_profile_info`` ``data.user`` to a flat profile item dict.""" username = user.get("username") @@ -185,39 +164,162 @@ def parse_profile(user: dict[str, Any]) -> dict[str, Any]: } -def parse_hashtag(data: dict[str, Any]) -> dict[str, Any]: - """Map a raw ``tags/web_info`` payload to a flat hashtag item dict.""" - node = data.get("data") if isinstance(data.get("data"), dict) else data - name = node.get("name") - top = _edges(node.get("edge_hashtag_to_top_posts")) - recent = _edges(node.get("edge_hashtag_to_media")) - return { - "detailKind": "hashtag", - "id": node.get("id"), - "name": name, - "url": f"{_BASE}/explore/tags/{name}/" if name else None, - "postsCount": _edge_count(node, "edge_hashtag_to_media"), - "topPosts": [parse_media(n) for n in top], - "posts": [parse_media(n) for n in recent], - } +# Anonymous single-post extraction (/p//, /reel//) -------- +# +# Instagram serves logged-out visitors the post's public metadata inside the +# document itself, not via a JSON XHR (the ``?__a=1`` API 404s / login-walls for +# anonymous callers). Two durable, anonymous surfaces carry it: +# 1. ``', + re.IGNORECASE | re.DOTALL, +) +_OG_RE = re.compile( + r' dict[str, Any]: - """Map a raw ``locations/web_info`` payload to a flat place item dict.""" - loc = data.get("location") if isinstance(data.get("location"), dict) else data - recent = _edges(loc.get("edge_location_to_media")) +def _html_int(value: Any) -> int | None: + """Coerce a string/number (``"1,234"``) to int, or ``None``.""" + if isinstance(value, bool): + return None + if isinstance(value, int | float): + return int(value) + if isinstance(value, str): + digits = value.replace(",", "").strip() + if digits.isdigit(): + return int(digits) + return None + + +def _ldjson_blocks(html: str) -> list[dict[str, Any]]: + """Parse every ``application/ld+json`` script block into dicts.""" + out: list[dict[str, Any]] = [] + for raw in _LDJSON_RE.findall(html): + try: + data = json.loads(raw.strip()) + except (ValueError, TypeError): + continue + for node in data if isinstance(data, list) else [data]: + if isinstance(node, dict): + out.append(node) + return out + + +def _og_tags(html: str) -> dict[str, str]: + """Map ``og:`` -> content for the post document.""" + return {k.lower(): v for k, v in _OG_RE.findall(html)} + + +def _ld_interaction(node: dict[str, Any]) -> dict[str, int]: + """Pull like/comment counts out of schema.org ``interactionStatistic``.""" + stats = node.get("interactionStatistic") + items = stats if isinstance(stats, list) else [stats] if stats else [] + out: dict[str, int] = {} + for stat in items: + if not isinstance(stat, dict): + continue + itype = str(stat.get("interactionType") or "") + count = _html_int(stat.get("userInteractionCount")) + if count is None: + continue + if "Like" in itype: + out["likes"] = count + elif "Comment" in itype: + out["comments"] = count + return out + + +def _ld_author_username(node: dict[str, Any]) -> str | None: + """Owner handle from a schema.org ``author`` (alternateName / identifier).""" + author = node.get("author") + author = author[0] if isinstance(author, list) and author else author + if not isinstance(author, dict): + return None + for key in ("alternateName", "identifier", "name"): + val = author.get(key) + if isinstance(val, dict): + val = val.get("value") + if isinstance(val, str) and val.strip(): + return val.strip().lstrip("@") or None + return None + + +def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> dict[str, Any] | None: + """Map an anonymous ``/p//`` (or ``/reel/``) HTML page to a media dict. + + Prefers the embedded schema.org ``ld+json`` block, falling back to Open Graph + meta tags for whatever it omits. Returns a dict shaped like + :func:`parse_media` (so it flows through ``InstagramMediaItem`` unchanged), or + ``None`` when the document carries neither surface (e.g. a login interstitial + slipped past the fetch-layer redirect check — the caller treats ``None`` as + "nothing to emit", never a silent success). + """ + if not isinstance(html, str) or not html.strip(): + return None + blocks = _ldjson_blocks(html) + og = _og_tags(html) + if not blocks and not og: + return None + + node = next( + (b for b in blocks if str(b.get("@type", "")).endswith(("Object", "Post"))), + blocks[0] if blocks else {}, + ) + counts = _ld_interaction(node) + if "likes" not in counts or "comments" not in counts: + m = _OG_COUNTS_RE.search(og.get("description", "")) + if m: + counts.setdefault("likes", _html_int(m.group(1)) or 0) + counts.setdefault("comments", _html_int(m.group(2)) or 0) + + caption = ( + node.get("articleBody") + or node.get("caption") + or node.get("description") + or og.get("description") + ) + caption = caption if isinstance(caption, str) else None + + video = node.get("video") + video = video[0] if isinstance(video, list) and video else video + video_url = ( + video.get("contentUrl") if isinstance(video, dict) else None + ) or og.get("video") + is_video = bool(video_url) or og.get("type") == "video.other" + + image = node.get("image") + image = image[0] if isinstance(image, list) and image else image + display_url = ( + image.get("url") if isinstance(image, dict) else image + if isinstance(image, str) + else None + ) or og.get("image") + return { - "detailKind": "place", - "name": loc.get("name"), - "location_id": str(loc.get("id")) if loc.get("id") is not None else None, - "slug": loc.get("slug"), - "lat": loc.get("lat"), - "lng": loc.get("lng"), - "location_address": loc.get("address_json") or loc.get("address"), - "location_city": loc.get("city"), - "phone": loc.get("phone"), - "website": loc.get("website"), - "category": loc.get("category"), - "media_count": _edge_count(loc, "edge_location_to_media"), - "posts": [parse_media(n) for n in recent], + "id": node.get("identifier") if isinstance(node.get("identifier"), str) else None, + "type": "Video" if is_video else "Image", + "shortCode": shortcode, + "caption": caption, + "hashtags": _HASHTAG_RE.findall(caption) if caption else [], + "mentions": _MENTION_RE.findall(caption) if caption else [], + "url": url, + "commentsCount": counts.get("comments"), + "displayUrl": display_url, + "videoUrl": video_url if is_video else None, + "likesCount": counts.get("likes"), + "timestamp": node.get("uploadDate") or node.get("datePublished"), + "ownerUsername": _ld_author_username(node), } diff --git a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py index 2175744a2..76ad81d12 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py @@ -19,11 +19,8 @@ from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field -InstagramResultsType = Literal[ - "posts", "details", "comments", "reels", "mentions", "stories" -] -InstagramSearchType = Literal["hashtag", "profile", "place", "user"] -InstagramDetailKind = Literal["profile", "hashtag", "place"] +InstagramResultsType = Literal["posts", "details", "reels", "mentions"] +InstagramSearchType = Literal["profile", "user"] class InstagramScrapeInput(BaseModel): @@ -41,12 +38,10 @@ class InstagramScrapeInput(BaseModel): resultsLimit: int | None = Field(default=None, ge=1) onlyPostsNewerThan: str | None = None search: str | None = None - searchType: InstagramSearchType = "hashtag" + searchType: InstagramSearchType = "profile" searchLimit: int | None = Field(default=None, ge=1, le=250) addParentData: bool = False skipPinnedPosts: bool = False - isNewestComments: bool = False - includeNestedComments: bool = False addProfileStatistics: bool = False @@ -109,22 +104,6 @@ class InstagramMediaItem(_ItemBase): dataSource: dict[str, Any] | None = None -class InstagramComment(_ItemBase): - """A comment on a post / reel.""" - - id: str | None = None - postUrl: str | None = None - commentUrl: str | None = None - text: str | None = None - ownerUsername: str | None = None - ownerProfilePicUrl: str | None = None - timestamp: str | None = None - repliesCount: int | None = None - replies: list[dict[str, Any]] = Field(default_factory=list) - likesCount: int | None = None - owner: dict[str, Any] | None = None - - class InstagramProfile(_ItemBase): """A profile detail item (``detailKind = "profile"``).""" @@ -149,39 +128,3 @@ class InstagramProfile(_ItemBase): relatedProfiles: list[dict[str, Any]] = Field(default_factory=list) latestPosts: list[dict[str, Any]] = Field(default_factory=list) statistics: dict[str, Any] | None = None - - -class InstagramHashtag(_ItemBase): - """A hashtag detail item (``detailKind = "hashtag"``).""" - - detailKind: Literal["hashtag"] = "hashtag" - id: str | None = None - name: str | None = None - url: str | None = None - postsCount: int | None = None - topPosts: list[dict[str, Any]] = Field(default_factory=list) - posts: list[dict[str, Any]] = Field(default_factory=list) - related: list[dict[str, Any]] = Field(default_factory=list) - searchTerm: str | None = None - searchSource: str | None = None - - -class InstagramPlace(_ItemBase): - """A place detail item (``detailKind = "place"``).""" - - detailKind: Literal["place"] = "place" - name: str | None = None - location_id: str | None = None - slug: str | None = None - lat: float | None = None - lng: float | None = None - location_address: str | None = None - location_city: str | None = None - location_zip: str | None = None - phone: str | None = None - website: str | None = None - category: str | None = None - media_count: int | None = None - posts: list[dict[str, Any]] = Field(default_factory=list) - searchTerm: str | None = None - searchSource: str | None = None diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index f748d5026..381b40d5b 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -10,11 +10,15 @@ stays sequential. ``fan_out`` is ported from ``../reddit/scraper.py`` but bound to *this* module's proxy holders so every worker warms its own session once and reuses it. +Anonymous-only. Every surface here is reachable without a login: profile web +info, the media embedded in a profile page, single-post/reel pages, and +Google-backed handle discovery. Login-walled surfaces (hashtag/place feeds, +comment threads, IG's native keyword search) are deliberately absent. + Flows are selected by ``resultsType``: -- ``posts`` / ``reels`` / ``mentions`` -> media items (profile / hashtag feeds, - or discovery search) -- ``comments`` -> comment items for post/reel URLs -- ``details`` -> profile / hashtag / place metadata (by URL or discovery search) +- ``posts`` / ``reels`` / ``mentions`` -> media items (profile feed, or a single + ``/p/``/``/reel/`` page, or discovery search) +- ``details`` -> profile metadata (by URL or discovery search) ponytail: deep feed pagination (past the first web page of media) needs the GraphQL cursor endpoint whose doc-id drifts; v1 emits the first page and stops. @@ -32,20 +36,18 @@ from contextlib import aclosing from datetime import UTC, datetime, timedelta from typing import Any +from app.proprietary.platforms.google_search.schemas import GoogleSearchScrapeInput +from app.proprietary.platforms.google_search.scraper import scrape_serps + from .fetch import ( InstagramAccessBlockedError, bind_proxy_holder, + fetch_html, fetch_json, now_iso, open_proxy_holder, ) -from .parsers import ( - parse_comment, - parse_hashtag, - parse_media, - parse_place, - parse_profile, -) +from .parsers import parse_media, parse_post, parse_profile from .schemas import InstagramScrapeInput from .url_resolver import ResolvedUrl, resolve_url @@ -62,14 +64,7 @@ __all__ = [ # residential pool with parallel login walls. _FANOUT_CONCURRENCY = 8 -# Per-post comment fetches fan across their own warm sessions; kept below the -# top-level width so N concurrent targets x this can't explode the IP count. -_COMMENT_CONCURRENCY = 4 - _PROFILE_PATH = "api/v1/users/web_profile_info/" -_HASHTAG_PATH = "api/v1/tags/web_info/" -_LOCATION_PATH = "api/v1/locations/web_info/" -_SEARCH_PATH = "web/search/topsearch/" # Instagram usernames: 1-30 chars of letters/digits/period/underscore. Used to # treat a profile/user discovery query as a direct (anonymous) handle lookup. @@ -235,7 +230,7 @@ async def _media_flow( cutoff: datetime | None, per_target: int, ) -> AsyncIterator[dict[str, Any]]: - """Emit media items for a profile / hashtag / place URL.""" + """Emit media items for a profile feed, or a single ``/p/``/``/reel/`` page.""" from .parsers import _edges result_type = input_model.resultsType @@ -258,128 +253,95 @@ async def _media_flow( if emitted >= per_target: return return - if resolved.kind == "hashtag": - data = await fetch_json(_HASHTAG_PATH, {"tag_name": resolved.value}) - if isinstance(data, dict): - parsed = parse_hashtag(data) - emitted = 0 - for node in [*parsed.get("topPosts", []), *parsed.get("posts", [])]: - if not _media_matches(node, result_type): - continue - if not _is_after(node.get("timestamp"), cutoff): - continue - yield _emit(node, input_url=resolved.url) - emitted += 1 - if emitted >= per_target: - return - return - if resolved.kind == "place": - data = await fetch_json(_LOCATION_PATH, {"location_id": resolved.value}) - if isinstance(data, dict): - parsed = parse_place(data) - emitted = 0 - for node in parsed.get("posts", []): - if not _is_after(node.get("timestamp"), cutoff): - continue - yield _emit(node, input_url=resolved.url) - emitted += 1 - if emitted >= per_target: - return - return - - -async def _comments_flow( - resolved: ResolvedUrl, - *, - input_model: InstagramScrapeInput, - per_target: int, -) -> AsyncIterator[dict[str, Any]]: - """Emit comment items for a post / reel URL. - - ponytail: the anonymous comment page uses a GraphQL cursor whose doc-id - drifts; v1 sources the comments embedded in the media info payload and caps - at the actor's 50/post ceiling. Deeper paging is the upgrade path in - ``fetch.py``. - """ - from .parsers import _edges - - path = f"p/{resolved.value}/" - data = await fetch_json(path, {"__a": 1, "__d": "dis"}) - node = None - if isinstance(data, dict): - items = data.get("items") - if isinstance(items, list) and items: - node = items[0] - else: - gql = data.get("graphql") - node = gql.get("shortcode_media") if isinstance(gql, dict) else None - if not isinstance(node, dict): - return - comment_nodes = _edges(node.get("edge_media_to_parent_comment")) or _edges( - node.get("edge_media_to_comment") - ) - cap = min(per_target, 50) - emitted = 0 - for cnode in comment_nodes: - item = parse_comment(cnode, post_url=resolved.url) - yield _emit(item, input_url=resolved.url) - emitted += 1 - if input_model.includeNestedComments: - for reply in _edges(cnode.get("edge_threaded_comments")): - if emitted >= cap: - return - yield _emit( - parse_comment(reply, post_url=resolved.url), - input_url=resolved.url, - ) - emitted += 1 - if emitted >= cap: + if resolved.kind in ("post", "reel"): + # Single-post extraction: the anonymous ``?__a=1`` JSON API 404s/login- + # walls, but the public /p// document embeds the post's og-meta + + # ld+json, which parse_post reads. Numeric-ID URLs can't be keyed this + # way (the page needs the shortCode), so they're skipped upstream. + if resolved.numeric_post_id: return + html = await fetch_html(f"p/{resolved.value}/") + item = parse_post(html, url=resolved.url, shortcode=resolved.value) + if item is None: + return + if not _media_matches(item, result_type): + return + if not _is_after(item.get("timestamp"), cutoff): + return + yield _emit(item, input_url=resolved.url) + return async def _details_flow( resolved: ResolvedUrl, *, input_model: InstagramScrapeInput ) -> AsyncIterator[dict[str, Any]]: - """Emit one profile / hashtag / place detail item for a URL.""" + """Emit one profile detail item for a URL (anonymous web_profile_info).""" if resolved.kind == "profile": user = await _profile_user(resolved.value) if user is not None: yield _emit(parse_profile(user), input_url=resolved.url) - return - if resolved.kind == "hashtag": - data = await fetch_json(_HASHTAG_PATH, {"tag_name": resolved.value}) - if isinstance(data, dict): - yield _emit(parse_hashtag(data), input_url=resolved.url) - return - if resolved.kind == "place": - data = await fetch_json(_LOCATION_PATH, {"location_id": resolved.value}) - if isinstance(data, dict): - yield _emit(parse_place(data), input_url=resolved.url) - return + + +def _kind_matches(resolved: ResolvedUrl, search_type: str) -> bool: + """True if a resolved IG URL is the kind the discovery query asked for. + + Discovery is profile-only now (hashtag/place feeds are login-walled), so + every supported ``search_type`` maps to a profile target. + """ + return resolved.kind == "profile" + + +async def _discover_via_google( + query: str, *, search_type: str, limit: int +) -> list[ResolvedUrl]: + """Discover IG profile targets via Google ``site:instagram.com`` (anonymous). + + Instagram's own keyword search is login-walled, so we reuse the existing + ``google_search`` platform, classify each organic URL with ``resolve_url``, + keep the profile hits, de-dup, and cap at ``limit``. + + Quality caveat: results reflect Google's index/ranking of instagram.com, not + IG's own relevance. This is discovery, not search parity (see README). + """ + serps = await scrape_serps( + GoogleSearchScrapeInput( + queries=query, site="instagram.com", maxPagesPerQuery=2 + ), + limit=2, + ) + resolved: list[ResolvedUrl] = [] + seen: set[tuple[str, str]] = set() + for serp in serps: + for org in serp.get("organicResults") or []: + url = org.get("url", "") if isinstance(org, dict) else "" + r = resolve_url(url) + if r is None or not _kind_matches(r, search_type): + continue + key = (r.kind, r.value) + if key in seen: + continue + seen.add(key) + resolved.append(r) + if len(resolved) >= limit: + return resolved + return resolved async def _discover( query: str, *, search_type: str, limit: int ) -> list[ResolvedUrl]: - """Resolve a discovery query into targets - anonymously. + """Resolve a discovery query into profile targets - anonymously. - Instagram's keyword search (``topsearch``) is login-walled, so we never call - it. A profile/user query is resolved as a direct handle lookup against the - anonymous profile endpoint ("messi" -> instagram.com/messi/). Hashtag/place - keyword discovery has NO anonymous endpoint (topsearch and the tag/location - feeds all require a session), so we surface a clear block instead of a - misleading empty success. + A query that is a valid handle resolves directly against the anonymous + profile endpoint ("messi" -> instagram.com/messi/). A non-handle query (e.g. + "national geographic") goes through Google ``site:instagram.com`` since IG's + native keyword search is login-walled. """ - if search_type in ("profile", "user"): - handle = query.strip().lstrip("@") - if _HANDLE_RE.match(handle): - url = f"https://www.instagram.com/{handle}/" - return [ResolvedUrl("profile", handle, url)][:limit] - return [] # not a handle, and no anonymous fuzzy search -> nothing to do - raise InstagramAccessBlockedError( - f"Instagram {search_type} search requires login and is unavailable in " - "anonymous mode - pass a profile URL or handle via directUrls instead" - ) + handle = query.strip().lstrip("@") + if _HANDLE_RE.match(handle): + url = f"https://www.instagram.com/{handle}/" + return [ResolvedUrl("profile", handle, url)][:limit] + return await _discover_via_google(query, search_type=search_type, limit=limit) def _resolve_inputs(input_model: InstagramScrapeInput) -> list[ResolvedUrl]: @@ -425,17 +387,6 @@ async def iter_instagram( cutoff = _parse_newer_than(input_model.onlyPostsNewerThan) per_target = input_model.resultsLimit or 10 - if result_type == "comments": - jobs = [ - _comments_flow(r, input_model=input_model, per_target=per_target) - for r in targets - if r.kind in ("post", "reel") - ] - async with aclosing(fan_out(jobs, concurrency=_COMMENT_CONCURRENCY)) as stream: - async for item in stream: - yield item - return - if result_type == "details": jobs = [_details_flow(r, input_model=input_model) for r in targets] async with aclosing(fan_out(jobs)) as stream: diff --git a/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py index 596b85c2c..aec46a678 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py @@ -1,18 +1,18 @@ """Classify and normalize an Instagram URL into a scrape job. -Covers the supported ``directUrls`` shapes: a profile, a post (``/p/``), a reel -(``/reel/``), a hashtag (``/explore/tags/``), and a place -(``/explore/locations/``), plus bare profile IDs. Non-Instagram hosts resolve to -``None`` so the orchestrator can skip them. Mirrors the frozen ``ResolvedUrl`` -dataclass shape of ``../reddit/url_resolver.py``. +Covers the anonymously-scrapable ``directUrls`` shapes: a profile, a post +(``/p/``), and a reel (``/reel/``), plus bare profile IDs. Hashtag and place +URLs are deliberately unsupported — their feeds are login-walled for anonymous +callers (use Google-backed discovery + single-post extraction instead). +Non-Instagram hosts resolve to ``None`` so the orchestrator can skip them. +Mirrors the frozen ``ResolvedUrl`` dataclass shape of ``../reddit/url_resolver.py``. Normalization rules (from the reference spec): - ``_u/`` and ``/profilecard/`` segments are stripped. - Story URLs (``/stories//...``) reduce to the profile. -- Location URLs are valid with the numeric ID alone (no trailing slug). -- Numeric post-ID URLs are only valid for the ``comments`` flow; elsewhere the - shortCode form is required, so a numeric-ID URL resolves with - ``numeric_post_id`` set and callers reject it outside comments. +- Numeric post-ID URLs cannot be single-post-extracted anonymously (the HTML + page keys on the shortCode), so they resolve with ``numeric_post_id`` set and + the media flow skips them. - ``share/`` redirect resolution is handled at fetch time (network), not here. """ @@ -22,7 +22,7 @@ from dataclasses import dataclass from typing import Literal from urllib.parse import urlparse -ResolvedKind = Literal["profile", "post", "reel", "hashtag", "place"] +ResolvedKind = Literal["profile", "post", "reel"] _INSTAGRAM_HOSTS = frozenset( {"m.instagram.com", "www.instagram.com", "instagram.com"} @@ -75,11 +75,6 @@ def resolve_url(url: str) -> ResolvedUrl | None: if not segments: return None head = segments[0] - if head == "explore" and len(segments) >= 3 and segments[1] == "tags": - return ResolvedUrl("hashtag", segments[2], url) - if head == "explore" and len(segments) >= 3 and segments[1] == "locations": - slug = segments[3] if len(segments) >= 4 else None - return ResolvedUrl("place", segments[2], url, slug=slug) if head == "p" and len(segments) >= 2: code = segments[1] return ResolvedUrl("post", code, url, numeric_post_id=code.isdigit()) From 36bb4a1bea72722a6cd4af5bdcbfa172b28f23ed Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:20:54 +0530 Subject: [PATCH 44/72] docs(instagram): update platform scraper README --- .../proprietary/platforms/instagram/README.md | 148 +++++++++++------- 1 file changed, 92 insertions(+), 56 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/README.md b/surfsense_backend/app/proprietary/platforms/instagram/README.md index fa42e30a6..e4d7ca0f7 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/README.md +++ b/surfsense_backend/app/proprietary/platforms/instagram/README.md @@ -1,37 +1,44 @@ -# Instagram scraper (anonymous, no browser) +# Instagram scraper (anonymous) -Platform-native Instagram scraper (anonymous, no browser). Standalone module: it -depends only on `app.utils.proxy` + `scrapling` and exposes a stable public API. -Its input/output surface is a **reference-compatible** mirror of the public -Instagram scraper actor spec (same `resultsType` / `directUrls` / camelCase field -names, additive `extra="allow"` parity), so callers written against that surface -work unchanged. It is **not** wired into ingestion or Celery — the capability -layer under `app/capabilities/instagram/` is what turns these primitives into -REST + agent + MCP surfaces. +Platform-native Instagram scraper. **Anonymous-only** and browser-free: every +flow stays on the cheap HTTP tier (`app.utils.proxy` + `scrapling`), and profile +discovery reuses the `google_search` platform (see below). It exposes a stable +public API whose input/output surface mirrors the public Instagram scraper actor +spec (same `resultsType` / `directUrls` / camelCase field names, additive +`extra="allow"` parity), so callers written against that surface work unchanged. +It is **not** wired into ingestion or Celery — the capability layer under +`app/capabilities/instagram/` turns these primitives into REST + agent + MCP +surfaces. ## Approach -Instagram's public web app exposes anonymous, logged-out JSON behind a handful of -`www.instagram.com` endpoints once a session carries an anonymous `csrftoken` + -`mid` cookie pair and the `x-ig-app-id` web header: +Instagram's public web app exposes anonymous, logged-out data once a session +carries an anonymous `csrftoken` + `mid` cookie pair and the `x-ig-app-id` web +header: > Warm an anonymous session with one plain GET to `www.instagram.com/` (mints -> `csrftoken` + `mid`), then GET the web JSON endpoints through that same +> `csrftoken` + `mid`), then GET the web endpoints through that same > Chrome-impersonated, sticky-IP session. Rotate the residential IP + re-warm on > a login wall (401/403), back off on 429. -Endpoints used (anonymous web tier only): +Surfaces used: -| Flow | Endpoint | -|---|---| -| profile / posts / reels | `api/v1/users/web_profile_info/?username=…` | -| comments | `p//?__a=1&__d=dis` | -| hashtag | `api/v1/tags/web_info/?tag_name=…` | -| place | `api/v1/locations/web_info/?location_id=…` | -| discovery search | `web/search/topsearch/?query=…` | +| Flow | Surface | Extractor | +|---|---|---| +| profile / details | `api/v1/users/web_profile_info/?username=…` (JSON) | `parse_profile` | +| profile feed (posts/reels/mentions) | the media embedded in the same profile JSON | `parse_media` | +| single post / reel | `/p//` (HTML: ld+json + og-meta) | `parse_post` | +| profile discovery | Google `site:instagram.com ` | `resolve_url` | -**No browser, no Chromium, no `solve_cloudflare`** — this stays on the cheap HTTP -tier the sibling scrapers already use. +**Why anonymous-only is a hard constraint.** Live logged-out probes show that +Instagram walls the interesting endpoints for anyone without a `sessionid` +account cookie: `api/v1/tags/web_info/`, `api/v1/locations/web_info/`, the +comment thread API (`?__a=1`), and `web/search/topsearch/` all **302 to +`/accounts/login/`**. We cannot log in (see below), so hashtag feeds, place +feeds, comment scraping, and IG's native keyword search were **removed** — they +can only ever return a login wall. What survives is what a logged-out browser can +actually read: a profile's web info + its embedded recent media, and a public +post/reel page's embedded metadata. ## Anonymous only — no authentication, ever @@ -47,59 +54,88 @@ so the capability layer can map it to a `403 INSTAGRAM_ACCESS_BLOCKED`. | File | Responsibility | |---|---| | `__init__.py` | Public exports: `InstagramScrapeInput`, item models, `iter_instagram`, `scrape_instagram`, `InstagramAccessBlockedError`. | -| `schemas.py` | `InstagramScrapeInput` (`extra="allow"`, no auth fields) + optional-field item models (`InstagramMediaItem`, `InstagramComment`, `InstagramProfile`, `InstagramHashtag`, `InstagramPlace`) each with `to_output()`. | -| `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (csrftoken/mid) + `fetch_json`. No browser imports. | -| `url_resolver.py` | Classify an Instagram URL → `profile`/`post`/`reel`/`hashtag`/`place`; non-Instagram → `None`. Strips `_u/`, `profilecard/`; story → profile. | -| `parsers.py` | Pure JSON→dict mapping (`parse_media`, `parse_comment`, `parse_profile`, `parse_hashtag`, `parse_place`, `_edges`). I/O-free. | -| `scraper.py` | Orchestrator: `_media_flow`/`_comments_flow`/`_details_flow`/`_discover`, `_targets`, `fan_out`, `iter_instagram`, `scrape_instagram`. | +| `schemas.py` | `InstagramScrapeInput` (`extra="allow"`, no auth fields) + optional-field item models (`InstagramMediaItem`, `InstagramProfile`) each with `to_output()`. | +| `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (csrftoken/mid) + `fetch_json` (JSON) / `fetch_html` (HTML) sharing one resilient `_fetch(path, params, extract)` loop. | +| `url_resolver.py` | Classify an Instagram URL → `profile`/`post`/`reel`; non-Instagram (and hashtag/place) → `None`. Strips `_u/`, `profilecard/`; story → profile. | +| `parsers.py` | Pure mapping (`parse_media`, `parse_profile`, `parse_post` [ld+json/og], `_edges`). I/O-free. | +| `scraper.py` | Orchestrator: `_media_flow`/`_details_flow`/`_discover` (+ `_discover_via_google`), `_targets`, `fan_out`, `iter_instagram`, `scrape_instagram`. | ## How it works 1. `iter_instagram` resolves `directUrls` (or runs a discovery `search` per the comma-split queries) into targets and fans them out on a pool of warm proxy - sessions (`fan_out`, 8-way; 4-way for comments). Each worker opens one - sticky-IP session and warms `csrftoken`/`mid` once, reusing it across the - sequential targets it pulls. -2. `resultsType` selects the flow: `posts`/`reels`/`mentions` → media feeds, - `comments` → per-post comment items, `details` → profile/hashtag/place - metadata. Media items de-dupe by `id` across targets. -3. `fetch_json` warms the session on first use, rotates the IP + re-warms on - 401/403, backs off on 429, returns `None` on 404. -4. Parsers map raw web JSON to flat dicts; the orchestrator stamps `scrapedAt` - and applies `resultsLimit` / `onlyPostsNewerThan` as request-time policy. + sessions (`fan_out`, 8-way). Each worker opens one sticky-IP session and warms + `csrftoken`/`mid` once, reusing it across the sequential targets it pulls. +2. `resultsType` selects the flow: `posts`/`reels`/`mentions` → media items, + `details` → profile metadata. Media items de-dupe by `id` across targets. + - A **profile** target → `web_profile_info` JSON → `parse_media` over the + embedded recent-media edges (feed) or `parse_profile` (details). + - A **post/reel** target → `fetch_html("p//")` → `parse_post`, which + reads the page's `application/ld+json` (preferred) and Open Graph meta + (fallback). Numeric-ID post URLs are skipped (the page keys on the shortCode). +3. `fetch_json` / `fetch_html` warm the session on first use, rotate the IP + + re-warm on 401/403, back off on 429, return `None` on 404, and raise + `InstagramAccessBlockedError` on a `/accounts/login/` redirect. +4. Parsers map raw web JSON/HTML to flat dicts; the orchestrator stamps + `scrapedAt` and applies `resultsLimit` / `onlyPostsNewerThan` as request-time + policy. + +## Profile discovery (Google-backed) + +Instagram's native keyword search is login-walled, so `_discover` resolves a +query that is a valid handle directly (`"messi"` → `instagram.com/messi/`) and +routes any other query (e.g. `"national geographic"`) through +`_discover_via_google`, which calls the `google_search` platform with +`site:instagram.com`, classifies each organic URL with `resolve_url`, keeps the +**profile** hits (discovery is profile-only), de-dupes, and caps at `searchLimit`. + +Caveats: + +- **Coupling**: Instagram depends on the `google_search` platform. The dependency + is one-directional and lives behind `_discover_via_google` so it stays testable. +- **Quality**: results reflect Google's index/ranking of `instagram.com`, not + IG's own relevance. This is discovery, not search parity. ## Observed limits & calibration caveats -- Anonymous web JSON is rate-limited per IP; the sticky-session pool keeps each - IP's request rate modest but a hot pool will still hit login walls — that's the - `InstagramAccessBlockedError` path, not a bug. +- Anonymous web JSON/HTML is rate-limited per IP; the sticky-session pool keeps + each IP's request rate modest but a hot pool will still hit login walls — that's + the `InstagramAccessBlockedError` path, not a bug. - `likesCount` is frequently withheld on anonymous responses (surfaces as `-1` or absent upstream); treat it as best-effort. -- Comments on the anonymous media page cap at ~50/post; deeper paging needs the - GraphQL cursor endpoint whose doc-id drifts (see the `ponytail:` note in - `scraper.py`/`fetch.py`). +- **Single-post extraction** reads whatever the public `/p/` document embeds + (ld+json + og-meta). If Instagram strips both for a given post (private, taken + down, or a login interstitial), `parse_post` returns `None` — an honest empty, + never a fabricated item. ponytail: the embedded-blob shapes can drift; a live + probe that dumps the raw HTML pins them (see Testing) and any change is contained + to `parse_post`. - The `$3.50 / 1k items` default meter assumes the proxy-bytes-per-item measured - on the reference targets; re-measure with the `references/` scale harness before - high-volume production use. + on the reference targets; re-measure with the scale harness before high-volume use. ## Testing - Offline unit tests: `tests/unit/platforms/instagram/` — `test_skeleton.py` - (schema + URL resolver), `test_parsers.py` (fixture-pinned mapping), - `test_fetch_resilience.py` (warm / rotate / backoff loop with fake sessions, no - network), `test_budget.py` (fair-share caps + de-dup). -- Live e2e (needs network + residential proxy): `scripts/e2e_instagram_scraper.py` - — step 0 is the go/no-go cookie probe; later steps exercise the flows and dump - trimmed, PII-anonymized fixtures. + (schema + URL resolver), `test_parsers.py` (mapping incl. `parse_post` + ld+json/og shapes; fixture-pinned tests skip when the fixture is absent), + `test_discovery.py` (Google-backed profile discovery with a fake `scrape_serps`), + `test_fetch_resilience.py` (warm / rotate / backoff loop + fan-out with fake + sessions, no network), `test_budget.py` (fair-share caps + de-dup). +- Stress / accuracy harness (live, needs network + residential proxy): + `scripts/stress/stress_instagram_scraper.py` — `--mode live-discovery` (profile + discovery accuracy), `--mode probe-post` (dumps a real anonymous `/p/` payload + to `fixtures/post.json` and shows what `parse_post` extracted), and + `--mode accuracy` (field coverage across the profile + single-post flows). ```bash cd surfsense_backend -.venv/bin/python -m pytest tests/unit/platforms/instagram/ -.venv/bin/python scripts/e2e_instagram_scraper.py # live; regenerates fixtures +uv run pytest tests/unit/platforms/instagram/ +# Live single-post probe: confirms /p/ is anonymously extractable + pins the shape +uv run python scripts/stress/stress_instagram_scraper.py --mode probe-post \ + --post https://www.instagram.com/p// ``` ## TODO / out of scope (v1) -- Deep feed pagination past the first web page of media (GraphQL cursor doc-id). -- Deep comment pagination past the ~50/post embedded ceiling. +- Deep feed pagination past the first web page of profile media (GraphQL cursor + doc-id). - Sticky-IP provider parity (same `__sid` caveat as the Reddit sibling). From 8fc86b7fa57bdc36366ab9979754a14275a6d4b6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:20:58 +0530 Subject: [PATCH 45/72] refactor(mcp): align Instagram scraper tool with new pipeline --- .../features/scrapers/platforms/instagram.py | 114 +++++------------- surfsense_mcp/mcp_server/selfcheck.py | 1 - surfsense_mcp/mcp_server/server.py | 2 +- 3 files changed, 30 insertions(+), 87 deletions(-) diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py index 39dfea71d..9683e6581 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py @@ -1,4 +1,4 @@ -"""Instagram scraper tools: posts/reels, comments, and details.""" +"""Instagram scraper tools: posts/reels and profile details (anonymous-only).""" from __future__ import annotations @@ -14,14 +14,13 @@ from ..annotations import SCRAPE from ..capability import run_scraper ResultType = Literal["posts", "reels", "mentions"] -SearchType = Literal["hashtag", "profile", "place", "user"] -DetailSearchType = Literal["hashtag", "profile", "place"] +SearchType = Literal["profile", "user"] def register( mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext ) -> None: - """Register the Instagram scrape, comments, and details tools.""" + """Register the Instagram scrape and details tools (anonymous-only).""" @mcp.tool( name="surfsense_instagram_scrape", @@ -33,21 +32,22 @@ def register( urls: Annotated[ list[str] | None, Field( - description="Instagram URLs: a profile, post (/p/), reel " - "(/reel/), hashtag (/explore/tags/), or place " - "(/explore/locations/). Provide urls OR search_queries." + description="Instagram URLs: a profile, post (/p/), or reel " + "(/reel/). Hashtag/place URLs are unsupported (login-walled). " + "Provide urls OR search_queries." ), ] = None, search_queries: Annotated[ list[str] | None, Field( - description="Terms to discover content for (hashtags as plain " - "text, no '#'). Provide search_queries OR urls." + description="Terms to discover public profiles for (resolved via " + "Google). Provide search_queries OR urls." ), ] = None, search_type: Annotated[ - SearchType, Field(description="What to discover from search_queries.") - ] = "hashtag", + SearchType, + Field(description="Discovery kind (profile-only)."), + ] = "profile", result_type: Annotated[ ResultType, Field(description="Which feed to return. 'mentions' needs profile URLs."), @@ -61,10 +61,10 @@ def register( """Scrape public Instagram posts or reels from URLs or search queries. Use this for Instagram content research: a creator's recent posts, a - hashtag or location feed, or discovering posts by keyword. For a post's - comment section use surfsense_instagram_comments; for profile/hashtag/ - place metadata use surfsense_instagram_details. Returns per-item caption, - likes, comments count, media URLs, and owner. + single post/reel, or discovering public profiles by keyword. For a + profile's metadata use surfsense_instagram_details. Returns per-item + caption, likes, comments count, media URLs, and owner. Anonymous-only: + hashtag/place feeds and comment threads are login-walled and unavailable. Example: urls=['https://www.instagram.com/natgeo/'], result_type='reels'. """ return await run_scraper( @@ -83,64 +83,9 @@ def register( response_format=response_format, ) - @mcp.tool( - name="surfsense_instagram_comments", - title="Fetch Instagram comments", - annotations=SCRAPE, - structured_output=False, - ) - async def instagram_comments( - urls: Annotated[ - list[str], - Field( - min_length=1, - description="Instagram post or reel URLs, e.g. " - "['https://www.instagram.com/p/Cabc123/'].", - ), - ], - max_comments_per_post: Annotated[ - int, - Field(ge=1, le=50, description="Max comments per post (Instagram caps at 50)."), - ] = 10, - include_replies: Annotated[ - bool, Field(description="Include nested replies.") - ] = False, - newest_first: Annotated[ - bool, Field(description="Return newest comments first.") - ] = False, - max_items: Annotated[ - int, Field(ge=1, description="Max total comments across all posts.") - ] = 20, - workspace: WorkspaceParam = None, - response_format: ResponseFormatParam = "markdown", - ) -> str: - """Fetch the comments (and optionally replies) on Instagram posts or reels. - - Use this when the user wants a post's discussion or audience reaction - rather than the post itself; get post URLs from surfsense_instagram_scrape - if you only have a topic or profile. Returns comment text, author, likes, - and replies. - Example: urls=['https://www.instagram.com/p/Cabc123/'], include_replies=True. - """ - return await run_scraper( - client, - context, - platform="instagram", - verb="comments", - payload={ - "urls": urls, - "max_comments_per_post": max_comments_per_post, - "include_replies": include_replies, - "newest_first": newest_first, - "max_items": max_items, - }, - workspace=workspace, - response_format=response_format, - ) - @mcp.tool( name="surfsense_instagram_details", - title="Fetch Instagram profile/hashtag/place details", + title="Fetch Instagram profile details", annotations=SCRAPE, structured_output=False, ) @@ -148,35 +93,34 @@ def register( urls: Annotated[ list[str] | None, Field( - description="Profile, hashtag, or place URLs (or bare profile " - "IDs). Provide urls OR search_queries." + description="Profile URLs (or bare profile IDs). Provide urls OR " + "search_queries." ), ] = None, search_queries: Annotated[ list[str] | None, Field( - description="Terms to discover profiles/hashtags/places for. " - "Provide search_queries OR urls." + description="Terms to discover public profiles for. Provide " + "search_queries OR urls." ), ] = None, search_type: Annotated[ - DetailSearchType, - Field(description="What to discover from search_queries."), - ] = "hashtag", + SearchType, + Field(description="Discovery kind (profile-only)."), + ] = "profile", max_items: Annotated[ int, Field(ge=1, description="Max detail items to return.") ] = 10, workspace: WorkspaceParam = None, response_format: ResponseFormatParam = "markdown", ) -> str: - """Fetch Instagram profile, hashtag, or place metadata. + """Fetch Instagram profile metadata. - Use this for entity lookups: a profile's follower/post counts and bio, a - hashtag's post volume and top posts, or a place's coordinates and post - count. For a feed of posts use surfsense_instagram_scrape instead. Each - item carries a detailKind field marking whether it is a profile, hashtag, - or place. - Example: urls=['https://www.instagram.com/explore/tags/crossfit/']. + Use this for entity lookups: a profile's follower/post counts and bio. + For a feed of posts use surfsense_instagram_scrape instead. Each item + carries a detailKind field (always "profile"). Anonymous-only: hashtag + and place details are login-walled and unavailable. + Example: urls=['https://www.instagram.com/natgeo/']. """ return await run_scraper( client, diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index f7b7a76c5..ba2e95a78 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -26,7 +26,6 @@ EXPECTED_TOOLS = { "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_instagram_scrape", - "surfsense_instagram_comments", "surfsense_instagram_details", "surfsense_list_scraper_runs", "surfsense_get_scraper_run", diff --git a/surfsense_mcp/mcp_server/server.py b/surfsense_mcp/mcp_server/server.py index 19cf966f7..088952fd1 100644 --- a/surfsense_mcp/mcp_server/server.py +++ b/surfsense_mcp/mcp_server/server.py @@ -37,7 +37,7 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: "Prefer these tools over generic/built-in web search whenever the " "task involves Reddit (posts, comments, finding subreddits or " "communities), YouTube (videos, transcripts, comments), Instagram " - "(posts, reels, comments, profile/hashtag/place details), Google " + "(posts, reels, profile details), Google " "Maps (places, reviews), Google Search results, or reading " "specific web pages. Scraper results are persisted as runs; if an " "inline result is truncated, fetch it in full with " From 9e9dd8a1243ae82d40fae2aec39bc4463e09a123 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:02 +0530 Subject: [PATCH 46/72] feat(subagent): update Instagram subagent prompts and tools --- .../subagents/builtins/instagram/description.md | 4 ++-- .../subagents/builtins/instagram/system_prompt.md | 9 +++------ .../subagents/builtins/instagram/tools/index.py | 5 ++--- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md index 9efedc52a..9d65defe4 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/description.md @@ -1,2 +1,2 @@ -Instagram specialist: pulls structured data from Instagram — posts and reels (caption, likes, comments count, media URLs, owner, timestamp), a post's comments and replies, and profile/hashtag/place details (follower and post counts, bio, hashtag volume and top posts, place coordinates). Finds content by hashtag, profile, or place search, and compares fresh Instagram data against earlier findings in this chat. -Use whenever the task involves Instagram content or an instagram.com link. Triggers include "get this Instagram profile/post/reel", "find posts about X on Instagram", "how many followers/likes", "get the comments on this post", "what are people saying about this reel", "look up this hashtag/location", and comparisons against earlier Instagram results in this chat. Not for general web pages (use the web crawling specialist for non-Instagram URLs). +Instagram specialist: pulls structured data from public Instagram — posts and reels (caption, likes, comments count, media URLs, owner, timestamp) and profile details (follower and post counts, bio). Finds public profiles by search and compares fresh Instagram data against earlier findings in this chat. Anonymous-only: hashtag/place feeds and comment threads are login-walled and unavailable. +Use whenever the task involves public Instagram content or an instagram.com profile/post/reel link. Triggers include "get this Instagram profile/post/reel", "find the Instagram profile for X", "how many followers/likes", and comparisons against earlier Instagram results in this chat. Not for general web pages (use the web crawling specialist for non-Instagram URLs). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md index 7d5278afa..617a376b7 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md @@ -7,19 +7,16 @@ Answer the delegated question from live Instagram data gathered with your verbs, - `instagram_scrape` -- `instagram_comments` - `instagram_details` - `read_run` / `search_run` (free readers for stored scrape output) -- Known profile/post/reel/hashtag/place links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts, reels, or mentions). -- Finding content on a topic: call `instagram_scrape` with `search_queries` and the matching `search_type` (hashtag, profile, or place). -- Comments / sentiment on specific posts or reels: call `instagram_comments` with the post `urls`. -- Profile, hashtag, or place metadata (follower counts, bio, hashtag volume, coordinates): call `instagram_details`. +- Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts, reels, or mentions). Hashtag/place URLs are unsupported (login-walled). +- Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). +- Profile metadata (follower counts, bio, post count): call `instagram_details`. - Batch multiple URLs (or queries) into one call rather than many single-item calls. -- Multi-post comment analysis: a batched comments result lists posts in order, so a truncated preview usually shows only the first post(s). Before summarizing, page the stored run (or `search_run` by post id) until you have read real comments for EVERY post in the batch — never infer one post's sentiment from another's, and never report a post as "limited data" while its comments sit unread in the run. - Comparison requests: pull the current values, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, old -> new). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py index 1bfbabad9..033ec6f0c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/tools/index.py @@ -1,4 +1,4 @@ -"""``instagram`` sub-agent tools: the three Instagram capability verbs.""" +"""``instagram`` sub-agent tools: the Instagram capability verbs.""" from __future__ import annotations @@ -8,7 +8,6 @@ from langchain_core.tools import BaseTool from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.capabilities.core.access.agent import build_capability_tools -from app.capabilities.instagram.comments.definition import INSTAGRAM_COMMENTS from app.capabilities.instagram.details.definition import INSTAGRAM_DETAILS from app.capabilities.instagram.scrape.definition import INSTAGRAM_SCRAPE @@ -16,7 +15,7 @@ NAME = "instagram" RULESET = Ruleset(origin=NAME, rules=[]) -_CI_VERBS = [INSTAGRAM_SCRAPE, INSTAGRAM_COMMENTS, INSTAGRAM_DETAILS] +_CI_VERBS = [INSTAGRAM_SCRAPE, INSTAGRAM_DETAILS] def load_tools( From 69a8433910377c0ea6d9fbb813cc8e7b748bfa93 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:05 +0530 Subject: [PATCH 47/72] docs(web): refresh Instagram connector and MCP docs --- .../docs/connectors/native/instagram.mdx | 40 ++++++------------- .../content/docs/how-to/mcp-server.mdx | 2 +- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/surfsense_web/content/docs/connectors/native/instagram.mdx b/surfsense_web/content/docs/connectors/native/instagram.mdx index f9758662b..5881ac832 100644 --- a/surfsense_web/content/docs/connectors/native/instagram.mdx +++ b/surfsense_web/content/docs/connectors/native/instagram.mdx @@ -1,9 +1,11 @@ --- title: Instagram -description: Scrape public Instagram posts, reels, comments, and profile/hashtag/place details +description: Scrape public Instagram posts, reels, and profile details (anonymous-only) --- -The Instagram scraper has three verbs: **scrape** for posts and reels, **comments** for a post's comment threads, and **details** for profile, hashtag, and place metadata. It reads only public, logged-out data — no account, login, or Graph API app required. +The Instagram scraper has two verbs: **scrape** for posts and reels, and **details** for profile metadata. It reads only public, logged-out data — no account, login, or Graph API app required. + +Instagram login-walls hashtag feeds, place feeds, and comment threads for anonymous visitors, so those are not supported. The scraper focuses on what is reliably public: profiles, posts, and reels. ## Scrape posts and reels @@ -11,15 +13,15 @@ The Instagram scraper has three verbs: **scrape** for posts and reels, **comment POST /api/v1/workspaces/{workspace_id}/scrapers/instagram/scrape ``` -Give it Instagram URLs (profile, post `/p/`, reel `/reel/`, hashtag `/explore/tags/`, or place `/explore/locations/`) or search queries. Returns structured media items — caption, hashtags, mentions, like and comment counts, media URLs, and owner. +Give it Instagram URLs (profile, post `/p/`, or reel `/reel/`) or search queries. Returns structured media items — caption, hashtags, mentions, like and comment counts, media URLs, and owner. Provide exactly one source: `urls` **or** `search_queries` (never both). | Field | Default | Description | |-------|---------|-------------| -| `urls` | — | Profile, post, reel, hashtag, or place URLs or bare profile IDs (max 20 sources per call) | -| `search_queries` | — | Discovery keywords (hashtags as plaintext, no `#`); each returns up to `max_per_target` items | -| `search_type` | `hashtag` | What to discover from `search_queries`: `hashtag`, `profile`, `place`, or `user` | +| `urls` | — | Profile, post, or reel URLs or bare profile IDs (max 20 sources per call) | +| `search_queries` | — | Discovery keywords resolved to public profiles via Google; each returns up to `max_per_target` items | +| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` | | `result_type` | `posts` | Which feed to return: `posts`, `reels`, or `mentions` (`mentions` needs profile URLs) | | `newer_than` | — | Only return posts newer than this: `YYYY-MM-DD`, ISO timestamp, or relative (`2 months`), UTC | | `skip_pinned_posts` | `false` | Exclude pinned posts in posts mode | @@ -39,37 +41,19 @@ curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/instagram/scrape" \ Billing is per returned item. -## Fetch comments - -```bash -POST /api/v1/workspaces/{workspace_id}/scrapers/instagram/comments -``` - -Give it post or reel URLs; returns comment items with author, text, like and reply counts, and optionally nested replies — useful for gauging sentiment on a specific post. - -| Field | Default | Description | -|-------|---------|-------------| -| `urls` | required | 1–20 post or reel URLs | -| `max_comments_per_post` | `10` | Max comments per post (Instagram caps at 50) | -| `include_replies` | `false` | Include nested replies | -| `newest_first` | `false` | Return newest comments first | -| `max_items` | `20` | Max total comments across all posts (max 100) | - -Billing is per returned comment or reply. - ## Fetch details ```bash POST /api/v1/workspaces/{workspace_id}/scrapers/instagram/details ``` -Give it profile, hashtag, or place URLs (or search queries) and get entity metadata back: a profile's follower/post counts and bio, a hashtag's post volume and top posts, or a place's coordinates and post count. Each item carries a `detailKind` field marking whether it is a `profile`, `hashtag`, or `place`. +Give it profile URLs (or search queries) and get profile metadata back: follower/post counts and bio. Each item carries a `detailKind` field, always `profile`. | Field | Default | Description | |-------|---------|-------------| -| `urls` | — | Profile, hashtag, or place URLs or bare profile IDs (max 20) | -| `search_queries` | — | Terms to discover profiles/hashtags/places for (provide these OR urls) | -| `search_type` | `hashtag` | What to discover from `search_queries`: `hashtag`, `profile`, or `place` | +| `urls` | — | Profile URLs or bare profile IDs (max 20) | +| `search_queries` | — | Terms to discover public profiles for (provide these OR urls) | +| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` | | `max_items` | `10` | Max detail items to return | Billing is per returned item. diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index 632dc5a4d..1318197f8 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -264,7 +264,7 @@ For self-host (stdio), all settings are environment variables passed by the clie | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_comments`, `surfsense_instagram_details`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. From 42d03fdbc7ab7e46974ce8b21c1d2a1dff76ea41 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:08 +0530 Subject: [PATCH 48/72] feat(web): update Instagram marketing and playground catalog --- surfsense_web/app/(home)/mcp-server/page.tsx | 1 - .../lib/connectors-marketing/instagram.tsx | 72 +++++++------------ surfsense_web/lib/playground/catalog.ts | 1 - 3 files changed, 25 insertions(+), 49 deletions(-) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index a5557e11e..3bdfcf100 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -94,7 +94,6 @@ const TOOL_GROUPS = [ "surfsense_youtube_scrape", "surfsense_youtube_comments", "surfsense_instagram_scrape", - "surfsense_instagram_comments", "surfsense_instagram_details", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx index 33b963eb8..39a4a04cb 100644 --- a/surfsense_web/lib/connectors-marketing/instagram.tsx +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -6,9 +6,9 @@ export const instagram: ConnectorPageContent = { name: "Instagram", icon: IconBrandInstagram, - metaTitle: "Instagram Scraper API for Social Listening | SurfSense", + metaTitle: "Instagram Scraper API for Creator Research | SurfSense", metaDescription: - "Scrape public Instagram posts, reels, comments, and profiles at scale with the SurfSense Instagram Scraper API. No login, no official API, plus a free tier. Start now.", + "Scrape public Instagram posts, reels, and profiles at scale with the SurfSense Instagram Scraper API. No login, no official API, plus a free tier. Start now.", keywords: [ "instagram scraper", "instagram scraper api", @@ -16,21 +16,21 @@ export const instagram: ConnectorPageContent = { "instagram api alternative", "scrape instagram", "instagram graph api alternative", - "instagram comment scraper", "instagram profile scraper", - "instagram hashtag scraper", + "instagram post scraper", + "instagram reel scraper", "instagram data api", "instagram mcp server", - "instagram sentiment analysis", + "creator research", "social listening", ], - h1: "Instagram Scraper API for Social Listening and Creator Research", + h1: "Instagram Scraper API for Creator Research and Social Listening", heroLede: - "The SurfSense Instagram API extracts public posts, reels, comments, and profile, hashtag, and place details without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post and what their audiences say, so you spot trends and sentiment first.", + "The SurfSense Instagram API extracts public posts, reels, and profile details without logging in or registering for the Instagram Graph API. Give your AI agents a live feed of what creators post, so you spot trends and shifts in engagement first.", transcript: { - prompt: "Pull recent reels from @competitor and tell me what the comments think", + prompt: "Pull recent reels from @competitor and summarize what they're posting", toolCall: 'instagram.scrape({ urls: ["instagram.com/competitor/"],\n result_type: "reels", max_items: 20 })', rows: [ @@ -40,9 +40,9 @@ export const instagram: ConnectorPageContent = { tag: "top reel", }, { - primary: "Comments skew positive on price, negative on shipping", - secondary: "1,203 comments · 0.71 positive", - tag: "sentiment", + primary: "Cadence up 40% this month, all short-form reels", + secondary: "20 reels · 12 days", + tag: "trend", }, { primary: "3 creators tagged asking for a collab", @@ -50,34 +50,22 @@ export const instagram: ConnectorPageContent = { tag: "lead signal", }, ], - resultSummary: "20 reels · 4,910 comments · surfaced in 2.4s", + resultSummary: "20 reels · surfaced in 2.4s", }, extractIntro: - "Every call returns structured items keyed by type. Point the API at a profile, post, reel, hashtag, or place URL, or discover content with a search query.", + "Every call returns structured items keyed by type. Point the API at a public profile, post, or reel URL, or discover creators with a search query.", extractFields: [ { label: "Posts & Reels", description: "Caption, hashtags, mentions, like and comment counts, media URLs, dimensions, and timestamp.", }, - { - label: "Comments", - description: "Comment text, author, like and reply counts, and nested replies for any post.", - }, { label: "Profiles", description: "Follower, following, and post counts, bio, external URL, verified and business flags.", }, - { - label: "Hashtags", - description: "Post volume, top posts, and recent posts for any /explore/tags/ hashtag.", - }, - { - label: "Places", - description: "Name, coordinates, address, and recent posts for any /explore/locations/ place.", - }, { label: "Owner & Media", description: @@ -90,17 +78,12 @@ export const instagram: ConnectorPageContent = { { title: "Creator and competitor monitoring", description: - "Track what your competitors and target creators post, and how their audiences react. Feed the stream to an agent that flags viral formats, launches, and shifts in engagement the moment they land.", + "Track what your competitors and target creators post, and how engagement moves. Feed the stream to an agent that flags viral formats, launches, and shifts in cadence the moment they land.", }, { - title: "Audience sentiment and comment mining", + title: "Content and format research", description: - "Pull full comment threads on a post or reel and score them for sentiment, so you can measure how a launch, a collab, or a campaign actually resonated with real followers.", - }, - { - title: "Hashtag and trend research", - description: - "Map the volume and top content behind any hashtag before you spend on a campaign. Turn trend research into a content calendar your team can act on.", + "Study a creator's recent posts and reels to see which formats earn the most likes and comments, and turn that into a content calendar your team can act on.", }, { title: "Influencer vetting and outreach", @@ -123,12 +106,7 @@ export const instagram: ConnectorPageContent = { { feature: "Coverage", official: "Mostly your own or connected accounts", - surfsense: "Any public profile, post, hashtag, or place", - }, - { - feature: "Comments", - official: "Limited to accounts you manage", - surfsense: "Public comments and replies on any post, up to 50 per post", + surfsense: "Any public profile, post, or reel", }, { feature: "Setup", @@ -163,21 +141,21 @@ export const instagram: ConnectorPageContent = { type: "string[]", defaultValue: "[]", description: - "Instagram URLs or bare profile IDs: profile, post (/p/), reel (/reel/), hashtag (/explore/tags/), or place (/explore/locations/). Max 20.", + "Instagram URLs or bare profile IDs: profile, post (/p/), or reel (/reel/). Hashtag and place URLs are login-walled and unsupported. Max 20.", }, { name: "search_queries", type: "string[]", defaultValue: "[]", description: - "Discovery keywords (hashtags as plaintext, no '#'). Provide these OR urls, not both. Max 20.", + "Discovery keywords resolved to public profiles via Google. Provide these OR urls, not both. Max 20.", }, { name: "search_type", type: "string", - defaultValue: '"hashtag"', + defaultValue: '"profile"', description: - "What to discover from search_queries: hashtag, profile, place, or user. Only used with search_queries.", + "What to discover from search_queries: profile or user. Only used with search_queries.", }, { name: "result_type", @@ -261,7 +239,7 @@ export const instagram: ConnectorPageContent = { { question: "Is scraping Instagram legal?", answer: - "SurfSense reads only public Instagram data, the same posts, reels, and comments any logged-out visitor can see. It never logs in and cannot access private accounts or stories. As always, review Instagram's terms and your own compliance needs before you run at scale.", + "SurfSense reads only public Instagram data, the same posts, reels, and profiles any logged-out visitor can see. It never logs in and cannot access private accounts, stories, or login-walled feeds. As always, review Instagram's terms and your own compliance needs before you run at scale.", }, { question: "Do I need an Instagram account or the Graph API?", @@ -271,12 +249,12 @@ export const instagram: ConnectorPageContent = { { question: "What are the rate limits?", answer: - "Each call caps at 100 returned items across all sources, with up to 20 URLs or search queries per request, and up to 50 comments per post. SurfSense manages the underlying request budget and proxy rotation for you, so you scale reads without managing tokens.", + "Each call caps at 100 returned items across all sources, with up to 20 URLs or search queries per request. SurfSense manages the underlying request budget and proxy rotation for you, so you scale reads without managing tokens.", }, { - question: "Can I get comments and replies?", + question: "Can I scrape hashtags, places, or comments?", answer: - "Yes. The instagram.comments verb returns public comments on any post or reel, with author, like and reply counts, and optionally the nested replies. Get post URLs first from instagram.scrape if you only have a topic or a profile.", + "No. Instagram login-walls hashtag feeds, place feeds, and comment threads for logged-out visitors, so SurfSense does not offer them. The API focuses on what is reliably public and anonymous: profiles, posts, and reels.", }, ], diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index b1287e447..5f72c07ec 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -55,7 +55,6 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ icon: InstagramIcon, verbs: [ { name: "instagram.scrape", verb: "scrape", label: "Scrape" }, - { name: "instagram.comments", verb: "comments", label: "Comments" }, { name: "instagram.details", verb: "details", label: "Details" }, ], }, From 82a63043f7f4833afa5f360988de4ba5fcc8f17b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:11 +0530 Subject: [PATCH 49/72] test(instagram): update capability tests for schema changes --- .../capabilities/instagram/test_executor.py | 28 ++----------------- .../capabilities/instagram/test_registry.py | 8 +----- .../capabilities/instagram/test_schemas.py | 23 +++++---------- 3 files changed, 11 insertions(+), 48 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py b/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py index 9087b0bcf..a19e597c5 100644 --- a/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_executor.py @@ -9,8 +9,6 @@ from __future__ import annotations import pytest -from app.capabilities.instagram.comments.executor import build_comments_executor -from app.capabilities.instagram.comments.schemas import CommentsInput, CommentsOutput from app.capabilities.instagram.details.executor import build_details_executor from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput from app.capabilities.instagram.scrape.executor import build_scrape_executor @@ -50,10 +48,10 @@ async def test_scrape_maps_urls_and_wraps_items(): async def test_scrape_joins_search_queries(): fake = _FakeScraper([]) execute = build_scrape_executor(fake) - await execute(ScrapeInput(search_queries=["fit", "gym"], search_type="hashtag")) + await execute(ScrapeInput(search_queries=["natgeo", "nasa"], search_type="profile")) actor_input, _ = fake.calls[0] - assert actor_input.search == "fit,gym" - assert actor_input.searchType == "hashtag" + assert actor_input.search == "natgeo,nasa" + assert actor_input.searchType == "profile" assert actor_input.directUrls == [] @@ -66,26 +64,6 @@ async def test_scrape_access_blocked_maps_to_forbidden(): await execute(ScrapeInput(urls=["x"])) -async def test_comments_maps_flags(): - fake = _FakeScraper([{"id": "c1", "text": "nice"}]) - execute = build_comments_executor(fake) - out = await execute( - CommentsInput( - urls=["https://www.instagram.com/p/Cabc/"], - newest_first=True, - include_replies=True, - max_comments_per_post=25, - ) - ) - assert isinstance(out, CommentsOutput) - assert out.items[0].text == "nice" - actor_input, _ = fake.calls[0] - assert actor_input.resultsType == "comments" - assert actor_input.isNewestComments is True - assert actor_input.includeNestedComments is True - assert actor_input.resultsLimit == 25 - - async def test_details_maps_and_wraps_discriminated_items(): fake = _FakeScraper( [ diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py b/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py index 89257253a..503feedff 100644 --- a/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_registry.py @@ -1,4 +1,4 @@ -"""The instagram namespace registers its three verbs for the doors/agent to read. +"""The instagram namespace registers its verbs for the doors/agent to read. Unlike the stale reddit assertion (``billing_unit is None``), these assert the real meters — the Capability definitions are the source of truth. @@ -23,12 +23,6 @@ def test_instagram_scrape_registered_with_item_meter(): assert cap.billing_unit is BillingUnit.INSTAGRAM_ITEM -def test_instagram_comments_registered_with_comment_meter(): - cap = get_capability("instagram.comments") - assert cap.name == "instagram.comments" - assert cap.billing_unit is BillingUnit.INSTAGRAM_COMMENT - - def test_instagram_details_registered_with_item_meter(): cap = get_capability("instagram.details") assert cap.name == "instagram.details" diff --git a/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py b/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py index 13efb8a48..afabbf833 100644 --- a/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py +++ b/surfsense_backend/tests/unit/capabilities/instagram/test_schemas.py @@ -5,7 +5,6 @@ from __future__ import annotations import pytest from pydantic import ValidationError -from app.capabilities.instagram.comments.schemas import CommentsInput from app.capabilities.instagram.details.schemas import DetailsInput, DetailsOutput from app.capabilities.instagram.scrape.schemas import ( MAX_INSTAGRAM_ITEMS, @@ -47,30 +46,22 @@ def test_scrape_bounds(): ) -def test_comments_requires_urls_and_caps_at_50(): +def test_scrape_rejects_walled_search_type(): + # Discovery is profile-only; hashtag/place are login-walled and rejected. with pytest.raises(ValidationError): - CommentsInput(urls=[]) - with pytest.raises(ValidationError): - CommentsInput( - urls=["https://www.instagram.com/p/Cabc/"], max_comments_per_post=51 - ) - ok = CommentsInput( - urls=["https://www.instagram.com/p/Cabc/"], max_comments_per_post=50 - ) - assert ok.max_comments_per_post == 50 + ScrapeInput(search_queries=["travel"], search_type="hashtag") -def test_details_discriminated_union_selects_by_detail_kind(): +def test_details_wraps_profile_items(): out = DetailsOutput( items=[ {"detailKind": "profile", "username": "natgeo"}, - {"detailKind": "hashtag", "name": "fit"}, - {"detailKind": "place", "name": "Copenhagen"}, + {"detailKind": "profile", "username": "nasa"}, ] ) kinds = [type(i).__name__ for i in out.items] - assert kinds == ["InstagramProfile", "InstagramHashtag", "InstagramPlace"] - assert out.billable_units == 3 + assert kinds == ["InstagramProfile", "InstagramProfile"] + assert out.billable_units == 2 def test_details_rejects_both_sources(): From ba70ae1f452ffb72c7c5a5ac565dd327c7afc56b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:14 +0530 Subject: [PATCH 50/72] test(instagram): update platform tests and add discovery fixtures --- .../platforms/instagram/fixtures/post.json | 1 + .../platforms/instagram/test_discovery.py | 78 +++++++++++ .../instagram/test_fetch_resilience.py | 32 +++-- .../unit/platforms/instagram/test_parsers.py | 124 ++++++++++-------- .../unit/platforms/instagram/test_skeleton.py | 26 ++-- 5 files changed, 178 insertions(+), 83 deletions(-) create mode 100644 surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json create mode 100644 surfsense_backend/tests/unit/platforms/instagram/test_discovery.py diff --git a/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json b/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json new file mode 100644 index 000000000..5eba4b5ad --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json @@ -0,0 +1 @@ +{"url": "https://www.instagram.com/p/Dan5IDtDE2I/", "shortcode": "Dan5IDtDE2I", "html": "\n\n\n\n\n\n\n\nInstagram\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n{\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_before_last_payload\"]]]}{\"require\":[[\"CometSSRMergedContentInjector\",\"onViewportGuessValidation\",null,[[{\"dimension\":\"width\",\"numPixels\":767,\"operation\":\"max\",\"result\":false},{\"dimension\":\"width\",\"numPixels\":1264,\"operation\":\"min\",\"result\":true}],{\"width_px\":1366,\"height_px\":722,\"guess_source\":\"fallback\"}]]]}\n \n {\"require\":[[\"CometSSRMergedContentInjector\",\"replayStyleInjectsForSSR\",null,[[]]]]}\n
Instagram
Close
\"natgeodocs's
Never miss a post from natgeodocs
Sign up for Instagram to stay in the loop.
Sign up
Log in
\"Photo
\"natgeo's
\"natgeodocs's
natgeodocs
 and 4 others
More options
\"natgeodocs's
natgeodocs
Verified
\u00a0Edited\u2022
The true story of four children who survived 40 days and 40 nights stranded in the Colombian rainforest. The siblings recount what happened in #LostInTheJungle, now streaming on @disneyplus and @hulu.
\"jeffred441's
jeffred441
\u00a0
Everyone will definitely going to enjoy watching Lost in the Jungle on @hulu and @disneyplus
Reply
\"saneehaazeemahmed's
saneehaazeemahmed
\u00a0
Fav survival story
Reply
\"daikon.do's
daikon.do
\u00a0
Eldest sister final boss
10 likes
Reply
\"fourmiboys's
fourmiboys
\u00a0
This was great, sorry they had to go through it.
1 like
Reply
\"babs_thecreator's
babs_thecreator
\u00a0
Incredible \u2764\ufe0f
Reply
\"urss_chhavi01's
urss_chhavi01
Verified
\u00a0
\ud83d\ude4c\ud83d\ude0d\ud83d\ude4c\ud83d\ude0d\ud83d\ude4c
1 like
Reply
\"urss_chhavi01's
urss_chhavi01
Verified
\u00a0
\ud83d\ude0d\u2764\ufe0f\u2764\ufe0f\ud83d\ude0d
1 like
Reply
\"joyetha_assam_vibes's
joyetha_assam_vibes
\u00a0
\ud83d\ude2e
Reply
\"alexottscience's
alexottscience
Verified
\u00a0
Ayahuasca saved them \ud83c\udf3f
1 like
Reply
\"laurelineorsettiworldwide's
laurelineorsettiworldwide
\u00a0
Spectacular! Inspirational!
1 like
Reply
Like
2.3K
Comment
10
Share
Save
Log in to like or comment.
More posts from natgeodocs
See more posts
Meta
About
Blog
Jobs
Help
API
Privacy
Consumer Health Privacy
Terms
Locations
Popular
Instagram Lite
Meta AI
Threads
Contact Uploading & Non-Users
Meta Verified
English
Down chevron icon
\u00a9 2026 Instagram from Meta
\n {\"require\":[[\"CometSSRMergedContentInjector\",\"onPayloadReceived\",null,[{\"fizzRootId\":\"ssrb_root_content\",\"id\":\"render_pass_0\",\"payloadType\":\"LAST\",\"readyPreloaders\":[\"adp_PolarisLoggedOutDesktopWWWPostRootContentQueryRelayPreloader_6a5159be0652c5722640683\",\"adp_PolarisWWWLoggedOutDynamicLandingDialogQueryRelayPreloader_6a5159be065439d65551266\"],\"renderPassCount\":0,\"status\":\"success\"},{\"clientRenderErrors\":[],\"productRecoverableErrors\":[]}]]]}\n \n {\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_after_last_payload\"]]]}{\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_last_payload_0\"]]]}\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"} \ No newline at end of file diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_discovery.py b/surfsense_backend/tests/unit/platforms/instagram/test_discovery.py new file mode 100644 index 000000000..c95a80315 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/instagram/test_discovery.py @@ -0,0 +1,78 @@ +"""Offline tests for Google-backed Instagram discovery. + +Discovery is profile-only (hashtag/place feeds are login-walled). A valid handle +resolves directly; any other query falls back to the ``google_search`` platform +(``site:instagram.com``), classifying organic results with ``resolve_url`` and +keeping only profile hits. These tests inject a fake ``scrape_serps`` so there is +no network: they pin the classification, de-dup, and ``limit`` cap. +""" + +from __future__ import annotations + +from app.proprietary.platforms.instagram import scraper + + +def _fake_serps(*organic_urls: str): + async def _scrape_serps(input_model, *, limit=None): + assert input_model.site == "instagram.com" + return [{"organicResults": [{"url": u} for u in organic_urls]}] + + return _scrape_serps + + +async def test_google_discovery_keeps_only_profiles(monkeypatch): + # A non-handle query goes to Google; only profile URLs survive (hashtag / + # post / non-instagram results are dropped since discovery is profile-only). + monkeypatch.setattr( + scraper, + "scrape_serps", + _fake_serps( + "https://www.instagram.com/natgeo/", + "https://www.instagram.com/explore/tags/travel/", + "https://www.instagram.com/p/ABC123/", + "https://example.com/not-instagram", + ), + ) + targets = await scraper._discover( + "nat geo photos", search_type="profile", limit=10 + ) + assert [(t.kind, t.value) for t in targets] == [("profile", "natgeo")] + + +async def test_google_discovery_dedupes(monkeypatch): + monkeypatch.setattr( + scraper, + "scrape_serps", + _fake_serps( + "https://www.instagram.com/natgeo/", + "https://www.instagram.com/natgeo/", + ), + ) + targets = await scraper._discover( + "nat geo photos", search_type="profile", limit=10 + ) + assert len(targets) == 1 + + +async def test_google_discovery_respects_limit(monkeypatch): + monkeypatch.setattr( + scraper, + "scrape_serps", + _fake_serps( + "https://www.instagram.com/a_a/", + "https://www.instagram.com/b_b/", + "https://www.instagram.com/c_c/", + ), + ) + targets = await scraper._discover("some brand name", search_type="profile", limit=2) + assert [t.value for t in targets] == ["a_a", "b_b"] + + +async def test_discover_profile_handle_fast_path_skips_google(monkeypatch): + # A valid handle resolves directly without touching Google. + async def _boom(input_model, *, limit=None): + raise AssertionError("Google should not be called for a valid handle") + + monkeypatch.setattr(scraper, "scrape_serps", _boom) + targets = await scraper._discover("messi", search_type="user", limit=10) + assert [(t.kind, t.value) for t in targets] == [("profile", "messi")] diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py index e142a33f7..859c70443 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_fetch_resilience.py @@ -172,7 +172,7 @@ async def test_login_redirect_fails_fast_without_rotating(): try: raised = False try: - await fetch_json("api/v1/tags/web_info/", {"tag_name": "travel"}) + await fetch_json("api/v1/users/web_profile_info/", {"username": "natgeo"}) except InstagramAccessBlockedError: raised = True finally: @@ -185,7 +185,7 @@ async def test_404_returns_none_without_rotating(): holder = _FakeHolder([_FakeSession(404), _FakeSession(200)]) token = _current_session.set(holder) try: - result = await fetch_json("api/v1/tags/web_info/") + result = await fetch_json("api/v1/users/web_profile_info/") finally: _current_session.reset(token) assert result is None @@ -399,12 +399,22 @@ async def test_discover_profile_is_anonymous_handle_lookup(): assert [(t.kind, t.value) for t in targets] == [("profile", "messi")] -async def test_discover_hashtag_search_blocks_anonymously(): - # hashtag/place keyword discovery has no anonymous endpoint at all, so it must - # fail loud (clear message) rather than return a misleading empty success. - raised = False - try: - await scraper._discover("travel", search_type="hashtag", limit=10) - except InstagramAccessBlockedError: - raised = True - assert raised +async def test_discover_nonhandle_routes_through_google(monkeypatch): + # A non-handle profile query goes through Google (site:instagram.com) and + # classifies the organic results into profile targets (the only kind now). + async def _fake_scrape_serps(input_model, *, limit=None): + assert input_model.site == "instagram.com" + return [ + { + "organicResults": [ + {"url": "https://www.instagram.com/natgeo/"}, + {"url": "https://www.instagram.com/p/Cabc/"}, # wrong kind + ] + } + ] + + monkeypatch.setattr(scraper, "scrape_serps", _fake_scrape_serps) + targets = await scraper._discover( + "national geographic", search_type="profile", limit=10 + ) + assert [(t.kind, t.value) for t in targets] == [("profile", "natgeo")] diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py index 6f6c1411e..a4c781776 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -14,10 +14,8 @@ from pathlib import Path import pytest from app.proprietary.platforms.instagram.parsers import ( - parse_comment, - parse_hashtag, parse_media, - parse_place, + parse_post, parse_profile, ) @@ -62,23 +60,6 @@ def test_parse_media_marks_video_type(): assert item["videoViewCount"] == 99 -def test_parse_comment(): - node = { - "id": "c1", - "text": "nice", - "created_at": 1_600_000_000, - "shortcode": "Cabc", - "owner": {"username": "bob", "id": "5"}, - "edge_liked_by": {"count": 3}, - } - item = parse_comment(node, post_url="https://www.instagram.com/p/Cabc/") - assert item["id"] == "c1" - assert item["text"] == "nice" - assert item["ownerUsername"] == "bob" - assert item["likesCount"] == 3 - assert item["postUrl"] == "https://www.instagram.com/p/Cabc/" - - def test_parse_profile_flattens_counts_and_latest_posts(): user = { "id": "9", @@ -100,45 +81,63 @@ def test_parse_profile_flattens_counts_and_latest_posts(): assert len(item["latestPosts"]) == 1 -def test_parse_hashtag(): - data = { - "data": { - "id": "h1", - "name": "crossfit", - "edge_hashtag_to_media": { - "count": 5, - "edges": [{"node": {"id": "m1", "shortcode": "A"}}], - }, - "edge_hashtag_to_top_posts": { - "edges": [{"node": {"id": "t1", "shortcode": "B"}}] - }, - } - } - item = parse_hashtag(data) - assert item["detailKind"] == "hashtag" - assert item["name"] == "crossfit" - assert item["postsCount"] == 5 - assert len(item["topPosts"]) == 1 - assert len(item["posts"]) == 1 +_POST_URL = "https://www.instagram.com/p/Cabc/" -def test_parse_place(): - data = { - "location": { - "id": "7538318", - "name": "Copenhagen", - "slug": "copenhagen", - "edge_location_to_media": { - "count": 3, - "edges": [{"node": {"id": "m1", "shortcode": "A"}}], - }, - } - } - item = parse_place(data) - assert item["detailKind"] == "place" - assert item["name"] == "Copenhagen" - assert item["location_id"] == "7538318" - assert len(item["posts"]) == 1 +def test_parse_post_prefers_ldjson(): + html = """ + + + + """ + item = parse_post(html, url=_POST_URL, shortcode="Cabc") + assert item is not None + assert item["type"] == "Video" + assert item["shortCode"] == "Cabc" + assert item["url"] == _POST_URL + assert item["ownerUsername"] == "natgeo" + assert item["caption"] == "sunset over #bali with @friend" + assert item["hashtags"] == ["bali"] + assert item["mentions"] == ["friend"] + assert item["likesCount"] == 4200 + assert item["commentsCount"] == 37 + assert item["videoUrl"] == "https://cdn/v.mp4" + assert item["timestamp"] == "2024-01-02T03:04:05Z" + + +def test_parse_post_falls_back_to_og_meta(): + html = """ + + + + + + """ + item = parse_post(html, url=_POST_URL, shortcode="Cabc") + assert item is not None + assert item["likesCount"] == 1234 + assert item["commentsCount"] == 56 + assert item["displayUrl"] == "https://cdn/i.jpg" + assert item["type"] == "Video" + + +def test_parse_post_returns_none_without_surfaces(): + # A login interstitial / empty doc carries neither ld+json nor og -> None, + # never a silent empty-success item. + assert parse_post("login", url=_POST_URL) is None + assert parse_post(None, url=_POST_URL) is None + assert parse_post("", url=_POST_URL) is None @pytest.mark.skipif( @@ -151,3 +150,14 @@ def test_fixture_profile_maps(): item = parse_profile(user) assert item["detailKind"] == "profile" assert item["username"] + + +@pytest.mark.skipif( + not (_FIXTURES / "post.json").exists(), + reason="captured fixture absent (run the single-post probe to dump /p/ HTML)", +) +def test_fixture_post_maps(): + raw = json.loads((_FIXTURES / "post.json").read_text()) + item = parse_post(raw["html"], url=raw["url"], shortcode=raw.get("shortcode")) + assert item is not None, "captured /p/ HTML produced no media item" + assert item["url"] == raw["url"] diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py b/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py index 59017c81c..f609883eb 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_skeleton.py @@ -1,9 +1,10 @@ """Offline skeleton tests: input surface parity + URL classification. No network. Locks the two invariants the reference-compatible surface promises — -no auth fields ever, and additive ``extra="allow"`` parity — plus the full +no auth fields ever, and additive ``extra="allow"`` parity — plus the ``url_resolver`` classification/normalization table (``_u/`` and profilecard -stripping, story→profile, ID-only locations, numeric post-ID flagging). +stripping, story→profile, numeric post-ID flagging). Hashtag/place URLs are +login-walled and deliberately resolve to ``None``. """ from __future__ import annotations @@ -32,7 +33,7 @@ def test_input_has_no_auth_fields(): def test_input_defaults(): model = InstagramScrapeInput() assert model.resultsType == "posts" - assert model.searchType == "hashtag" + assert model.searchType == "profile" assert model.directUrls == [] assert model.addParentData is False @@ -70,19 +71,14 @@ def test_resolve_post_and_reel(): assert r.kind == "reel" and r.value == "Cxyz" -def test_resolve_hashtag(): - r = resolve_url("https://www.instagram.com/explore/tags/crossfit/") - assert r.kind == "hashtag" and r.value == "crossfit" - - -def test_resolve_place_with_slug_and_id_only(): - with_slug = resolve_url( - "https://www.instagram.com/explore/locations/7538318/copenhagen/" +def test_resolve_hashtag_and_place_unsupported(): + # Login-walled surfaces: they must resolve to None so the orchestrator skips + # them rather than building a target that can only return a login wall. + assert resolve_url("https://www.instagram.com/explore/tags/crossfit/") is None + assert ( + resolve_url("https://www.instagram.com/explore/locations/7538318/copenhagen/") + is None ) - assert with_slug.kind == "place" and with_slug.value == "7538318" - assert with_slug.slug == "copenhagen" - id_only = resolve_url("https://www.instagram.com/explore/locations/7538318/") - assert id_only.kind == "place" and id_only.value == "7538318" def test_resolve_strips_u_and_profilecard(): From 949cc1b29f7bb795b330893e2cee783c77c1b185 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:21:26 +0530 Subject: [PATCH 51/72] chore(scripts): update Instagram e2e scraper script --- .../scripts/e2e_instagram_scraper.py | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/surfsense_backend/scripts/e2e_instagram_scraper.py b/surfsense_backend/scripts/e2e_instagram_scraper.py index 95625fd06..c81075932 100644 --- a/surfsense_backend/scripts/e2e_instagram_scraper.py +++ b/surfsense_backend/scripts/e2e_instagram_scraper.py @@ -14,11 +14,14 @@ It: approach is invalid — later steps are skipped. Step 1 — scrape a profile's posts. Step 2 — scrape a profile's reels. - Step 3 — fetch comments for a discovered post URL. - Step 4 — fetch profile / hashtag / place details. - Step 5 — run a discovery search. + Step 3 — anonymous single-post extraction for a discovered ``/p/`` URL. + Step 4 — fetch profile details. + Step 5 — run a profile discovery search (Google-backed). Step 6 — dump trimmed, PII-anonymized raw fixtures into tests/unit/platforms/instagram/fixtures/ for the offline parser tests. + +Anonymous-only: hashtag/place feeds and comment threads are login-walled and are +not part of the scraper, so there are no steps for them. """ import asyncio @@ -41,15 +44,15 @@ from app.proprietary.platforms.instagram import ( # noqa: E402 scrape_instagram, ) from app.proprietary.platforms.instagram.fetch import ( # noqa: E402 + fetch_html, fetch_json, proxy_session, warm_session, ) +from app.proprietary.platforms.instagram.url_resolver import resolve_url # noqa: E402 _PROFILE = "natgeo" -_HASHTAG = "https://www.instagram.com/explore/tags/travel/" -_PLACE = "https://www.instagram.com/explore/locations/213385402/new-york-new-york/" -_SEARCH_TERM = "coffee" +_SEARCH_TERM = "national geographic" _FIXTURE_DIR = ( _BACKEND_ROOT / "tests" / "unit" / "platforms" / "instagram" / "fixtures" @@ -129,30 +132,27 @@ async def step2_reels() -> bool: return _check("reels returned items", len(items) >= 0, f"{len(items)} reels") -async def step3_comments(post_url: str | None) -> bool: - _hr("STEP 3 — comments for a post") +async def step3_single_post(post_url: str | None) -> bool: + _hr("STEP 3 — single-post extraction for a /p/ URL") if not post_url: return _check("had a post URL", False, "step 1 found no post") items = await scrape_instagram( InstagramScrapeInput( - resultsType="comments", directUrls=[post_url], resultsLimit=10 + resultsType="posts", directUrls=[post_url], resultsLimit=1 ), - limit=10, + limit=1, ) - print(f" {len(items)} comments for {post_url}") - return _check("comments returned", len(items) >= 0, f"{len(items)} comments") + got = items[0] if items else {} + print(f" {len(items)} item for {post_url} | owner={got.get('ownerUsername')}") + return _check("single post returned an item", len(items) > 0, post_url) async def step4_details() -> bool: - _hr("STEP 4 — profile / hashtag / place details") + _hr("STEP 4 — profile details") items = await scrape_instagram( InstagramScrapeInput( resultsType="details", - directUrls=[ - f"https://www.instagram.com/{_PROFILE}/", - _HASHTAG, - _PLACE, - ], + directUrls=[f"https://www.instagram.com/{_PROFILE}/"], ), limit=10, ) @@ -162,12 +162,12 @@ async def step4_details() -> bool: async def step5_search() -> bool: - _hr("STEP 5 — discovery search") + _hr("STEP 5 — profile discovery search (Google-backed)") items = await scrape_instagram( InstagramScrapeInput( resultsType="posts", search=_SEARCH_TERM, - searchType="hashtag", + searchType="profile", searchLimit=3, resultsLimit=3, ), @@ -177,7 +177,7 @@ async def step5_search() -> bool: return _check("search returned results", len(items) >= 0, f"{len(items)} items") -async def step6_dump_fixtures() -> bool: +async def step6_dump_fixtures(post_url: str | None) -> bool: _hr("STEP 6 — dump trimmed, anonymized fixtures for offline tests") profile = await fetch_json( "api/v1/users/web_profile_info/", {"username": _PROFILE} @@ -189,6 +189,17 @@ async def step6_dump_fixtures() -> bool: json.dumps(_anonymize(profile)), encoding="utf-8" ) wrote.append("profile.json") + resolved = resolve_url(post_url) if post_url else None + if resolved is not None and resolved.kind in ("post", "reel"): + html = await fetch_html(f"p/{resolved.value}/") + if html: + (_FIXTURE_DIR / "post.json").write_text( + json.dumps( + {"url": post_url, "shortcode": resolved.value, "html": html} + ), + encoding="utf-8", + ) + wrote.append("post.json") return _check("dumped fixtures", bool(wrote), f"{wrote} -> {_FIXTURE_DIR}") @@ -214,10 +225,10 @@ async def main() -> int: results.append(await step1_posts()) results.append(await step2_reels()) post_url = await _first_post_url() - results.append(await step3_comments(post_url)) + results.append(await step3_single_post(post_url)) results.append(await step4_details()) results.append(await step5_search()) - results.append(await step6_dump_fixtures()) + results.append(await step6_dump_fixtures(post_url)) _hr("SUMMARY") print(f" {sum(results)}/{len(results)} steps passed") return 0 if all(results) else 1 From 4bd3ec275c2b4b16d417a5b88724a0e202bfb748 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:50:26 +0530 Subject: [PATCH 52/72] delete(instagram): remove obsolete post fixture for Instagram tests --- .../tests/unit/platforms/instagram/fixtures/post.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json diff --git a/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json b/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json deleted file mode 100644 index 5eba4b5ad..000000000 --- a/surfsense_backend/tests/unit/platforms/instagram/fixtures/post.json +++ /dev/null @@ -1 +0,0 @@ -{"url": "https://www.instagram.com/p/Dan5IDtDE2I/", "shortcode": "Dan5IDtDE2I", "html": "\n\n\n\n\n\n\n\nInstagram\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n{\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_before_last_payload\"]]]}{\"require\":[[\"CometSSRMergedContentInjector\",\"onViewportGuessValidation\",null,[[{\"dimension\":\"width\",\"numPixels\":767,\"operation\":\"max\",\"result\":false},{\"dimension\":\"width\",\"numPixels\":1264,\"operation\":\"min\",\"result\":true}],{\"width_px\":1366,\"height_px\":722,\"guess_source\":\"fallback\"}]]]}\n \n {\"require\":[[\"CometSSRMergedContentInjector\",\"replayStyleInjectsForSSR\",null,[[]]]]}\n
Instagram
Close
\"natgeodocs's
Never miss a post from natgeodocs
Sign up for Instagram to stay in the loop.
Sign up
Log in
\"Photo
\"natgeo's
\"natgeodocs's
natgeodocs
 and 4 others
More options
\"natgeodocs's
natgeodocs
Verified
\u00a0Edited\u2022
The true story of four children who survived 40 days and 40 nights stranded in the Colombian rainforest. The siblings recount what happened in #LostInTheJungle, now streaming on @disneyplus and @hulu.
\"jeffred441's
jeffred441
\u00a0
Everyone will definitely going to enjoy watching Lost in the Jungle on @hulu and @disneyplus
Reply
\"saneehaazeemahmed's
saneehaazeemahmed
\u00a0
Fav survival story
Reply
\"daikon.do's
daikon.do
\u00a0
Eldest sister final boss
10 likes
Reply
\"fourmiboys's
fourmiboys
\u00a0
This was great, sorry they had to go through it.
1 like
Reply
\"babs_thecreator's
babs_thecreator
\u00a0
Incredible \u2764\ufe0f
Reply
\"urss_chhavi01's
urss_chhavi01
Verified
\u00a0
\ud83d\ude4c\ud83d\ude0d\ud83d\ude4c\ud83d\ude0d\ud83d\ude4c
1 like
Reply
\"urss_chhavi01's
urss_chhavi01
Verified
\u00a0
\ud83d\ude0d\u2764\ufe0f\u2764\ufe0f\ud83d\ude0d
1 like
Reply
\"joyetha_assam_vibes's
joyetha_assam_vibes
\u00a0
\ud83d\ude2e
Reply
\"alexottscience's
alexottscience
Verified
\u00a0
Ayahuasca saved them \ud83c\udf3f
1 like
Reply
\"laurelineorsettiworldwide's
laurelineorsettiworldwide
\u00a0
Spectacular! Inspirational!
1 like
Reply
Like
2.3K
Comment
10
Share
Save
Log in to like or comment.
More posts from natgeodocs
See more posts
Meta
About
Blog
Jobs
Help
API
Privacy
Consumer Health Privacy
Terms
Locations
Popular
Instagram Lite
Meta AI
Threads
Contact Uploading & Non-Users
Meta Verified
English
Down chevron icon
\u00a9 2026 Instagram from Meta
\n {\"require\":[[\"CometSSRMergedContentInjector\",\"onPayloadReceived\",null,[{\"fizzRootId\":\"ssrb_root_content\",\"id\":\"render_pass_0\",\"payloadType\":\"LAST\",\"readyPreloaders\":[\"adp_PolarisLoggedOutDesktopWWWPostRootContentQueryRelayPreloader_6a5159be0652c5722640683\",\"adp_PolarisWWWLoggedOutDynamicLandingDialogQueryRelayPreloader_6a5159be065439d65551266\"],\"renderPassCount\":0,\"status\":\"success\"},{\"clientRenderErrors\":[],\"productRecoverableErrors\":[]}]]]}\n \n {\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_after_last_payload\"]]]}{\"require\":[[\"CometSSRMergedContentInjector\",\"logQPLPoint\",null,[\"ssr_last_payload_0\"]]]}\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"} \ No newline at end of file From 467f51fd5c8a14e402adda613ff581a4dfc05197 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:50:43 +0530 Subject: [PATCH 53/72] chore(.gitignore): add Instagram test fixtures for unit tests --- surfsense_backend/.gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/.gitignore b/surfsense_backend/.gitignore index bda5961fe..13c2d40e3 100644 --- a/surfsense_backend/.gitignore +++ b/surfsense_backend/.gitignore @@ -15,4 +15,7 @@ celerybeat-schedule.* celerybeat-schedule.dir celerybeat-schedule.bak /app/config/global_llm_config.yaml -app/templates/_generated/ \ No newline at end of file +app/templates/_generated/ + +/tests/unit/platforms/instagram/fixtures/post.json +/tests/unit/platforms/instagram/fixtures/profile.json \ No newline at end of file From d41ccb7edd70a1774f3c6c307bfebc7903da1345 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:56:29 +0530 Subject: [PATCH 54/72] feat(instagram): enhance Open Graph parsing for anonymous posts - Added support for extracting likes, comments, username, timestamp, and caption from Open Graph meta tags. - Implemented fallback mechanisms to ensure graceful degradation when expected data is missing. - Updated unit tests to validate new parsing logic and ensure robustness against unrecognized formats. --- .../platforms/instagram/parsers.py | 85 ++++++++++++++++--- .../unit/platforms/instagram/test_parsers.py | 35 +++++++- 2 files changed, 108 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py index 55884537d..b4c2aeba3 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -15,6 +15,7 @@ parity is additive. from __future__ import annotations +import html import json import re from datetime import UTC, datetime @@ -185,10 +186,70 @@ _LDJSON_RE = re.compile( _OG_RE = re.compile( r' on Instagram: ". +_OG_TITLE_RE = re.compile(r"^(.+?)\s+on Instagram:\s*(.*)$", re.DOTALL) + + +def _og_date_to_iso(value: str) -> str | None: + """``"July 9, 2026"`` -> ``"2026-07-09"`` (date-only; og carries no time).""" + try: + return datetime.strptime(value, "%B %d, %Y").replace(tzinfo=UTC).date().isoformat() + except ValueError: + return None + + +def _clean_caption(raw: str) -> str | None: + """HTML-unescape and unwrap the surrounding quotes off an og caption preview.""" + return html.unescape(raw).strip().strip('"').strip() or None + + +def _parse_og_meta(og: dict[str, str]) -> dict[str, Any]: + """Lift post fields out of Instagram's Open Graph tags (see module notes above). + + Caption + full name come from ``og:title``; counts + username + date from + ``og:description``. Every field is optional and independently guarded, so a + shape we don't recognise yields a partial (or empty) dict instead of raising. + """ + out: dict[str, Any] = {} + desc = og.get("description", "") + title = og.get("title", "") + + counts = _OG_COUNTS_RE.search(desc) + if counts: + out["likes"] = _html_int(counts.group(1)) + out["comments"] = _html_int(counts.group(2)) + + owner_date = _OG_OWNER_DATE_RE.search(desc) + if owner_date: + out["ownerUsername"] = owner_date.group(1).strip().lstrip("@") or None + out["timestamp"] = _og_date_to_iso(owner_date.group(2)) + + tm = _OG_TITLE_RE.match(title) + if tm: + out["ownerFullName"] = tm.group(1).strip() or None + out["caption"] = _clean_caption(tm.group(2)) + elif owner_date: + # No usable og:title: fall back to the caption after og:description's + # date prefix — still clean (the counts/username/date are stripped). + out["caption"] = _clean_caption(desc[owner_date.end():]) + return out def _html_int(value: Any) -> int | None: @@ -278,18 +339,19 @@ def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> d (b for b in blocks if str(b.get("@type", "")).endswith(("Object", "Post"))), blocks[0] if blocks else {}, ) + # ld+json wins when present (richer, structured); og fills every gap. On + # anonymous /p/ pages ld+json is absent, so og_meta is the de-facto source. + og_meta = _parse_og_meta(og) + counts = _ld_interaction(node) - if "likes" not in counts or "comments" not in counts: - m = _OG_COUNTS_RE.search(og.get("description", "")) - if m: - counts.setdefault("likes", _html_int(m.group(1)) or 0) - counts.setdefault("comments", _html_int(m.group(2)) or 0) + counts.setdefault("likes", og_meta.get("likes")) + counts.setdefault("comments", og_meta.get("comments")) caption = ( node.get("articleBody") or node.get("caption") or node.get("description") - or og.get("description") + or og_meta.get("caption") ) caption = caption if isinstance(caption, str) else None @@ -313,13 +375,14 @@ def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> d "type": "Video" if is_video else "Image", "shortCode": shortcode, "caption": caption, - "hashtags": _HASHTAG_RE.findall(caption) if caption else [], - "mentions": _MENTION_RE.findall(caption) if caption else [], + "hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [], + "mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [], "url": url, "commentsCount": counts.get("comments"), "displayUrl": display_url, "videoUrl": video_url if is_video else None, "likesCount": counts.get("likes"), - "timestamp": node.get("uploadDate") or node.get("datePublished"), - "ownerUsername": _ld_author_username(node), + "timestamp": node.get("uploadDate") or node.get("datePublished") or og_meta.get("timestamp"), + "ownerUsername": _ld_author_username(node) or og_meta.get("ownerUsername"), + "ownerFullName": og_meta.get("ownerFullName"), } diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py index a4c781776..d6be49da4 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -116,12 +116,17 @@ def test_parse_post_prefers_ldjson(): def test_parse_post_falls_back_to_og_meta(): + # Anonymous /p/ pages carry no ld+json; everything is lifted from the og + # tags. og:description gives counts + username + date; og:title gives the + # clean caption + full name. Entities in the caption are deduped. html = """ + + content="1,234 likes, 56 comments - natgeo on January 2, 2024: "a caption #wow #wow @buzz"" /> """ item = parse_post(html, url=_POST_URL, shortcode="Cabc") @@ -130,6 +135,34 @@ def test_parse_post_falls_back_to_og_meta(): assert item["commentsCount"] == 56 assert item["displayUrl"] == "https://cdn/i.jpg" assert item["type"] == "Video" + assert item["ownerUsername"] == "natgeo" + assert item["ownerFullName"] == "Nat Geo" + assert item["timestamp"] == "2024-01-02" # og carries date only, no time + assert item["caption"] == "a caption #wow #wow @buzz" # unescaped, unwrapped + assert item["hashtags"] == ["wow"] # deduped, no counts-prefix pollution + assert item["mentions"] == ["buzz"] + + +def test_parse_post_og_degrades_without_crashing(): + # A shape we don't recognise (hidden likes / a non-English locale that + # slipped the en-US header) must yield a partial item with None fields, + # never an exception or a caption polluted with the counts/date prefix. + html = """ + + + + + + + """ + item = parse_post(html, url=_POST_URL, shortcode="Cabc") + assert item is not None # og:image present -> still emits + assert item["displayUrl"] == "https://cdn/i.jpg" + assert item["likesCount"] is None + assert item["commentsCount"] is None + assert item["ownerUsername"] is None + assert item["timestamp"] is None + assert item["caption"] is None # unrecognised prefix -> no pollution def test_parse_post_returns_none_without_surfaces(): From 457be1871c7e09c5a5b31cd962494ac37c7869ce Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:06:04 +0530 Subject: [PATCH 55/72] feat(instagram): improve parsing of Instagram media IDs and mentions - Enhanced the regular expression for mentions to prevent trailing punctuation from being included in handles. - Added support for extracting media IDs from deep-link meta tags in anonymous posts. - Updated unit tests to validate the new media ID extraction and ensure proper handling of mentions. --- .../proprietary/platforms/instagram/parsers.py | 16 ++++++++++++++-- .../unit/platforms/instagram/test_parsers.py | 12 +++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py index b4c2aeba3..f28ca58e0 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -23,7 +23,10 @@ from typing import Any _BASE = "https://www.instagram.com" _HASHTAG_RE = re.compile(r"#(\w+)") -_MENTION_RE = re.compile(r"@([\w.]+)") +# Instagram handles are letters/digits/period/underscore but never start or end +# with a period, so anchor both ends to alphanumerics/underscore — otherwise +# trailing sentence punctuation ("@hulu.") leaks into the handle. +_MENTION_RE = re.compile(r"@([A-Za-z0-9_](?:[A-Za-z0-9_.]*[A-Za-z0-9_])?)") _TYPE_MAP = { "GraphImage": "Image", "GraphVideo": "Video", @@ -205,6 +208,10 @@ _OG_OWNER_DATE_RE = re.compile( # og:title is the cleaner caption source (no counts/date prefix): the caption is # everything after " on Instagram: ". _OG_TITLE_RE = re.compile(r"^(.+?)\s+on Instagram:\s*(.*)$", re.DOTALL) +# The numeric media id (pk) rides in the App Link deep-link meta tags +# (al:ios:url / al:android:url = "instagram://media?id=") on anonymous pages, +# even though og:* and ld+json omit it. +_MEDIA_ID_RE = re.compile(r"instagram://media\?id=(\d+)") def _og_date_to_iso(value: str) -> str | None: @@ -370,8 +377,13 @@ def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> d else None ) or og.get("image") + media_id = node.get("identifier") if isinstance(node.get("identifier"), str) else None + if media_id is None: + id_match = _MEDIA_ID_RE.search(html) + media_id = id_match.group(1) if id_match else None + return { - "id": node.get("identifier") if isinstance(node.get("identifier"), str) else None, + "id": media_id, "type": "Video" if is_video else "Image", "shortCode": shortcode, "caption": caption, diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py index d6be49da4..73e043671 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -123,14 +123,16 @@ def test_parse_post_falls_back_to_og_meta(): + + content="Nat Geo on Instagram: "a caption #wow #wow @buzz."" /> + content="1,234 likes, 56 comments - natgeo on January 2, 2024: "a caption #wow #wow @buzz."" /> """ item = parse_post(html, url=_POST_URL, shortcode="Cabc") assert item is not None + assert item["id"] == "3938367641542741384" # numeric pk from al:ios:url meta assert item["likesCount"] == 1234 assert item["commentsCount"] == 56 assert item["displayUrl"] == "https://cdn/i.jpg" @@ -138,9 +140,9 @@ def test_parse_post_falls_back_to_og_meta(): assert item["ownerUsername"] == "natgeo" assert item["ownerFullName"] == "Nat Geo" assert item["timestamp"] == "2024-01-02" # og carries date only, no time - assert item["caption"] == "a caption #wow #wow @buzz" # unescaped, unwrapped - assert item["hashtags"] == ["wow"] # deduped, no counts-prefix pollution - assert item["mentions"] == ["buzz"] + assert item["caption"] == "a caption #wow #wow @buzz." # @ -> @, unescaped + assert item["hashtags"] == ["wow"] # deduped, no @-as-#064 pollution + assert item["mentions"] == ["buzz"] # trailing sentence dot stripped def test_parse_post_og_degrades_without_crashing(): From 5cac6612c39dcb918deee0b0c8369f96c629b2cf Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:17 +0530 Subject: [PATCH 56/72] refactor(instagram): update platform schemas and scraper for tagged media --- .../app/proprietary/platforms/instagram/schemas.py | 2 +- .../app/proprietary/platforms/instagram/scraper.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py index 76ad81d12..63c76da7b 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py @@ -19,7 +19,7 @@ from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field -InstagramResultsType = Literal["posts", "details", "reels", "mentions"] +InstagramResultsType = Literal["posts", "details", "reels"] InstagramSearchType = Literal["profile", "user"] diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index 381b40d5b..0aed53bb3 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -16,7 +16,7 @@ Google-backed handle discovery. Login-walled surfaces (hashtag/place feeds, comment threads, IG's native keyword search) are deliberately absent. Flows are selected by ``resultsType``: -- ``posts`` / ``reels`` / ``mentions`` -> media items (profile feed, or a single +- ``posts`` / ``reels`` -> media items (profile feed, or a single ``/p/``/``/reel/`` page, or discovery search) - ``details`` -> profile metadata (by URL or discovery search) @@ -394,7 +394,7 @@ async def iter_instagram( yield item return - # posts / reels / mentions -> media feeds, de-duped by id across targets. + # posts / reels -> media feeds, de-duped by id across targets. jobs = [ _media_flow( r, input_model=input_model, cutoff=cutoff, per_target=per_target From 27d22a9a2a081ac5ddd7482cfa4fe1dee0a591f5 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:20 +0530 Subject: [PATCH 57/72] refactor(instagram): drop mentions from scrape capability --- .../app/capabilities/instagram/scrape/__init__.py | 2 +- .../app/capabilities/instagram/scrape/definition.py | 4 ++-- .../app/capabilities/instagram/scrape/schemas.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/capabilities/instagram/scrape/__init__.py b/surfsense_backend/app/capabilities/instagram/scrape/__init__.py index de8b3c7c4..be8086485 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/__init__.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/__init__.py @@ -1,3 +1,3 @@ -"""``instagram.scrape`` verb: Instagram URLs / search terms → posts, reels, mentions.""" +"""``instagram.scrape`` verb: Instagram URLs / search terms → posts, reels.""" from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/instagram/scrape/definition.py b/surfsense_backend/app/capabilities/instagram/scrape/definition.py index 7b5d9769f..931ddfe01 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/definition.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/definition.py @@ -10,8 +10,8 @@ from app.capabilities.instagram.scrape.schemas import ScrapeInput, ScrapeOutput INSTAGRAM_SCRAPE = Capability( name="instagram.scrape", description=( - "Scrape public Instagram posts, reels, or mentions from profile/post/" - "reel URLs, or discover public profiles via search queries." + "Scrape public Instagram posts or reels from profile/post/reel URLs, " + "or discover public profiles via search queries." ), input_schema=ScrapeInput, output_schema=ScrapeOutput, diff --git a/surfsense_backend/app/capabilities/instagram/scrape/schemas.py b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py index 03e03e55d..f30078d9b 100644 --- a/surfsense_backend/app/capabilities/instagram/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/instagram/scrape/schemas.py @@ -43,9 +43,9 @@ class ScrapeInput(BaseModel): default="profile", description="Discovery kind (profile-only; hashtag/place are login-walled).", ) - result_type: Literal["posts", "reels", "mentions"] = Field( + result_type: Literal["posts", "reels"] = Field( default="posts", - description="Which feed to return. 'mentions' requires profile URLs.", + description="Which feed to return: 'posts' or 'reels'.", ) newer_than: str | None = Field( default=None, From ca7d23d33a6da5b92397a6567ecd88a34ab46b65 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:23 +0530 Subject: [PATCH 58/72] refactor(mcp): align Instagram scraper tool with capability changes --- .../mcp_server/features/scrapers/platforms/instagram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py index 9683e6581..28ad34e66 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/instagram.py @@ -13,7 +13,7 @@ from ....core.workspace_context import WorkspaceContext, WorkspaceParam from ..annotations import SCRAPE from ..capability import run_scraper -ResultType = Literal["posts", "reels", "mentions"] +ResultType = Literal["posts", "reels"] SearchType = Literal["profile", "user"] @@ -50,7 +50,7 @@ def register( ] = "profile", result_type: Annotated[ ResultType, - Field(description="Which feed to return. 'mentions' needs profile URLs."), + Field(description="Which feed to return: 'posts' or 'reels'."), ] = "posts", max_items: Annotated[ int, Field(ge=1, description="Maximum items to return across sources.") From c001f4b16e724e198f21e2fc67b59ed3afb9d2df Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:25 +0530 Subject: [PATCH 59/72] feat(subagent): update Instagram subagent system prompt --- .../subagents/builtins/instagram/system_prompt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md index 617a376b7..42713cc38 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/instagram/system_prompt.md @@ -12,7 +12,7 @@ Answer the delegated question from live Instagram data gathered with your verbs, -- Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts, reels, or mentions). Hashtag/place URLs are unsupported (login-walled). +- Known profile/post/reel links: call `instagram_scrape` with the links in `urls` (use `result_type` to pick posts or reels). Hashtag/place URLs are unsupported (login-walled). - Finding a profile on a topic: call `instagram_scrape` with `search_queries` (resolved to public profiles via Google; `search_type` is profile-only). - Profile metadata (follower counts, bio, post count): call `instagram_details`. - Batch multiple URLs (or queries) into one call rather than many single-item calls. From b75ab81261933f0372ced1f9662614af37aa51eb Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:30 +0530 Subject: [PATCH 60/72] docs(web): refresh Instagram connector docs --- surfsense_web/content/docs/connectors/native/instagram.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_web/content/docs/connectors/native/instagram.mdx b/surfsense_web/content/docs/connectors/native/instagram.mdx index 5881ac832..ad2b14ebe 100644 --- a/surfsense_web/content/docs/connectors/native/instagram.mdx +++ b/surfsense_web/content/docs/connectors/native/instagram.mdx @@ -22,7 +22,7 @@ Provide exactly one source: `urls` **or** `search_queries` (never both). | `urls` | — | Profile, post, or reel URLs or bare profile IDs (max 20 sources per call) | | `search_queries` | — | Discovery keywords resolved to public profiles via Google; each returns up to `max_per_target` items | | `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` | -| `result_type` | `posts` | Which feed to return: `posts`, `reels`, or `mentions` (`mentions` needs profile URLs) | +| `result_type` | `posts` | Which feed to return: `posts` or `reels` | | `newer_than` | — | Only return posts newer than this: `YYYY-MM-DD`, ISO timestamp, or relative (`2 months`), UTC | | `skip_pinned_posts` | `false` | Exclude pinned posts in posts mode | | `max_per_target` | `10` | Max results per URL or per discovered target | From 3e8f31152e36f099f90b75baaeb135d8f80d92df Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:34 +0530 Subject: [PATCH 61/72] feat(web): update Instagram marketing copy --- surfsense_web/lib/connectors-marketing/instagram.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx index 39a4a04cb..4c779b591 100644 --- a/surfsense_web/lib/connectors-marketing/instagram.tsx +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -88,7 +88,7 @@ export const instagram: ConnectorPageContent = { { title: "Influencer vetting and outreach", description: - "Verify a creator's follower count, post cadence, and real engagement from public data before you pay for a partnership, and surface who is already mentioning your brand.", + "Verify a creator's follower count, post cadence, and real engagement from public data before you pay for a partnership.", }, ], @@ -161,7 +161,7 @@ export const instagram: ConnectorPageContent = { name: "result_type", type: "string", defaultValue: '"posts"', - description: "Which feed to return: posts, reels, or mentions. 'mentions' requires profile URLs.", + description: "Which feed to return: posts or reels.", }, { name: "newer_than", From b1fe739308a85e5f8e93252afa4ac0dc58a97e94 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:36 +0530 Subject: [PATCH 62/72] docs(instagram): update platform scraper README --- .../app/proprietary/platforms/instagram/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/README.md b/surfsense_backend/app/proprietary/platforms/instagram/README.md index e4d7ca0f7..5bb64cdb2 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/README.md +++ b/surfsense_backend/app/proprietary/platforms/instagram/README.md @@ -26,7 +26,7 @@ Surfaces used: | Flow | Surface | Extractor | |---|---|---| | profile / details | `api/v1/users/web_profile_info/?username=…` (JSON) | `parse_profile` | -| profile feed (posts/reels/mentions) | the media embedded in the same profile JSON | `parse_media` | +| profile feed (posts/reels) | the media embedded in the same profile JSON | `parse_media` | | single post / reel | `/p//` (HTML: ld+json + og-meta) | `parse_post` | | profile discovery | Google `site:instagram.com ` | `resolve_url` | @@ -66,7 +66,7 @@ so the capability layer can map it to a `403 INSTAGRAM_ACCESS_BLOCKED`. comma-split queries) into targets and fans them out on a pool of warm proxy sessions (`fan_out`, 8-way). Each worker opens one sticky-IP session and warms `csrftoken`/`mid` once, reusing it across the sequential targets it pulls. -2. `resultsType` selects the flow: `posts`/`reels`/`mentions` → media items, +2. `resultsType` selects the flow: `posts`/`reels` → media items, `details` → profile metadata. Media items de-dupe by `id` across targets. - A **profile** target → `web_profile_info` JSON → `parse_media` over the embedded recent-media edges (feed) or `parse_profile` (details). From 69bf748c2ce693d4995dbe1bad557535ea0ec72a Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:25:39 +0530 Subject: [PATCH 63/72] chore(.gitignore): ignore Instagram tagged test fixture --- surfsense_backend/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/.gitignore b/surfsense_backend/.gitignore index 13c2d40e3..b8be5ae6c 100644 --- a/surfsense_backend/.gitignore +++ b/surfsense_backend/.gitignore @@ -18,4 +18,5 @@ celerybeat-schedule.bak app/templates/_generated/ /tests/unit/platforms/instagram/fixtures/post.json -/tests/unit/platforms/instagram/fixtures/profile.json \ No newline at end of file +/tests/unit/platforms/instagram/fixtures/profile.json +/tests/unit/platforms/instagram/fixtures/tagged.json \ No newline at end of file From cb4adab852dbfdb780e21a1106b2bf3b9647f569 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:10 +0530 Subject: [PATCH 64/72] feat(instagram): extract PolarisMedia relay JSON with carousel, tags, and location --- .../platforms/instagram/parsers.py | 372 ++++++++++++------ .../platforms/instagram/scraper.py | 7 +- 2 files changed, 264 insertions(+), 115 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py index f28ca58e0..5c4e8ea03 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -35,6 +35,10 @@ _TYPE_MAP = { "XDTGraphVideo": "Video", "XDTGraphSidecar": "Sidecar", } +# Mobile v1 ``media_type``: 1 = image, 2 = video, 8 = carousel/sidecar. Used by +# the single-post relay parser (the embedded PolarisMedia blob uses this int, not +# the GraphQL ``__typename`` the profile feed uses). +_MEDIA_TYPE = {1: "Image", 2: "Video", 8: "Sidecar"} def _int(value: Any) -> int | None: @@ -108,6 +112,80 @@ def _shortcode(node: dict[str, Any]) -> str | None: return None +def _user_ref(user: Any) -> dict[str, Any] | None: + """A trimmed public-user dict (tagged users / coauthor producers), or None. + + Normalizes the two anonymous dialects: the profile feed nests the handle + under ``edge_media_to_tagged_user...node.user`` / ``coauthor_producers`` while + the single-post relay blob uses ``usertags.in[].user`` — both carry the same + scalar user fields, so this trims them to one shape. + """ + if not isinstance(user, dict): + return None + ref = { + "username": user.get("username"), + "fullName": user.get("full_name"), + "id": user.get("id") or user.get("pk"), + "isVerified": user.get("is_verified"), + "profilePicUrl": user.get("profile_pic_url"), + } + return ref if ref["username"] or ref["id"] else None + + +def _iv2_url(iv2: Any) -> str | None: + """First candidate URL from a mobile ``image_versions2`` container, or None.""" + if isinstance(iv2, dict): + cands = iv2.get("candidates") + if isinstance(cands, list) and cands and isinstance(cands[0], dict): + url = cands[0].get("url") + return url if isinstance(url, str) else None + return None + + +def _location_ref(loc: Any) -> tuple[str | None, str | None]: + """``(name, id)`` from a location node, or ``(None, None)``.""" + if isinstance(loc, dict): + lid = loc.get("id") or loc.get("pk") + return loc.get("name"), (str(lid) if lid is not None else None) + return None, None + + +def _feed_child(node: dict[str, Any]) -> dict[str, Any]: + """Map a profile-feed ``edge_sidecar_to_children`` child to a childPost dict.""" + dims = node.get("dimensions") if isinstance(node.get("dimensions"), dict) else {} + is_video = bool(node.get("is_video")) + return { + "id": node.get("id"), + "shortCode": node.get("shortcode"), + "type": "Video" if is_video else "Image", + "displayUrl": node.get("display_url"), + "videoUrl": node.get("video_url") if is_video else None, + "alt": node.get("accessibility_caption"), + "dimensionsHeight": _int(dims.get("height")), + "dimensionsWidth": _int(dims.get("width")), + } + + +def _relay_child(node: dict[str, Any]) -> dict[str, Any]: + """Map a single-post relay ``carousel_media`` child to a childPost dict.""" + mt = node.get("media_type") + vv = node.get("video_versions") + video_url = ( + vv[0].get("url") if isinstance(vv, list) and vv and isinstance(vv[0], dict) else None + ) + is_video = mt == 2 or bool(video_url) + return { + "id": node.get("id"), + "shortCode": node.get("code"), + "type": _MEDIA_TYPE.get(mt) or ("Video" if is_video else "Image"), + "displayUrl": _iv2_url(node.get("image_versions2")) or node.get("display_uri"), + "videoUrl": video_url, + "alt": node.get("accessibility_caption"), + "dimensionsHeight": _int(node.get("original_height")), + "dimensionsWidth": _int(node.get("original_width")), + } + + def parse_media(node: dict[str, Any]) -> dict[str, Any]: """Map a raw timeline/feed media node to a flat media item dict.""" code = _shortcode(node) @@ -116,6 +194,16 @@ def parse_media(node: dict[str, Any]) -> dict[str, Any]: owner = node.get("owner") if isinstance(node.get("owner"), dict) else {} dims = node.get("dimensions") if isinstance(node.get("dimensions"), dict) else {} is_video = bool(node.get("is_video")) + children = _edges(node.get("edge_sidecar_to_children")) + tagged = [ + ref + for n in _edges(node.get("edge_media_to_tagged_user")) + if (ref := _user_ref(n.get("user"))) + ] + coauthors = [ + ref for c in (node.get("coauthor_producers") or []) if (ref := _user_ref(c)) + ] + loc_name, loc_id = _location_ref(node.get("location")) return { "id": node.get("id"), "type": _TYPE_MAP.get(typename) or ("Video" if is_video else "Image"), @@ -129,14 +217,25 @@ def parse_media(node: dict[str, Any]) -> dict[str, Any]: "dimensionsHeight": _int(dims.get("height")), "dimensionsWidth": _int(dims.get("width")), "displayUrl": node.get("display_url"), + "images": [c.get("display_url") for c in children if c.get("display_url")], + "childPosts": [_feed_child(c) for c in children], "videoUrl": node.get("video_url") if is_video else None, "alt": node.get("accessibility_caption"), "likesCount": _likes_count(node), "videoViewCount": _int(node.get("video_view_count")) if is_video else None, + "videoDuration": node.get("video_duration") if is_video else None, "timestamp": _utc_from_sec(node.get("taken_at_timestamp")), "ownerUsername": owner.get("username"), "ownerId": owner.get("id") or node.get("owner_id"), "ownerFullName": owner.get("full_name"), + "isPinned": bool(node.get("pinned_for_users")), + "productType": node.get("product_type"), + "paidPartnership": node.get("is_paid_partnership"), + "taggedUsers": tagged, + "coauthorProducers": coauthors, + "musicInfo": node.get("clips_music_attribution_info"), + "locationName": loc_name, + "locationId": loc_id, "isCommentsDisabled": node.get("comments_disabled"), } @@ -145,6 +244,9 @@ def parse_profile(user: dict[str, Any]) -> dict[str, Any]: """Map a raw ``web_profile_info`` ``data.user`` to a flat profile item dict.""" username = user.get("username") latest = [parse_media(n) for n in _edges(user.get("edge_owner_to_timeline_media"))] + related = [ + ref for n in _edges(user.get("edge_related_profiles")) if (ref := _user_ref(n)) + ] return { "detailKind": "profile", "id": user.get("id"), @@ -164,32 +266,34 @@ def parse_profile(user: dict[str, Any]) -> dict[str, Any]: "verified": user.get("is_verified"), "profilePicUrl": user.get("profile_pic_url"), "profilePicUrlHD": user.get("profile_pic_url_hd"), + "relatedProfiles": related, "latestPosts": latest, } # Anonymous single-post extraction (/p//, /reel//) -------- # -# Instagram serves logged-out visitors the post's public metadata inside the +# Instagram serves logged-out visitors the post's full metadata inside the # document itself, not via a JSON XHR (the ``?__a=1`` API 404s / login-walls for -# anonymous callers). Two durable, anonymous surfaces carry it: -# 1. ``', - re.IGNORECASE | re.DOTALL, +_APP_JSON_RE = re.compile( + r'', re.DOTALL ) _OG_RE = re.compile( r' int | None: return None -def _ldjson_blocks(html: str) -> list[dict[str, Any]]: - """Parse every ``application/ld+json`` script block into dicts.""" - out: list[dict[str, Any]] = [] - for raw in _LDJSON_RE.findall(html): - try: - data = json.loads(raw.strip()) - except (ValueError, TypeError): - continue - for node in data if isinstance(data, list) else [data]: - if isinstance(node, dict): - out.append(node) - return out - - def _og_tags(html: str) -> dict[str, str]: """Map ``og:`` -> content for the post document.""" return {k.lower(): v for k, v in _OG_RE.findall(html)} -def _ld_interaction(node: dict[str, Any]) -> dict[str, int]: - """Pull like/comment counts out of schema.org ``interactionStatistic``.""" - stats = node.get("interactionStatistic") - items = stats if isinstance(stats, list) else [stats] if stats else [] - out: dict[str, int] = {} - for stat in items: - if not isinstance(stat, dict): - continue - itype = str(stat.get("interactionType") or "") - count = _html_int(stat.get("userInteractionCount")) - if count is None: - continue - if "Like" in itype: - out["likes"] = count - elif "Comment" in itype: - out["comments"] = count - return out +def _find_media(root: Any, shortcode: str | None) -> dict[str, Any] | None: + """Depth-first search a JSON tree for the post's mobile-v1 media object. - -def _ld_author_username(node: dict[str, Any]) -> str | None: - """Owner handle from a schema.org ``author`` (alternateName / identifier).""" - author = node.get("author") - author = author[0] if isinstance(author, list) and author else author - if not isinstance(author, dict): - return None - for key in ("alternateName", "identifier", "name"): - val = author.get(key) - if isinstance(val, dict): - val = val.get("value") - if isinstance(val, str) and val.strip(): - return val.strip().lstrip("@") or None + Matches on ``code == shortcode`` (so a carousel *child* or a related post + can't be picked instead of the target) plus ``taken_at`` and an id, which + together uniquely identify the top-level ``PolarisMedia`` node. + """ + stack = [root] + while stack: + cur = stack.pop() + if isinstance(cur, dict): + if ( + cur.get("taken_at") is not None + and ("pk" in cur or "id" in cur) + and (shortcode is None or cur.get("code") == shortcode) + ): + return cur + stack.extend(cur.values()) + elif isinstance(cur, list): + stack.extend(cur) return None -def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> dict[str, Any] | None: +def _relay_media(html: str, shortcode: str | None) -> dict[str, Any] | None: + """Locate the embedded ``PolarisMedia`` object for this post, or ``None``. + + The logged-out media payload is inlined as one of ~40 ``application/json`` + script blocks. We only ``json.loads`` blocks that mention ``taken_at`` (and + the shortcode when known) so a single post fetch doesn't parse every blob. + """ + for raw in _APP_JSON_RE.findall(html): + if "taken_at" not in raw: + continue + if shortcode and shortcode not in raw: + continue + try: + data = json.loads(raw) + except (ValueError, TypeError): + continue + media = _find_media(data, shortcode) + if media is not None: + return media + return None + + +def _media_from_relay( + media: dict[str, Any], *, url: str, shortcode: str | None +) -> dict[str, Any]: + """Map an embedded mobile-v1 ``PolarisMedia`` object to a flat media item. + + Same output shape as :func:`parse_media` (so it flows through + ``InstagramMediaItem`` unchanged), sourced from the relay dialect + (``user``/``taken_at``/``usertags.in``/``carousel_media``/flat counts). + """ + mt = media.get("media_type") + cap = media.get("caption") + caption = ( + cap.get("text") if isinstance(cap, dict) else (cap if isinstance(cap, str) else None) + ) + carousel = media.get("carousel_media") + carousel = [c for c in carousel if isinstance(c, dict)] if isinstance(carousel, list) else [] + vv = media.get("video_versions") + video_url = ( + vv[0].get("url") if isinstance(vv, list) and vv and isinstance(vv[0], dict) else None + ) + is_video = mt == 2 or bool(video_url) + owner = media.get("user") if isinstance(media.get("user"), dict) else {} + tagged = [ + ref + for t in ((media.get("usertags") or {}).get("in") or []) + if isinstance(t, dict) and (ref := _user_ref(t.get("user"))) + ] + coauthors = [ + ref for c in (media.get("coauthor_producers") or []) if (ref := _user_ref(c)) + ] + loc_name, loc_id = _location_ref(media.get("location")) + # The relay ``id`` is ``POLARIS_``; strip the prefix so single-post ids + # match the numeric pk that og-fallback + the al:ios meta also yield. + ident = media.get("id") + if isinstance(ident, str) and ident.startswith("POLARIS_"): + ident = ident[len("POLARIS_") :] + pk = media.get("pk") + media_id = ident or (str(pk) if pk is not None else None) + return { + "id": media_id, + "type": _MEDIA_TYPE.get(mt) or ("Video" if is_video else "Image"), + "shortCode": media.get("code") or shortcode, + "caption": caption, + "hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [], + "mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [], + "url": url, + "commentsCount": _int(media.get("comment_count")), + "dimensionsHeight": _int(media.get("original_height")), + "dimensionsWidth": _int(media.get("original_width")), + "displayUrl": _iv2_url(media.get("image_versions2")) or media.get("display_uri"), + "images": [ + u + for c in carousel + if (u := _iv2_url(c.get("image_versions2")) or c.get("display_uri")) + ], + "childPosts": [_relay_child(c) for c in carousel], + "videoUrl": video_url, + "alt": media.get("accessibility_caption"), + "likesCount": _int(media.get("like_count")), + "videoViewCount": _int(media.get("view_count") or media.get("play_count")) + if is_video + else None, + "videoDuration": media.get("video_duration") if is_video else None, + "timestamp": _utc_from_sec(media.get("taken_at")), + "ownerUsername": owner.get("username"), + "ownerId": owner.get("id") or owner.get("pk"), + "ownerFullName": owner.get("full_name"), + "productType": media.get("product_type"), + "taggedUsers": tagged, + "coauthorProducers": coauthors, + "locationName": loc_name, + "locationId": loc_id, + } + + +def parse_post( + html: str | None, *, url: str, shortcode: str | None = None +) -> dict[str, Any] | None: """Map an anonymous ``/p//`` (or ``/reel/``) HTML page to a media dict. - Prefers the embedded schema.org ``ld+json`` block, falling back to Open Graph - meta tags for whatever it omits. Returns a dict shaped like - :func:`parse_media` (so it flows through ``InstagramMediaItem`` unchanged), or - ``None`` when the document carries neither surface (e.g. a login interstitial - slipped past the fetch-layer redirect check — the caller treats ``None`` as - "nothing to emit", never a silent success). + Prefers the embedded mobile-v1 ``PolarisMedia`` relay JSON (full fidelity), + falling back to the lossy Open Graph meta tags only if that blob is absent. + Returns a dict shaped like :func:`parse_media` (so it flows through + ``InstagramMediaItem`` unchanged), or ``None`` when the document carries + neither surface (e.g. a login interstitial slipped past the fetch-layer + redirect check — the caller treats ``None`` as "nothing to emit", never a + silent success). """ if not isinstance(html, str) or not html.strip(): return None - blocks = _ldjson_blocks(html) + + media = _relay_media(html, shortcode) + if media is not None: + return _media_from_relay(media, url=url, shortcode=shortcode) + + # Fallback: no embedded relay blob -> Open Graph meta only. og = _og_tags(html) - if not blocks and not og: + if not og: return None - - node = next( - (b for b in blocks if str(b.get("@type", "")).endswith(("Object", "Post"))), - blocks[0] if blocks else {}, - ) - # ld+json wins when present (richer, structured); og fills every gap. On - # anonymous /p/ pages ld+json is absent, so og_meta is the de-facto source. og_meta = _parse_og_meta(og) - - counts = _ld_interaction(node) - counts.setdefault("likes", og_meta.get("likes")) - counts.setdefault("comments", og_meta.get("comments")) - - caption = ( - node.get("articleBody") - or node.get("caption") - or node.get("description") - or og_meta.get("caption") - ) - caption = caption if isinstance(caption, str) else None - - video = node.get("video") - video = video[0] if isinstance(video, list) and video else video - video_url = ( - video.get("contentUrl") if isinstance(video, dict) else None - ) or og.get("video") + caption = og_meta.get("caption") + video_url = og.get("video") is_video = bool(video_url) or og.get("type") == "video.other" - - image = node.get("image") - image = image[0] if isinstance(image, list) and image else image - display_url = ( - image.get("url") if isinstance(image, dict) else image - if isinstance(image, str) - else None - ) or og.get("image") - - media_id = node.get("identifier") if isinstance(node.get("identifier"), str) else None - if media_id is None: - id_match = _MEDIA_ID_RE.search(html) - media_id = id_match.group(1) if id_match else None - + id_match = _MEDIA_ID_RE.search(html) return { - "id": media_id, + "id": id_match.group(1) if id_match else None, "type": "Video" if is_video else "Image", "shortCode": shortcode, "caption": caption, "hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [], "mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [], "url": url, - "commentsCount": counts.get("comments"), - "displayUrl": display_url, + "commentsCount": og_meta.get("comments"), + "displayUrl": og.get("image"), "videoUrl": video_url if is_video else None, - "likesCount": counts.get("likes"), - "timestamp": node.get("uploadDate") or node.get("datePublished") or og_meta.get("timestamp"), - "ownerUsername": _ld_author_username(node) or og_meta.get("ownerUsername"), + "likesCount": og_meta.get("likes"), + "timestamp": og_meta.get("timestamp"), + "ownerUsername": og_meta.get("ownerUsername"), "ownerFullName": og_meta.get("ownerFullName"), } diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py index 0aed53bb3..a4c22bde7 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py @@ -255,9 +255,10 @@ async def _media_flow( return if resolved.kind in ("post", "reel"): # Single-post extraction: the anonymous ``?__a=1`` JSON API 404s/login- - # walls, but the public /p// document embeds the post's og-meta + - # ld+json, which parse_post reads. Numeric-ID URLs can't be keyed this - # way (the page needs the shortCode), so they're skipped upstream. + # walls, but the public /p// document embeds the mobile-v1 + # PolarisMedia JSON (og-meta fallback), which parse_post reads. Numeric-ID + # URLs can't be keyed this way (the page needs the shortCode), so they're + # skipped upstream. if resolved.numeric_post_id: return html = await fetch_html(f"p/{resolved.value}/") From 452fe30aa4606f53687757001e163502cb385b06 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:15 +0530 Subject: [PATCH 65/72] refactor(instagram): drop login-walled comment fields and note search_type aliasing --- .../app/proprietary/platforms/instagram/schemas.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py index 63c76da7b..930a2acb9 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/schemas.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/schemas.py @@ -20,6 +20,9 @@ from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field InstagramResultsType = Literal["posts", "details", "reels"] +# Kept as distinct values for actor-spec parity, but under anonymous Google-backed +# discovery ``user`` and ``profile`` are aliases: both resolve to profile targets +# (IG's own hashtag/place/keyword search is login-walled). InstagramSearchType = Literal["profile", "user"] @@ -64,7 +67,12 @@ class _ItemBase(BaseModel): class InstagramMediaItem(_ItemBase): - """A post / reel / mention. One flat schema per the actor FAQ.""" + """A post or reel. One flat schema per the actor FAQ. + + ``firstComment``/``latestComments`` are intentionally absent: comment + *content* is login-walled (only the anonymous comment *count* is exposed, as + ``commentsCount``), so this scraper can never source them. + """ id: str | None = None type: Literal["Image", "Video", "Sidecar"] | None = None @@ -74,8 +82,6 @@ class InstagramMediaItem(_ItemBase): mentions: list[str] = Field(default_factory=list) url: str | None = None commentsCount: int | None = None - firstComment: str | None = None - latestComments: list[dict[str, Any]] = Field(default_factory=list) dimensionsHeight: int | None = None dimensionsWidth: int | None = None displayUrl: str | None = None From 4bc3d43b5639c24b5cac137c430ee2abeaa28c38 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:23 +0530 Subject: [PATCH 66/72] refactor(instagram): remove unused share-link redirect resolver --- .../proprietary/platforms/instagram/fetch.py | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py index e2b7c5818..41f692bf1 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/fetch.py @@ -297,25 +297,6 @@ async def _get_page(session: Any, url: str) -> Any: ) -async def resolve_redirect(url: str) -> str | None: - """Follow a ``share/`` short URL to its canonical target, or ``None``. - - ``share/`` links redirect to the real post/profile URL; the resolver records - the original as ``redirectedFromUrl``. Best-effort: returns the final URL - when the session exposes it, else ``None``. - """ - holder = _current_session.get() - if holder is None: - async with proxy_session(): - return await resolve_redirect(url) - with suppress(Exception): - page = await _get_page(holder.session, url) - final = getattr(page, "url", None) - if isinstance(final, str) and final and final != url: - return final - return None - - async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | None: """GET an Instagram web endpoint through a warmed session; parse JSON. @@ -331,7 +312,8 @@ async def fetch_html(path: str, params: dict[str, Any] | None = None) -> str | N Same warm/rotate/backoff resilience as :func:`fetch_json` (a login-wall redirect still raises :class:`InstagramAccessBlockedError`), but hands back the raw HTML body for the pages that embed their data in the document - (``/p//`` og-meta / ld+json) instead of a JSON XHR endpoint. + (``/p//`` embedded PolarisMedia JSON / og-meta) instead of a JSON + XHR endpoint. """ return await _fetch(path, params, _page_text) From d3c65a37b148e7e5c0317a3d9ecd77842416fd52 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:28 +0530 Subject: [PATCH 67/72] refactor(instagram): drop unused slug field and mark share links unsupported --- .../app/proprietary/platforms/instagram/url_resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py index aec46a678..08f5c6553 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/url_resolver.py @@ -13,7 +13,8 @@ Normalization rules (from the reference spec): - Numeric post-ID URLs cannot be single-post-extracted anonymously (the HTML page keys on the shortCode), so they resolve with ``numeric_post_id`` set and the media flow skips them. -- ``share/`` redirect resolution is handled at fetch time (network), not here. +- ``share/`` links are unsupported (they need a network redirect to resolve to a + canonical post/profile URL); pass the resolved ``/p/`` or profile URL instead. """ from __future__ import annotations @@ -40,7 +41,6 @@ class ResolvedUrl: kind: ResolvedKind value: str url: str - slug: str | None = None numeric_post_id: bool = False From 75c3addd2bcd5083e1f4f5a57a9780fc916d7603 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:33 +0530 Subject: [PATCH 68/72] test(instagram): cover PolarisMedia relay parsing and enriched media fields --- .../unit/platforms/instagram/test_parsers.py | 144 +++++++++++++++--- 1 file changed, 122 insertions(+), 22 deletions(-) diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py index 73e043671..8cba74b3a 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -60,6 +60,59 @@ def test_parse_media_marks_video_type(): assert item["videoViewCount"] == 99 +def test_parse_media_extracts_sidecar_tags_location_pinned(): + # The anonymous profile feed node carries far more than the core fields: + # sidecar children, tagged users, coauthors, location, product type and pin + # state — all populated here from the real GraphQL key shapes. + node = { + "id": "1", + "shortcode": "Cabc", + "__typename": "GraphSidecar", + "taken_at_timestamp": 1_704_164_645, + "edge_media_to_caption": _edge([{"text": "x #tag @me"}]), + "pinned_for_users": [{"id": "9"}], + "product_type": "feed", + "location": {"id": "55", "name": "Paris"}, + "coauthor_producers": [{"username": "co", "id": "7"}], + "edge_media_to_tagged_user": _edge( + [{"user": {"username": "tg", "id": "3"}, "x": 0.1, "y": 0.2}] + ), + "edge_sidecar_to_children": _edge( + [ + { + "id": "c1", + "shortcode": "s1", + "display_url": "https://cdn/1.jpg", + "dimensions": {"height": 10, "width": 20}, + }, + { + "id": "c2", + "shortcode": "s2", + "is_video": True, + "video_url": "https://cdn/2.mp4", + "display_url": "https://cdn/2.jpg", + }, + ] + ), + } + item = parse_media(node) + assert item["type"] == "Sidecar" + assert item["isPinned"] is True + assert item["productType"] == "feed" + assert item["locationName"] == "Paris" + assert item["locationId"] == "55" + assert item["taggedUsers"][0]["username"] == "tg" + assert item["coauthorProducers"][0]["username"] == "co" + assert item["images"] == ["https://cdn/1.jpg", "https://cdn/2.jpg"] + assert len(item["childPosts"]) == 2 + assert item["childPosts"][1]["type"] == "Video" + assert item["childPosts"][1]["videoUrl"] == "https://cdn/2.mp4" + + +def test_parse_media_unpinned_is_false(): + assert parse_media({"id": "1"})["isPinned"] is False + + def test_parse_profile_flattens_counts_and_latest_posts(): user = { "id": "9", @@ -71,6 +124,9 @@ def test_parse_profile_flattens_counts_and_latest_posts(): "count": 2, "edges": [{"node": {"id": "p1", "shortcode": "A"}}], }, + "edge_related_profiles": _edge( + [{"username": "similar1", "id": "11"}, {"username": "similar2", "id": "12"}] + ), } item = parse_profile(user) assert item["detailKind"] == "profile" @@ -79,40 +135,79 @@ def test_parse_profile_flattens_counts_and_latest_posts(): assert item["followsCount"] == 50 assert item["postsCount"] == 2 assert len(item["latestPosts"]) == 1 + assert [p["username"] for p in item["relatedProfiles"]] == ["similar1", "similar2"] _POST_URL = "https://www.instagram.com/p/Cabc/" -def test_parse_post_prefers_ldjson(): - html = """ - - - - """ +def test_parse_post_prefers_relay_json(): + # Anonymous /p/ pages inline the mobile-v1 PolarisMedia object in an + # application/json script. It's the full-fidelity source (carousel children, + # tagged users, coauthors, location, precise timestamp), preferred over og. + media = { + "pk": "3938367641542741384", + "id": "POLARIS_3938367641542741384", + "code": "Cabc", + "taken_at": 1_704_164_645, + "media_type": 8, + "product_type": "carousel_container", + "like_count": 4200, + "comment_count": 37, + "accessibility_caption": "alt text", + "caption": {"text": "sunset over #bali with @friend @friend"}, + "user": {"username": "natgeo", "full_name": "Nat Geo", "id": "9"}, + "image_versions2": {"candidates": [{"url": "https://cdn/i.jpg"}]}, + "carousel_media": [ + { + "id": "m1", + "code": "c1", + "media_type": 1, + "image_versions2": {"candidates": [{"url": "https://cdn/c1.jpg"}]}, + "original_height": 1080, + "original_width": 1080, + }, + { + "id": "m2", + "code": "c2", + "media_type": 2, + "video_versions": [{"url": "https://cdn/c2.mp4"}], + "image_versions2": {"candidates": [{"url": "https://cdn/c2.jpg"}]}, + }, + ], + "usertags": {"in": [{"position": [0.5, 0.5], "user": {"username": "tagged1", "id": "77"}}]}, + "coauthor_producers": [{"username": "coauthor1", "id": "88", "is_verified": True}], + "location": {"id": "123", "name": "Bali"}, + } + html = ( + '" + ) item = parse_post(html, url=_POST_URL, shortcode="Cabc") assert item is not None - assert item["type"] == "Video" + assert item["id"] == "3938367641542741384" # POLARIS_ prefix stripped + assert item["type"] == "Sidecar" # media_type 8 assert item["shortCode"] == "Cabc" assert item["url"] == _POST_URL - assert item["ownerUsername"] == "natgeo" - assert item["caption"] == "sunset over #bali with @friend" + assert item["caption"] == "sunset over #bali with @friend @friend" assert item["hashtags"] == ["bali"] - assert item["mentions"] == ["friend"] + assert item["mentions"] == ["friend"] # deduped assert item["likesCount"] == 4200 assert item["commentsCount"] == 37 - assert item["videoUrl"] == "https://cdn/v.mp4" - assert item["timestamp"] == "2024-01-02T03:04:05Z" + assert item["displayUrl"] == "https://cdn/i.jpg" + assert item["timestamp"].startswith("2024-01-02T") # real epoch -> ISO w/ time + assert item["ownerUsername"] == "natgeo" + assert item["ownerFullName"] == "Nat Geo" + assert item["images"] == ["https://cdn/c1.jpg", "https://cdn/c2.jpg"] + assert len(item["childPosts"]) == 2 + assert item["childPosts"][1]["type"] == "Video" + assert item["childPosts"][1]["videoUrl"] == "https://cdn/c2.mp4" + assert item["taggedUsers"][0]["username"] == "tagged1" + assert item["coauthorProducers"][0]["username"] == "coauthor1" + assert item["locationName"] == "Bali" + assert item["locationId"] == "123" + assert item["productType"] == "carousel_container" def test_parse_post_falls_back_to_og_meta(): @@ -196,3 +291,8 @@ def test_fixture_post_maps(): item = parse_post(raw["html"], url=raw["url"], shortcode=raw.get("shortcode")) assert item is not None, "captured /p/ HTML produced no media item" assert item["url"] == raw["url"] + # The relay blob (not og-meta) should drive extraction: numeric id + a + # precise timestamp with a time component (og-only would be date-only). + assert item["id"] and item["id"].isdigit() + assert item["ownerUsername"] + assert item["timestamp"] and "T" in item["timestamp"] From 819486ac464a88e08e516f0e333e37c1e63a6b70 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:38 +0530 Subject: [PATCH 69/72] docs(instagram): document PolarisMedia extraction and richer media fields --- .../proprietary/platforms/instagram/README.md | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/README.md b/surfsense_backend/app/proprietary/platforms/instagram/README.md index 5bb64cdb2..a3c6f01ba 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/README.md +++ b/surfsense_backend/app/proprietary/platforms/instagram/README.md @@ -27,9 +27,16 @@ Surfaces used: |---|---|---| | profile / details | `api/v1/users/web_profile_info/?username=…` (JSON) | `parse_profile` | | profile feed (posts/reels) | the media embedded in the same profile JSON | `parse_media` | -| single post / reel | `/p//` (HTML: ld+json + og-meta) | `parse_post` | +| single post / reel | `/p//` (embedded mobile-v1 `PolarisMedia` JSON, og-meta fallback) | `parse_post` | | profile discovery | Google `site:instagram.com ` | `resolve_url` | +All of these are richer than the core fields: the feed node and the single-post +relay blob both carry carousel children (`images`/`childPosts`), tagged users, +coauthor producers, location, product type, and pin state; `web_profile_info` +also carries related profiles. Comment **content** stays login-walled — only the +anonymous comment **count** (`commentsCount`) is exposed, so `firstComment` / +`latestComments` are intentionally absent from the item schema. + **Why anonymous-only is a hard constraint.** Live logged-out probes show that Instagram walls the interesting endpoints for anyone without a `sessionid` account cookie: `api/v1/tags/web_info/`, `api/v1/locations/web_info/`, the @@ -57,7 +64,7 @@ so the capability layer can map it to a `403 INSTAGRAM_ACCESS_BLOCKED`. | `schemas.py` | `InstagramScrapeInput` (`extra="allow"`, no auth fields) + optional-field item models (`InstagramMediaItem`, `InstagramProfile`) each with `to_output()`. | | `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (csrftoken/mid) + `fetch_json` (JSON) / `fetch_html` (HTML) sharing one resilient `_fetch(path, params, extract)` loop. | | `url_resolver.py` | Classify an Instagram URL → `profile`/`post`/`reel`; non-Instagram (and hashtag/place) → `None`. Strips `_u/`, `profilecard/`; story → profile. | -| `parsers.py` | Pure mapping (`parse_media`, `parse_profile`, `parse_post` [ld+json/og], `_edges`). I/O-free. | +| `parsers.py` | Pure mapping (`parse_media`, `parse_profile`, `parse_post` [relay `PolarisMedia` JSON, og-meta fallback], `_edges`). I/O-free. | | `scraper.py` | Orchestrator: `_media_flow`/`_details_flow`/`_discover` (+ `_discover_via_google`), `_targets`, `fan_out`, `iter_instagram`, `scrape_instagram`. | ## How it works @@ -71,8 +78,9 @@ so the capability layer can map it to a `403 INSTAGRAM_ACCESS_BLOCKED`. - A **profile** target → `web_profile_info` JSON → `parse_media` over the embedded recent-media edges (feed) or `parse_profile` (details). - A **post/reel** target → `fetch_html("p//")` → `parse_post`, which - reads the page's `application/ld+json` (preferred) and Open Graph meta - (fallback). Numeric-ID post URLs are skipped (the page keys on the shortCode). + reads the embedded mobile-v1 `PolarisMedia` JSON (full fidelity) and falls + back to Open Graph meta only if that blob is absent. Numeric-ID post URLs are + skipped (the page keys on the shortCode). 3. `fetch_json` / `fetch_html` warm the session on first use, rotate the IP + re-warm on 401/403, back off on 429, return `None` on 404, and raise `InstagramAccessBlockedError` on a `/accounts/login/` redirect. @@ -103,12 +111,12 @@ Caveats: the `InstagramAccessBlockedError` path, not a bug. - `likesCount` is frequently withheld on anonymous responses (surfaces as `-1` or absent upstream); treat it as best-effort. -- **Single-post extraction** reads whatever the public `/p/` document embeds - (ld+json + og-meta). If Instagram strips both for a given post (private, taken - down, or a login interstitial), `parse_post` returns `None` — an honest empty, - never a fabricated item. ponytail: the embedded-blob shapes can drift; a live - probe that dumps the raw HTML pins them (see Testing) and any change is contained - to `parse_post`. +- **Single-post extraction** reads the mobile-v1 `PolarisMedia` object embedded in + the public `/p/` document (og-meta is a lossy fallback). If Instagram strips both + for a given post (private, taken down, or a login interstitial), `parse_post` + returns `None` — an honest empty, never a fabricated item. ponytail: the + embedded-blob shape can drift; a live probe that dumps the raw HTML pins it (see + Testing) and any change is contained to `_find_media` / `parse_post`. - The `$3.50 / 1k items` default meter assumes the proxy-bytes-per-item measured on the reference targets; re-measure with the scale harness before high-volume use. @@ -116,14 +124,15 @@ Caveats: - Offline unit tests: `tests/unit/platforms/instagram/` — `test_skeleton.py` (schema + URL resolver), `test_parsers.py` (mapping incl. `parse_post` - ld+json/og shapes; fixture-pinned tests skip when the fixture is absent), + relay-JSON/og shapes; fixture-pinned tests skip when the fixture is absent), `test_discovery.py` (Google-backed profile discovery with a fake `scrape_serps`), `test_fetch_resilience.py` (warm / rotate / backoff loop + fan-out with fake sessions, no network), `test_budget.py` (fair-share caps + de-dup). - Stress / accuracy harness (live, needs network + residential proxy): `scripts/stress/stress_instagram_scraper.py` — `--mode live-discovery` (profile discovery accuracy), `--mode probe-post` (dumps a real anonymous `/p/` payload - to `fixtures/post.json` and shows what `parse_post` extracted), and + to `fixtures/post.json` and shows what `parse_post` extracted), `--mode + probe-mentions` (settles that the tagged/`mentions` feed is login-walled), and `--mode accuracy` (field coverage across the profile + single-post flows). ```bash From 42f9f35ec7ca1ab2945f5cc2d9a80baf6514e29d Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:45 +0530 Subject: [PATCH 70/72] docs(web): document new media fields and search_type aliasing --- surfsense_web/content/docs/connectors/native/instagram.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/surfsense_web/content/docs/connectors/native/instagram.mdx b/surfsense_web/content/docs/connectors/native/instagram.mdx index ad2b14ebe..54b8781e0 100644 --- a/surfsense_web/content/docs/connectors/native/instagram.mdx +++ b/surfsense_web/content/docs/connectors/native/instagram.mdx @@ -13,7 +13,7 @@ Instagram login-walls hashtag feeds, place feeds, and comment threads for anonym POST /api/v1/workspaces/{workspace_id}/scrapers/instagram/scrape ``` -Give it Instagram URLs (profile, post `/p/`, or reel `/reel/`) or search queries. Returns structured media items — caption, hashtags, mentions, like and comment counts, media URLs, and owner. +Give it Instagram URLs (profile, post `/p/`, or reel `/reel/`) or search queries. Returns structured media items — caption, hashtags, mentions, like and comment counts, media URLs, owner, carousel children, tagged users, coauthors, and location. Comment *content* is login-walled, so only the comment *count* is returned (never comment text). Provide exactly one source: `urls` **or** `search_queries` (never both). @@ -21,7 +21,7 @@ Provide exactly one source: `urls` **or** `search_queries` (never both). |-------|---------|-------------| | `urls` | — | Profile, post, or reel URLs or bare profile IDs (max 20 sources per call) | | `search_queries` | — | Discovery keywords resolved to public profiles via Google; each returns up to `max_per_target` items | -| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` | +| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` (aliases — anonymous discovery resolves both to profiles) | | `result_type` | `posts` | Which feed to return: `posts` or `reels` | | `newer_than` | — | Only return posts newer than this: `YYYY-MM-DD`, ISO timestamp, or relative (`2 months`), UTC | | `skip_pinned_posts` | `false` | Exclude pinned posts in posts mode | @@ -53,7 +53,7 @@ Give it profile URLs (or search queries) and get profile metadata back: follower |-------|---------|-------------| | `urls` | — | Profile URLs or bare profile IDs (max 20) | | `search_queries` | — | Terms to discover public profiles for (provide these OR urls) | -| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` | +| `search_type` | `profile` | What to discover from `search_queries`: `profile` or `user` (aliases — anonymous discovery resolves both to profiles) | | `max_items` | `10` | Max detail items to return | Billing is per returned item. From ec3f47abf88a254640e49297ba7d1f65b6845cee Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:44:51 +0530 Subject: [PATCH 71/72] feat(web): surface carousel, tagged users, and location in connector marketing --- .../lib/connectors-marketing/instagram.tsx | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx index 4c779b591..d22400b7a 100644 --- a/surfsense_web/lib/connectors-marketing/instagram.tsx +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -232,6 +232,21 @@ export const instagram: ConnectorPageContent = { type: "string", description: "ISO timestamp for when the post was published.", }, + { + name: "images / childPosts", + type: "string[] / object[]", + description: "Carousel (sidecar) children: each child's media URL and metadata.", + }, + { + name: "taggedUsers / coauthorProducers", + type: "object[]", + description: "Users tagged in the media and any co-authors credited on it.", + }, + { + name: "locationName / locationId / productType / isPinned", + type: "string / boolean", + description: "Location tag, product type (feed/clips), and whether the post is pinned.", + }, ], }, @@ -254,7 +269,7 @@ export const instagram: ConnectorPageContent = { { question: "Can I scrape hashtags, places, or comments?", answer: - "No. Instagram login-walls hashtag feeds, place feeds, and comment threads for logged-out visitors, so SurfSense does not offer them. The API focuses on what is reliably public and anonymous: profiles, posts, and reels.", + "No. Instagram login-walls hashtag feeds, place feeds, and comment threads for logged-out visitors, so SurfSense does not offer them. You still get each post's comment count, just not the comment text. The API focuses on what is reliably public and anonymous: profiles, posts, and reels.", }, ], From cb256729a3c5b23b9d6fc4673bc48cd8eabc0461 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 04:38:42 +0530 Subject: [PATCH 72/72] chore: Enhance SurfSense MCP server documentation and toolset by adding TikTok scraping capabilities, including comments, user search, and trending features. Updated README and documentation to reflect the addition of 24 native tools. --- surfsense_mcp/README.md | 4 +++- surfsense_web/app/(home)/mcp-server/page.tsx | 3 +++ surfsense_web/content/docs/how-to/mcp-server.mdx | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/surfsense_mcp/README.md b/surfsense_mcp/README.md index 5bac5f8b2..95ac293d9 100644 --- a/surfsense_mcp/README.md +++ b/surfsense_mcp/README.md @@ -21,7 +21,9 @@ Connect it two ways: **Scrapers (all platforms)** - `surfsense_web_crawl`, `surfsense_google_search`, `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, - `surfsense_tiktok_scrape`, + `surfsense_instagram_scrape`, `surfsense_instagram_details`, + `surfsense_tiktok_scrape`, `surfsense_tiktok_comments`, + `surfsense_tiktok_user_search`, `surfsense_tiktok_trending`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews` - `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` — retrieve past results in full (useful when a large result was truncated inline) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index 3d00c44c9..ce4caf992 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -96,6 +96,9 @@ const TOOL_GROUPS = [ "surfsense_instagram_scrape", "surfsense_instagram_details", "surfsense_tiktok_scrape", + "surfsense_tiktok_comments", + "surfsense_tiktok_user_search", + "surfsense_tiktok_trending", "surfsense_google_maps_scrape", "surfsense_google_maps_reviews", "surfsense_google_search", diff --git a/surfsense_web/content/docs/how-to/mcp-server.mdx b/surfsense_web/content/docs/how-to/mcp-server.mdx index 782476d07..71d55ea6c 100644 --- a/surfsense_web/content/docs/how-to/mcp-server.mdx +++ b/surfsense_web/content/docs/how-to/mcp-server.mdx @@ -7,7 +7,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; # SurfSense MCP Server -The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 21 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. +The SurfSense MCP server exposes your workspace to any [Model Context Protocol](https://modelcontextprotocol.io/) client. Your agent gets 24 native, typed tools: every scraper (Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, web crawl), full knowledge-base access (search, read, add, upload, update, delete), and a workspace selector. Connect it two ways: the **hosted** server at `https://mcp.surfsense.com/mcp` (nothing to install — just an API key), or run it yourself over **stdio** against any SurfSense backend, cloud or self-hosted. @@ -133,7 +133,7 @@ Add to `~/.cursor/mcp.json` (global — keeps the key out of your repo) or a pro } ``` -Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 21 tools should appear with a green dot. +Then open **Cursor Settings → MCP** and refresh the `surfsense` server; its 24 tools should appear with a green dot. @@ -257,14 +257,14 @@ For self-host (stdio), all settings are environment variables passed by the clie - **401 errors** — the API key is wrong or expired; create a new one. - **403 errors** — API access is disabled for the workspace; toggle **API key access** on under **API Playground → API Keys**. - **"Could not reach SurfSense"** — the backend isn't running or `SURFSENSE_BASE_URL` is wrong. -- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 21 tools register without needing a backend. +- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 24 tools register without needing a backend. ## Tools reference | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_tiktok_comments`, `surfsense_tiktok_user_search`, `surfsense_tiktok_trending`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**.