From d41ccb7edd70a1774f3c6c307bfebc7903da1345 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:56:29 +0530 Subject: [PATCH] feat(instagram): enhance Open Graph parsing for anonymous posts - Added support for extracting likes, comments, username, timestamp, and caption from Open Graph meta tags. - Implemented fallback mechanisms to ensure graceful degradation when expected data is missing. - Updated unit tests to validate new parsing logic and ensure robustness against unrecognized formats. --- .../platforms/instagram/parsers.py | 85 ++++++++++++++++--- .../unit/platforms/instagram/test_parsers.py | 35 +++++++- 2 files changed, 108 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py index 55884537d..b4c2aeba3 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -15,6 +15,7 @@ parity is additive. from __future__ import annotations +import html import json import re from datetime import UTC, datetime @@ -185,10 +186,70 @@ _LDJSON_RE = re.compile( _OG_RE = re.compile( r' on Instagram: ". +_OG_TITLE_RE = re.compile(r"^(.+?)\s+on Instagram:\s*(.*)$", re.DOTALL) + + +def _og_date_to_iso(value: str) -> str | None: + """``"July 9, 2026"`` -> ``"2026-07-09"`` (date-only; og carries no time).""" + try: + return datetime.strptime(value, "%B %d, %Y").replace(tzinfo=UTC).date().isoformat() + except ValueError: + return None + + +def _clean_caption(raw: str) -> str | None: + """HTML-unescape and unwrap the surrounding quotes off an og caption preview.""" + return html.unescape(raw).strip().strip('"').strip() or None + + +def _parse_og_meta(og: dict[str, str]) -> dict[str, Any]: + """Lift post fields out of Instagram's Open Graph tags (see module notes above). + + Caption + full name come from ``og:title``; counts + username + date from + ``og:description``. Every field is optional and independently guarded, so a + shape we don't recognise yields a partial (or empty) dict instead of raising. + """ + out: dict[str, Any] = {} + desc = og.get("description", "") + title = og.get("title", "") + + counts = _OG_COUNTS_RE.search(desc) + if counts: + out["likes"] = _html_int(counts.group(1)) + out["comments"] = _html_int(counts.group(2)) + + owner_date = _OG_OWNER_DATE_RE.search(desc) + if owner_date: + out["ownerUsername"] = owner_date.group(1).strip().lstrip("@") or None + out["timestamp"] = _og_date_to_iso(owner_date.group(2)) + + tm = _OG_TITLE_RE.match(title) + if tm: + out["ownerFullName"] = tm.group(1).strip() or None + out["caption"] = _clean_caption(tm.group(2)) + elif owner_date: + # No usable og:title: fall back to the caption after og:description's + # date prefix — still clean (the counts/username/date are stripped). + out["caption"] = _clean_caption(desc[owner_date.end():]) + return out def _html_int(value: Any) -> int | None: @@ -278,18 +339,19 @@ def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> d (b for b in blocks if str(b.get("@type", "")).endswith(("Object", "Post"))), blocks[0] if blocks else {}, ) + # ld+json wins when present (richer, structured); og fills every gap. On + # anonymous /p/ pages ld+json is absent, so og_meta is the de-facto source. + og_meta = _parse_og_meta(og) + counts = _ld_interaction(node) - if "likes" not in counts or "comments" not in counts: - m = _OG_COUNTS_RE.search(og.get("description", "")) - if m: - counts.setdefault("likes", _html_int(m.group(1)) or 0) - counts.setdefault("comments", _html_int(m.group(2)) or 0) + counts.setdefault("likes", og_meta.get("likes")) + counts.setdefault("comments", og_meta.get("comments")) caption = ( node.get("articleBody") or node.get("caption") or node.get("description") - or og.get("description") + or og_meta.get("caption") ) caption = caption if isinstance(caption, str) else None @@ -313,13 +375,14 @@ def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> d "type": "Video" if is_video else "Image", "shortCode": shortcode, "caption": caption, - "hashtags": _HASHTAG_RE.findall(caption) if caption else [], - "mentions": _MENTION_RE.findall(caption) if caption else [], + "hashtags": list(dict.fromkeys(_HASHTAG_RE.findall(caption))) if caption else [], + "mentions": list(dict.fromkeys(_MENTION_RE.findall(caption))) if caption else [], "url": url, "commentsCount": counts.get("comments"), "displayUrl": display_url, "videoUrl": video_url if is_video else None, "likesCount": counts.get("likes"), - "timestamp": node.get("uploadDate") or node.get("datePublished"), - "ownerUsername": _ld_author_username(node), + "timestamp": node.get("uploadDate") or node.get("datePublished") or og_meta.get("timestamp"), + "ownerUsername": _ld_author_username(node) or og_meta.get("ownerUsername"), + "ownerFullName": og_meta.get("ownerFullName"), } diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py index a4c781776..d6be49da4 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -116,12 +116,17 @@ def test_parse_post_prefers_ldjson(): def test_parse_post_falls_back_to_og_meta(): + # Anonymous /p/ pages carry no ld+json; everything is lifted from the og + # tags. og:description gives counts + username + date; og:title gives the + # clean caption + full name. Entities in the caption are deduped. html = """ + + content="1,234 likes, 56 comments - natgeo on January 2, 2024: "a caption #wow #wow @buzz"" /> """ item = parse_post(html, url=_POST_URL, shortcode="Cabc") @@ -130,6 +135,34 @@ def test_parse_post_falls_back_to_og_meta(): assert item["commentsCount"] == 56 assert item["displayUrl"] == "https://cdn/i.jpg" assert item["type"] == "Video" + assert item["ownerUsername"] == "natgeo" + assert item["ownerFullName"] == "Nat Geo" + assert item["timestamp"] == "2024-01-02" # og carries date only, no time + assert item["caption"] == "a caption #wow #wow @buzz" # unescaped, unwrapped + assert item["hashtags"] == ["wow"] # deduped, no counts-prefix pollution + assert item["mentions"] == ["buzz"] + + +def test_parse_post_og_degrades_without_crashing(): + # A shape we don't recognise (hidden likes / a non-English locale that + # slipped the en-US header) must yield a partial item with None fields, + # never an exception or a caption polluted with the counts/date prefix. + html = """ + + + + + + + """ + item = parse_post(html, url=_POST_URL, shortcode="Cabc") + assert item is not None # og:image present -> still emits + assert item["displayUrl"] == "https://cdn/i.jpg" + assert item["likesCount"] is None + assert item["commentsCount"] is None + assert item["ownerUsername"] is None + assert item["timestamp"] is None + assert item["caption"] is None # unrecognised prefix -> no pollution def test_parse_post_returns_none_without_surfaces():