diff --git a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py index b4c2aeba3..f28ca58e0 100644 --- a/surfsense_backend/app/proprietary/platforms/instagram/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/instagram/parsers.py @@ -23,7 +23,10 @@ from typing import Any _BASE = "https://www.instagram.com" _HASHTAG_RE = re.compile(r"#(\w+)") -_MENTION_RE = re.compile(r"@([\w.]+)") +# Instagram handles are letters/digits/period/underscore but never start or end +# with a period, so anchor both ends to alphanumerics/underscore — otherwise +# trailing sentence punctuation ("@hulu.") leaks into the handle. +_MENTION_RE = re.compile(r"@([A-Za-z0-9_](?:[A-Za-z0-9_.]*[A-Za-z0-9_])?)") _TYPE_MAP = { "GraphImage": "Image", "GraphVideo": "Video", @@ -205,6 +208,10 @@ _OG_OWNER_DATE_RE = re.compile( # og:title is the cleaner caption source (no counts/date prefix): the caption is # everything after " on Instagram: ". _OG_TITLE_RE = re.compile(r"^(.+?)\s+on Instagram:\s*(.*)$", re.DOTALL) +# The numeric media id (pk) rides in the App Link deep-link meta tags +# (al:ios:url / al:android:url = "instagram://media?id=") on anonymous pages, +# even though og:* and ld+json omit it. +_MEDIA_ID_RE = re.compile(r"instagram://media\?id=(\d+)") def _og_date_to_iso(value: str) -> str | None: @@ -370,8 +377,13 @@ def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> d else None ) or og.get("image") + media_id = node.get("identifier") if isinstance(node.get("identifier"), str) else None + if media_id is None: + id_match = _MEDIA_ID_RE.search(html) + media_id = id_match.group(1) if id_match else None + return { - "id": node.get("identifier") if isinstance(node.get("identifier"), str) else None, + "id": media_id, "type": "Video" if is_video else "Image", "shortCode": shortcode, "caption": caption, diff --git a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py index d6be49da4..73e043671 100644 --- a/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/instagram/test_parsers.py @@ -123,14 +123,16 @@ def test_parse_post_falls_back_to_og_meta(): + + content="Nat Geo on Instagram: "a caption #wow #wow @buzz."" /> + content="1,234 likes, 56 comments - natgeo on January 2, 2024: "a caption #wow #wow @buzz."" /> """ item = parse_post(html, url=_POST_URL, shortcode="Cabc") assert item is not None + assert item["id"] == "3938367641542741384" # numeric pk from al:ios:url meta assert item["likesCount"] == 1234 assert item["commentsCount"] == 56 assert item["displayUrl"] == "https://cdn/i.jpg" @@ -138,9 +140,9 @@ def test_parse_post_falls_back_to_og_meta(): assert item["ownerUsername"] == "natgeo" assert item["ownerFullName"] == "Nat Geo" assert item["timestamp"] == "2024-01-02" # og carries date only, no time - assert item["caption"] == "a caption #wow #wow @buzz" # unescaped, unwrapped - assert item["hashtags"] == ["wow"] # deduped, no counts-prefix pollution - assert item["mentions"] == ["buzz"] + assert item["caption"] == "a caption #wow #wow @buzz." # @ -> @, unescaped + assert item["hashtags"] == ["wow"] # deduped, no @-as-#064 pollution + assert item["mentions"] == ["buzz"] # trailing sentence dot stripped def test_parse_post_og_degrades_without_crashing():