apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-16 23:01:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(instagram): improve parsing of Instagram media IDs and mentions

Browse source 
- Enhanced the regular expression for mentions to prevent trailing punctuation from being included in handles.
- Added support for extracting media IDs from deep-link meta tags in anonymous posts.
- Updated unit tests to validate the new media ID extraction and ensure proper handling of mentions.
This commit is contained in:
Anish Sarkar 2026-07-11 03:06:04 +05:30
parent d41ccb7edd
commit 457be1871c
2 changed files with 21 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

16
surfsense_backend/app/proprietary/platforms/instagram/parsers.py
View file

@ -23,7 +23,10 @@ from typing import Any
_BASE = "https://www.instagram.com"
_HASHTAG_RE = re.compile(r"#(\w+)")
_MENTION_RE = re.compile(r"@([\w.]+)")
# Instagram handles are letters/digits/period/underscore but never start or end
# with a period, so anchor both ends to alphanumerics/underscore — otherwise
# trailing sentence punctuation ("@hulu.") leaks into the handle.
_MENTION_RE = re.compile(r"@([A-Za-z0-9_](?:[A-Za-z0-9_.]*[A-Za-z0-9_])?)")
_TYPE_MAP = {
"GraphImage": "Image",
"GraphVideo": "Video",
@ -205,6 +208,10 @@ _OG_OWNER_DATE_RE = re.compile(
# og:title is the cleaner caption source (no counts/date prefix): the caption is
# everything after "<fullName> on Instagram: ".
_OG_TITLE_RE = re.compile(r"^(.+?)\s+on Instagram:\s*(.*)$", re.DOTALL)
# The numeric media id (pk) rides in the App Link deep-link meta tags
# (al:ios:url / al:android:url = "instagram://media?id=<pk>") on anonymous pages,
# even though og:* and ld+json omit it.
_MEDIA_ID_RE = re.compile(r"instagram://media\?id=(\d+)")
def _og_date_to_iso(value: str) -> str | None:
@ -370,8 +377,13 @@ def parse_post(html: str | None, *, url: str, shortcode: str | None = None) -> d
else None
) or og.get("image")
media_id = node.get("identifier") if isinstance(node.get("identifier"), str) else None
if media_id is None:
id_match = _MEDIA_ID_RE.search(html)
media_id = id_match.group(1) if id_match else None
return {
"id": node.get("identifier") if isinstance(node.get("identifier"), str) else None,
"id": media_id,
"type": "Video" if is_video else "Image",
"shortCode": shortcode,
"caption": caption,

12
surfsense_backend/tests/unit/platforms/instagram/test_parsers.py
View file

@ -123,14 +123,16 @@ def test_parse_post_falls_back_to_og_meta():
<html><head>
<meta property="og:type" content="video.other" />
<meta property="og:image" content="https://cdn/i.jpg" />
<meta property="al:ios:url" content="instagram://media?id=3938367641542741384" />
<meta property="og:title"
content="Nat Geo on Instagram: &quot;a caption #wow #wow @buzz&quot;" />
content="Nat Geo on Instagram: &quot;a caption #wow #wow &#064;buzz.&quot;" />
<meta property="og:description"
content="1,234 likes, 56 comments - natgeo on January 2, 2024: &quot;a caption #wow #wow @buzz&quot;" />
content="1,234 likes, 56 comments - natgeo on January 2, 2024: &quot;a caption #wow #wow &#064;buzz.&quot;" />
</head></html>
"""
item = parse_post(html, url=_POST_URL, shortcode="Cabc")
assert item is not None
assert item["id"] == "3938367641542741384" # numeric pk from al:ios:url meta
assert item["likesCount"] == 1234
assert item["commentsCount"] == 56
assert item["displayUrl"] == "https://cdn/i.jpg"
@ -138,9 +140,9 @@ def test_parse_post_falls_back_to_og_meta():
assert item["ownerUsername"] == "natgeo"
assert item["ownerFullName"] == "Nat Geo"
assert item["timestamp"] == "2024-01-02" # og carries date only, no time
assert item["caption"] == "a caption #wow #wow @buzz" # unescaped, unwrapped
assert item["hashtags"] == ["wow"] # deduped, no counts-prefix pollution
assert item["mentions"] == ["buzz"]
assert item["caption"] == "a caption #wow #wow @buzz." # &#064; -> @, unescaped
assert item["hashtags"] == ["wow"] # deduped, no &#064;-as-#064 pollution
assert item["mentions"] == ["buzz"] # trailing sentence dot stripped
def test_parse_post_og_degrades_without_crashing():