From e8f8eeab270b65c5b51fadc548df6065f8a96942 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Thu, 9 Jul 2026 19:08:18 +0530
Subject: [PATCH] fix(instagram): improve error handling for Instagram access
 blocks in scraper

Updated the fan_out function to handle InstagramAccessBlockedError more gracefully. Instead of raising the error directly, it now puts the error into the results queue to prevent deadlocks. This change ensures that the consumer can handle access block scenarios without interrupting the processing of other jobs.
---
 .../platforms/instagram/scraper.py            | 20 +++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py
index 46569f4dd..34f6658a3 100644
--- a/surfsense_backend/app/proprietary/platforms/instagram/scraper.py
+++ b/surfsense_backend/app/proprietary/platforms/instagram/scraper.py
@@ -132,7 +132,13 @@ async def fan_out(
     job_queue: asyncio.Queue[AsyncIterator[dict[str, Any]]] = asyncio.Queue()
     for job in jobs:
         job_queue.put_nowait(job)
-    results: asyncio.Queue[list[dict[str, Any]]] = asyncio.Queue()
+    # A batch of items on success, or a hard-block exception to re-raise on the
+    # consumer side. The consumer reads exactly one entry per job, so a worker
+    # MUST put something for every job it pulls — raising instead would strand
+    # the error on a dead task and deadlock the consumer on ``results.get()``.
+    results: asyncio.Queue[list[dict[str, Any]] | InstagramAccessBlockedError] = (
+        asyncio.Queue()
+    )
 
     async def worker() -> None:
         holder = None
@@ -153,8 +159,11 @@ async def fan_out(
                             items = [item async for item in job]
                     else:
                         items = [item async for item in job]
-                except InstagramAccessBlockedError:
-                    raise  # a hard login wall must abort the batch, not be swallowed
+                except InstagramAccessBlockedError as e:
+                    # A hard login wall aborts the batch. Hand it to the consumer
+                    # via the queue (not ``raise``) so it never deadlocks waiting.
+                    await results.put(e)
+                    return
                 except Exception as e:  # one bad target must not kill the run
                     logger.warning("[instagram] fan-out job failed: %s", e)
                 await results.put(items)
@@ -165,7 +174,10 @@ async def fan_out(
     tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))]
     try:
         for _ in range(len(jobs)):
-            for item in await results.get():
+            batch = await results.get()
+            if isinstance(batch, InstagramAccessBlockedError):
+                raise batch
+            for item in batch:
                 yield item
     finally:
         for task in tasks: