From 54eab56b252857b8d24f25c7aebbb71c769899b6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:26:12 +0530 Subject: [PATCH 01/16] docs(native-connector): document anonymous reddit scraper design --- .../app/proprietary/scrapers/reddit/README.md | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 surfsense_backend/app/proprietary/scrapers/reddit/README.md diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/README.md b/surfsense_backend/app/proprietary/scrapers/reddit/README.md new file mode 100644 index 000000000..9f4cff5f2 --- /dev/null +++ b/surfsense_backend/app/proprietary/scrapers/reddit/README.md @@ -0,0 +1,85 @@ +# Reddit scraper (anonymous, no browser) + +Platform-native Reddit scraper, Apify "Reddit Scraper"-compatible. Standalone +module: it depends only on `app.utils.proxy` + `scrapling` and exposes a stable +public API. It is **not** wired into routes, `connector_service.py`, ingestion, +or Celery — integration later is a thin `reddit_routes.py` + one `include_router` +line, identical to the `youtube` / `google_maps` siblings. + +## Approach + +Reddit deprecated **cold** unauthenticated `.json` (r/modnews, May 2026): a bare +anonymous GET to `/…/.json` now 403s. What still works — and what maintained +anonymous scrapers (e.g. yt-dlp) now do — is: + +> Bootstrap an anonymous session cookie (`loid`) with one plain HTTP GET to +> `www.reddit.com/svc/shreddit/`, then GET +> `www.reddit.com//.json?raw_json=1` through that same +> Chrome-impersonated, sticky-IP session. Rotate the residential IP + re-warm on +> block. + +`loid` is Reddit's equivalent of the Google Maps scraper's `NID`: an anonymous, +logged-out id that unlocks the public API. **No browser, no Chromium, no +`solve_cloudflare`** — this collapses Reddit onto the cheap HTTP tier the +siblings already use. Confirmed live 2026-07-04 through a residential proxy (see +`scripts/e2e_reddit_scraper.py` step 0): `svc/shreddit` warm-up → 200 + `loid` +(with `old.reddit` as a fallback), then sequential `.json` fetches → 200. + +## Anonymous only — no authentication, ever + +No OAuth, no login, no `reddit_session` account cookie, no Devvit. The only +cookie held is the anonymous `loid`. There is **no** authenticate option in the +input surface or the fetch layer, by design. A persistent block after IP +rotation surfaces as `RedditAccessBlockedError` (mirrors google_maps' +`SignInRequiredError`) rather than a silent empty result. + +## Module map + +| File | Responsibility | +|---|---| +| `__init__.py` | Public exports: `RedditScrapeInput`, `RedditItem`, `iter_reddit`, `scrape_reddit`, `RedditAccessBlockedError`. | +| `schemas.py` | Apify-shaped `RedditScrapeInput` (`extra="allow"`, no auth fields) + single flat `RedditItem` keyed by `dataType` + `StartUrl`. | +| `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (loid) + `fetch_json`. No browser imports. | +| `url_resolver.py` | Classify a Reddit URL → `post`/`subreddit`/`user`/`search`; non-Reddit → `None`. | +| `parsers.py` | Pure JSON→item mapping (`parse_post`, `parse_comment`, `parse_community`, `flatten_comments`, `children`/`after`). I/O-free. | +| `scraper.py` | Orchestrator: `_post_flow`/`_subreddit_flow`/`_user_flow`/`_search_flow`, `_paginate_listing`, `_dispatch`, `fan_out`, `iter_reddit`, `scrape_reddit`. | + +## How it works + +1. `iter_reddit` resolves `startUrls` (or builds a search per `searches` entry) + and fans them out on a pool of warm proxy sessions (`fan_out`, 16-way). Each + worker opens one sticky-IP session and warms `loid` once, reusing it across + the sequential targets it pulls. +2. Each flow pages its listing via the `after` cursor (`_paginate_listing`), + filtering by child `kind`, the NSFW gate, and `postDateLimit`. +3. `fetch_json` warms `loid` on first use, rotates the IP + re-warms on 403, + backs off on 429, returns `None` on 404, and paces each sticky IP to ~1 req/s + to stay under Reddit's per-IP rate limit. +4. Parsers map raw `.json` things to flat `RedditItem`s; the orchestrator stamps + `scrapedAt` and applies caps as request-time policy. + +## Testing + +- Offline unit tests: `tests/unit/scrapers/reddit/` — `test_skeleton.py` + (schema + URL resolver), `test_parsers.py` (fixture-pinned mapping), + `test_fetch_resilience.py` (warm / rotate / backoff loop with fake sessions, + no network). +- Live e2e (needs network + residential proxy): `scripts/e2e_reddit_scraper.py` + — step 0 is the go/no-go `loid` probe; later steps exercise the flows and dump + trimmed fixtures into `tests/unit/scrapers/reddit/fixtures/`. + +```bash +cd surfsense_backend +.venv/bin/python -m pytest tests/unit/scrapers/reddit/ +.venv/bin/python scripts/e2e_reddit_scraper.py # live; regenerates fixtures +``` + +## TODO / out of scope (v1) + +- Sticky-IP provider support: the fetch layer assumes a sticky exit IP per + session (the `loid` binds to it). `anonymous_proxies` does not yet set the + `"si"` sticky key — add it (proxy layer) before high-volume production use. +- `/api/morechildren` deep-comment expansion — `more` stubs terminate the tree + walk today. +- Routes / `connector_service.py` / ingestion / Celery wiring. +- RSS degraded-mode path (documented, not implemented). From 1115d647cba1db64d213b5dddf5dc9e29172a9ce Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:26:58 +0530 Subject: [PATCH 02/16] feat(native-connector): expose reddit scraper public API --- .../app/proprietary/scrapers/reddit/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 surfsense_backend/app/proprietary/scrapers/reddit/__init__.py diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py b/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py new file mode 100644 index 000000000..8b8c75f11 --- /dev/null +++ b/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py @@ -0,0 +1,13 @@ +"""Platform-native Reddit scraper (Apify Reddit Scraper-compatible, anonymous).""" + +from .fetch import RedditAccessBlockedError +from .schemas import RedditItem, RedditScrapeInput +from .scraper import iter_reddit, scrape_reddit + +__all__ = [ + "RedditAccessBlockedError", + "RedditItem", + "RedditScrapeInput", + "iter_reddit", + "scrape_reddit", +] From 7151465e129ffd6dd5782d80cea3fb4efa0d9be6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:27:13 +0530 Subject: [PATCH 03/16] feat(native-connector): add anonymous reddit fetch layer --- .../app/proprietary/scrapers/reddit/fetch.py | 343 ++++++++++++++++++ 1 file changed, 343 insertions(+) create mode 100644 surfsense_backend/app/proprietary/scrapers/reddit/fetch.py diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py b/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py new file mode 100644 index 000000000..02ed3ac88 --- /dev/null +++ b/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py @@ -0,0 +1,343 @@ +"""Proxy-aware fetch seam for the Reddit scraper (no browser). + +All network I/O flows through :func:`fetch_json` and always egresses through the +residential proxy (a direct hit would expose and risk-block the server IP). + +Reddit deprecated *cold* unauthenticated ``.json`` (a bare anonymous GET now +403s). The maintained anonymous recipe (proven live 2026-07-04, see +``scripts/e2e_reddit_scraper.py`` step 0) is: + + warm one anonymous session cookie (``loid``) with a plain GET to + ``old.reddit.com`` (``www.reddit.com/svc/shreddit/`` fallback), then + GET ``www.reddit.com//.json?raw_json=1`` through that same + Chrome-impersonated, sticky-IP session. Which warm URL mints ``loid`` is + exit-IP dependent, so the order is a tiebreak — see :func:`warm_session`. + +``loid`` is Reddit's equivalent of Google Maps' ``NID`` session cookie: an +anonymous, logged-out id that unlocks the public API — no account, no browser. +This module is a direct port of ``../youtube/innertube.py``'s rotate-on-block +sticky-session pattern (``_RotatingSession`` + ``_current_session`` ContextVar + +``open_proxy_holder``/``bind_proxy_holder``/``proxy_session``), with a +Reddit-specific :func:`warm_session` bolted on and a ``.json``-shaped +:func:`fetch_json` instead of an InnerTube POST. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import random +import time +from contextlib import asynccontextmanager, suppress +from contextvars import ContextVar +from datetime import UTC, datetime +from typing import Any +from urllib.parse import urlencode + +from scrapling.fetchers import AsyncFetcher, FetcherSession + +from app.utils.proxy import get_proxy_url + +logger = logging.getLogger(__name__) + + +class RedditAccessBlockedError(RuntimeError): + """Raised when every rotated IP is refused anonymous access. + + This is the yt-dlp "account authentication is required" / "your IP is unable + to access the Reddit API" branch. We are anonymous-only and cannot log in, + so instead of silently returning nothing we surface it as a clear error + (mirrors google_maps' ``SignInRequiredError``). The route can turn it into a + 403 later; for now it just fails loudly. + """ + + +# Per-flow proxy session, set by ``bind_proxy_holder`` around one continuation +# chain. Reusing one keep-alive connection pins a single sticky exit IP so the +# ``loid`` cookie (bound to that IP) stays valid across the warm-up + every +# subsequent ``.json`` fetch. A ContextVar keeps each concurrent fan-out flow on +# its own session/IP without threading a param through every call. +_current_session: ContextVar[_RotatingSession | None] = ContextVar( + "reddit_proxy_session", default=None +) + +# 403 => this IP is blocked; rotate to a fresh one and re-warm. 429 => rate +# limited; back off on the SAME IP (rotating wouldn't help and burns the pool). +# Split out from youtube's combined ``_BLOCK_STATUSES`` because Reddit wants +# different handling per status (spec section 3). +_ROTATE_STATUS = 403 +_BACKOFF_STATUS = 429 +_MAX_ROTATIONS = 3 +_MAX_BACKOFFS = 4 +_BACKOFF_BASE_S = 5.0 + +# Reddit 429s aggressively (~60-100 req/min/IP). Pace each sticky session to +# ~1 req/s + jitter to stay under the per-IP threshold. +_MIN_INTERVAL_S = 1.0 +_PACE_JITTER_S = 0.5 + +_HEADERS = {"Accept-Language": "en-US,en;q=0.9"} + +# Age-gate opt-in, sent on every ``.json`` fetch so NSFW listings aren't blanked +# (the caller filters on ``includeNSFW`` downstream). Mirrors the probe. +_OVER18_COOKIES = {"over18": "1"} + +# ``svc/shreddit`` needs *a* path to render; any always-public subreddit mints +# ``loid`` just the same. ``r/popular`` always exists, so it's a safe default +# regardless of which target this session ends up serving. +_WARM_SLUG = "r/popular" +_SHREDDIT_URL = ( + "https://www.reddit.com/svc/shreddit/{slug}" + "?render-mode=partial&seeker-session=false" +) +_OLD_REDDIT_URL = "https://old.reddit.com/" +_LOID_COOKIE = "loid" + + +def now_iso() -> str: + """UTC timestamp in the millisecond ISO shape the Apify actor stamps.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def _response_cookie_names(page: Any) -> set[str]: + """Cookie names set by a response (best-effort across scrapling shapes).""" + cookies = getattr(page, "cookies", None) + if isinstance(cookies, dict): + return set(cookies.keys()) + return set() + + +def _parse_json(page: Any) -> Any | None: + """Parse a scrapling response body into JSON, or ``None``. + + Prefers ``page.json()``; falls back to ``json.loads`` on the raw body when + the impersonated response hands back text. No ``
`` unwrapping — that was
+    a v1 browser artifact and does not apply to plain-HTTP ``.json``.
+    """
+    fn = getattr(page, "json", None)
+    if callable(fn):
+        with suppress(Exception):
+            return fn()
+    for attr in ("body", "text"):
+        val = getattr(page, attr, None)
+        if isinstance(val, bytes):
+            val = val.decode("utf-8", "replace")
+        if isinstance(val, str) and val.strip():
+            with suppress(Exception):
+                return json.loads(val)
+            return None
+    return None
+
+
+def _build_url(path: str, params: dict[str, Any] | None) -> str:
+    """``https://www.reddit.com//.json?raw_json=1&...`` (always raw_json)."""
+    clean = path.strip("/")
+    query = {"raw_json": "1", **(params or {})}
+    qs = urlencode({k: v for k, v in query.items() if v is not None})
+    return f"https://www.reddit.com/{clean}/.json?{qs}"
+
+
+class _RotatingSession:
+    """Owns one live ``FetcherSession`` (sticky IP) and can swap it for a fresh one.
+
+    ``rotate()`` closes the current keep-alive connection and opens a new one, so
+    the rotating gateway hands out a different residential exit IP. Because the
+    ``loid`` cookie binds to the exit IP, ``rotate()`` also drops the warmed
+    state — the next fetch re-warms on the new IP. Used sequentially within a
+    single flow (never shared across concurrent tasks), so no locking is needed.
+    ``session`` is ``None`` only when no proxy is configured.
+    """
+
+    def __init__(self) -> None:
+        self._cm: Any | None = None
+        self.session: Any | None = None
+        self.rotations = 0
+        self.warmed = False
+        self._last_at = 0.0
+
+    async def _open(self) -> None:
+        proxy = get_proxy_url()
+        self.warmed = False
+        if proxy is None:
+            self._cm = self.session = None
+            return
+        self._cm = FetcherSession(
+            proxy=proxy, stealthy_headers=True, impersonate="chrome"
+        )
+        self.session = await self._cm.__aenter__()
+
+    async def close(self) -> None:
+        if self._cm is not None:
+            with suppress(Exception):  # best-effort teardown
+                await self._cm.__aexit__(None, None, None)
+        self._cm = self.session = None
+
+    async def rotate(self) -> Any | None:
+        """Drop the current IP and connect through a fresh one. Returns new session."""
+        await self.close()
+        self.rotations += 1
+        await self._open()
+        logger.info("[reddit] rotated proxy session (rotation #%d)", self.rotations)
+        return self.session
+
+    async def pace(self) -> None:
+        """Sleep to hold this sticky IP under Reddit's per-IP rate threshold."""
+        wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_at)
+        if wait > 0:
+            await asyncio.sleep(wait + random.uniform(0, _PACE_JITTER_S))
+        self._last_at = time.monotonic()
+
+
+async def open_proxy_holder() -> _RotatingSession:
+    """Open a warm rotate-on-block session holder (caller owns ``close()``)."""
+    holder = _RotatingSession()
+    await holder._open()
+    return holder
+
+
+@asynccontextmanager
+async def bind_proxy_holder(holder: _RotatingSession):
+    """Route this task's fetches through ``holder`` for the enclosed block.
+
+    Does NOT close the holder — enables pooling warm sessions across sequential
+    jobs so each job skips the ~2s proxy handshake AND the ``loid`` warm-up.
+    """
+    token = _current_session.set(holder)
+    try:
+        yield holder
+    finally:
+        _current_session.reset(token)
+
+
+@asynccontextmanager
+async def proxy_session():
+    """Open one reused, rotate-on-block proxy session for a continuation chain."""
+    holder = await open_proxy_holder()
+    try:
+        async with bind_proxy_holder(holder):
+            yield holder
+    finally:
+        await holder.close()
+
+
+async def warm_session(session: Any, *, slug: str = _WARM_SLUG) -> bool:
+    """Mint an anonymous ``loid`` cookie on a freshly opened session.
+
+    Returns ``True`` when a ``loid`` was issued (the session can now reach
+    ``.json``), else ``False`` (caller rotates the IP and retries).
+
+    Tries ``old.reddit`` (yt-dlp's primary), then ``svc/shreddit``. WHICH one
+    mints is exit-IP dependent and roughly random: live probes 2026-07-04 saw
+    both directions across sessions on the rotating residential/custom proxy
+    (one IP 403s ``old.reddit`` but mints on ``shreddit``; another does the
+    reverse, sometimes with an ``rdt`` bot-interstitial). So the order is a
+    tiebreak, not an optimization — a fresh session pays ~one wasted warm 403
+    either way. That cost is amortized: ``fan_out`` reuses one warmed session
+    per worker across many jobs, so warm-up runs once per worker, not per fetch.
+    The fallback is what actually matters — it preserves correctness whichever
+    way a given IP leans.
+
+    ponytail: sequential two-source warm burns 1 wasted request on ~half of new
+    sessions. A parallel warm (gather both, take whichever mints) removes the
+    latency but always spends 2 requests; not worth it while warm-up is
+    once-per-worker. Revisit only if session churn (not reuse) dominates.
+
+    Takes an already-open ``session`` (never constructs one) so tests can drive
+    warm/rotate deterministically with a fake session, exactly like the youtube
+    sibling's fetch-resilience tests.
+    """
+    seen: set[str] = set()
+    with suppress(Exception):
+        page = await session.get(_OLD_REDDIT_URL, headers=_HEADERS)
+        seen |= _response_cookie_names(page)
+    if _LOID_COOKIE in seen:
+        return True
+
+    # Fallback: mints loid on exit IPs where old.reddit 403s instead.
+    with suppress(Exception):
+        page = await session.get(_SHREDDIT_URL.format(slug=slug), headers=_HEADERS)
+        seen |= _response_cookie_names(page)
+    return _LOID_COOKIE in seen
+
+
+async def _get_page(session: Any, url: str) -> Any:
+    """GET through the warmed sticky session, or a one-shot proxied fetch."""
+    if session is not None:
+        return await session.get(url, headers=_HEADERS, cookies=_OVER18_COOKIES)
+    return await AsyncFetcher.get(
+        url,
+        headers=_HEADERS,
+        cookies=_OVER18_COOKIES,
+        proxy=get_proxy_url(),
+        stealthy_headers=True,
+    )
+
+
+async def fetch_json(path: str, params: dict[str, Any] | None = None) -> Any | None:
+    """GET a Reddit ``.json`` endpoint through a ``loid``-warmed HTTP session.
+
+    Returns parsed JSON (dict or list), or ``None`` on 404 / non-block failure.
+    Warms the ``loid`` session once per session; rotates the residential IP and
+    re-warms on 403; backs off on 429. Raises :class:`RedditAccessBlockedError`
+    only when every rotated IP refuses anonymous access (the yt-dlp
+    "login required" branch, which we cannot satisfy).
+    """
+    holder = _current_session.get()
+    if holder is None:
+        # No bound session (e.g. a direct call outside fan_out): open a
+        # short-lived warmed session for this one fetch, then tear it down.
+        async with proxy_session():
+            return await fetch_json(path, params)
+
+    url = _build_url(path, params)
+    attempt = 0
+    backoffs = 0
+    while True:
+        session = holder.session
+        try:
+            if session is not None and not holder.warmed:
+                warmed_ok = await warm_session(session)
+                holder.warmed = True  # attempted; don't re-warm this IP
+                if not warmed_ok:
+                    if attempt < _MAX_ROTATIONS:
+                        attempt += 1
+                        await holder.rotate()
+                        continue
+                    raise RedditAccessBlockedError(
+                        f"could not mint loid after {attempt} IP rotations for {path}"
+                    )
+
+            await holder.pace()
+            page = await _get_page(session, url)
+            status = page.status
+
+            if status == 200:
+                return _parse_json(page)
+            if status == 404:
+                return None
+            if status == _BACKOFF_STATUS and backoffs < _MAX_BACKOFFS:
+                backoffs += 1
+                delay = _BACKOFF_BASE_S * (2 ** (backoffs - 1))
+                logger.warning("[reddit] 429 on %s; backing off %.1fs", path, delay)
+                await asyncio.sleep(delay + random.uniform(0, 1))
+                continue
+            if status == _ROTATE_STATUS and attempt < _MAX_ROTATIONS:
+                attempt += 1
+                await holder.rotate()
+                continue
+            if status == _ROTATE_STATUS:
+                raise RedditAccessBlockedError(
+                    f"Reddit refused {path} on {attempt} rotated IPs (403)"
+                )
+            logger.warning("[reddit] GET %s returned %s", path, status)
+            return None
+        except RedditAccessBlockedError:
+            raise
+        except Exception as e:
+            logger.warning("[reddit] GET %s failed: %s", path, e)
+            if attempt < _MAX_ROTATIONS:
+                attempt += 1
+                await holder.rotate()
+                continue
+            return None

From 0745f433ab64341c8f76506a31505691ba7ef19e Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:27:22 +0530
Subject: [PATCH 04/16] feat(native-connector): add reddit json parsers

---
 .../proprietary/scrapers/reddit/parsers.py    | 250 ++++++++++++++++++
 1 file changed, 250 insertions(+)
 create mode 100644 surfsense_backend/app/proprietary/scrapers/reddit/parsers.py

diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/parsers.py b/surfsense_backend/app/proprietary/scrapers/reddit/parsers.py
new file mode 100644
index 000000000..10cc2150a
--- /dev/null
+++ b/surfsense_backend/app/proprietary/scrapers/reddit/parsers.py
@@ -0,0 +1,250 @@
+"""Pure JSON -> item mapping for the Reddit scraper.
+
+Framework-agnostic and I/O-free so it can be unit-tested against captured
+fixtures. Every function takes raw Reddit ``.json`` data and returns plain
+dicts / lists — no network, no proxy, no ``scrapedAt`` stamp (the orchestrator
+adds the timestamp so these stay deterministic under fixture tests).
+
+Reddit's ``.json`` wraps everything in "things" (``{"kind": "t3", "data":
+{...}}``) and "Listings" (``{"kind": "Listing", "data": {"children": [...],
+"after": ...}}``). ``t3`` = post, ``t1`` = comment, ``more`` = a truncated-reply
+stub (out of scope v1 — treated as a branch stop).
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+_REDDIT_BASE = "https://www.reddit.com"
+
+
+def _int(value: Any) -> int | None:
+    """Coerce to int, or ``None`` (Reddit sometimes sends floats/None)."""
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(value)
+    return None
+
+
+def _utc_from_sec(value: Any) -> str | None:
+    """Epoch seconds -> millisecond ISO string, or ``None``."""
+    if not isinstance(value, int | float):
+        return None
+    dt = datetime.fromtimestamp(float(value), tz=UTC)
+    return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+
+
+def _parse_iso(value: str | None) -> datetime | None:
+    """Parse an ISO date (tolerating a trailing ``Z``) to aware UTC, else None."""
+    if not value:
+        return None
+    try:
+        dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return dt if dt.tzinfo else dt.replace(tzinfo=UTC)
+
+
+def _before(created_utc: Any, date_limit: str | None) -> bool:
+    """True if ``created_utc`` (epoch secs) predates the ISO ``date_limit``."""
+    limit = _parse_iso(date_limit)
+    if limit is None or not isinstance(created_utc, int | float):
+        return False
+    return datetime.fromtimestamp(float(created_utc), tz=UTC) < limit
+
+
+def _thumbnail_url(value: Any) -> str | None:
+    """Reddit uses sentinels like 'self'/'default'/'nsfw' for no thumbnail."""
+    return value if isinstance(value, str) and value.startswith("http") else None
+
+
+def _permalink_url(data: dict[str, Any]) -> str | None:
+    permalink = data.get("permalink")
+    return f"{_REDDIT_BASE}{permalink}" if isinstance(permalink, str) else None
+
+
+def _strip_prefix(fullname: Any) -> str | None:
+    """``t3_abc`` -> ``abc`` (Reddit fullname without its type prefix)."""
+    if isinstance(fullname, str) and "_" in fullname:
+        return fullname.split("_", 1)[1]
+    return fullname if isinstance(fullname, str) else None
+
+
+def _image_urls(data: dict[str, Any]) -> list[str]:
+    """Best-effort image URLs from a post's ``preview.images[].source.url``."""
+    preview = data.get("preview")
+    if not isinstance(preview, dict):
+        return []
+    urls: list[str] = []
+    for image in preview.get("images") or []:
+        source = image.get("source") if isinstance(image, dict) else None
+        url = source.get("url") if isinstance(source, dict) else None
+        if isinstance(url, str) and url:
+            urls.append(url)
+    return urls
+
+
+def _video_urls(data: dict[str, Any]) -> list[str]:
+    """Best-effort video URL from ``media.reddit_video.fallback_url``."""
+    for key in ("media", "secure_media"):
+        media = data.get(key)
+        if isinstance(media, dict):
+            rv = media.get("reddit_video")
+            url = rv.get("fallback_url") if isinstance(rv, dict) else None
+            if isinstance(url, str) and url:
+                return [url]
+    return []
+
+
+def children(listing: Any) -> list[dict[str, Any]]:
+    """Return a Listing's ``data.children`` array (empty list if malformed)."""
+    if isinstance(listing, dict):
+        data = listing.get("data")
+        if isinstance(data, dict):
+            kids = data.get("children")
+            if isinstance(kids, list):
+                return kids
+    return []
+
+
+def after(listing: Any) -> str | None:
+    """Return a Listing's ``data.after`` pagination cursor, or ``None``."""
+    if isinstance(listing, dict):
+        data = listing.get("data")
+        if isinstance(data, dict):
+            cursor = data.get("after")
+            return cursor if isinstance(cursor, str) else None
+    return None
+
+
+def _unwrap(thing: dict[str, Any]) -> dict[str, Any]:
+    """Accept a ``{"kind","data"}`` thing or a bare data dict; return the data."""
+    data = thing.get("data")
+    return data if isinstance(data, dict) else thing
+
+
+def parse_post(thing: dict[str, Any]) -> dict[str, Any]:
+    """Map a ``t3`` post thing (or its data dict) to a flat item dict."""
+    data = _unwrap(thing)
+    is_self = bool(data.get("is_self"))
+    external = data.get("url") if not is_self else None
+    return {
+        "dataType": "post",
+        "id": data.get("name"),
+        "parsedId": data.get("id"),
+        "url": _permalink_url(data),
+        "username": data.get("author"),
+        "userId": data.get("author_fullname"),
+        "title": data.get("title"),
+        "body": data.get("selftext") or None,
+        "html": data.get("selftext_html"),
+        "link": external,
+        "externalUrl": external,
+        "communityName": data.get("subreddit_name_prefixed"),
+        "parsedCommunityName": data.get("subreddit"),
+        "numberOfComments": _int(data.get("num_comments")),
+        "upVotes": _int(data.get("score", data.get("ups"))),
+        "upVoteRatio": data.get("upvote_ratio"),
+        "over18": data.get("over_18"),
+        "isVideo": data.get("is_video"),
+        "flair": data.get("link_flair_text"),
+        "authorFlair": data.get("author_flair_text"),
+        "thumbnailUrl": _thumbnail_url(data.get("thumbnail")),
+        "imageUrls": _image_urls(data),
+        "videoUrls": _video_urls(data),
+        "numberOfMembers": _int(data.get("subreddit_subscribers")),
+        "createdAt": _utc_from_sec(data.get("created_utc")),
+    }
+
+
+def parse_community(thing: dict[str, Any]) -> dict[str, Any]:
+    """Map a ``t5`` subreddit thing (``about.json``) to a flat community item."""
+    data = _unwrap(thing)
+    url = data.get("url")
+    return {
+        "dataType": "community",
+        "id": data.get("name"),
+        "parsedId": data.get("id"),
+        "url": f"{_REDDIT_BASE}{url}" if isinstance(url, str) else None,
+        "title": data.get("title"),
+        "body": data.get("public_description") or None,
+        "communityName": data.get("display_name_prefixed"),
+        "parsedCommunityName": data.get("display_name"),
+        "numberOfMembers": _int(data.get("subscribers")),
+        "over18": data.get("over18"),
+        "createdAt": _utc_from_sec(data.get("created_utc")),
+    }
+
+
+def parse_comment(thing: dict[str, Any], *, depth: int = 0) -> dict[str, Any]:
+    """Map a ``t1`` comment thing (or its data dict) to a flat item dict.
+
+    ``numberOfReplies`` is left at ``0`` here; :func:`flatten_comments` fills it
+    with the count of descendant comments it actually emits.
+    """
+    data = _unwrap(thing)
+    return {
+        "dataType": "comment",
+        "id": data.get("name"),
+        "parsedId": data.get("id"),
+        "url": _permalink_url(data),
+        "username": data.get("author"),
+        "userId": data.get("author_fullname"),
+        "body": data.get("body") or None,
+        "html": data.get("body_html"),
+        "communityName": data.get("subreddit_name_prefixed"),
+        "parsedCommunityName": data.get("subreddit"),
+        "upVotes": _int(data.get("score", data.get("ups"))),
+        "over18": data.get("over_18"),
+        "authorFlair": data.get("author_flair_text"),
+        "postId": _strip_prefix(data.get("link_id")),
+        "parentId": data.get("parent_id"),
+        "numberOfReplies": 0,
+        "createdAt": _utc_from_sec(data.get("created_utc")),
+        "depth": depth,
+    }
+
+
+def flatten_comments(
+    comment_children: list[dict[str, Any]] | None,
+    *,
+    max_comments: int,
+    date_limit: str | None = None,
+    depth: int = 0,
+    out: list[dict[str, Any]] | None = None,
+) -> list[dict[str, Any]]:
+    """Depth-first flatten a comment tree into a bounded flat list.
+
+    Skips non-``t1`` children (``more`` stubs terminate that branch — v1),
+    honors ``max_comments`` (total emitted, across all depths) and
+    ``date_limit`` (drops comments older than the ISO cutoff). Each comment's
+    ``numberOfReplies`` is set to the number of its descendants that were
+    emitted.
+    """
+    if out is None:
+        out = []
+    for child in comment_children or []:
+        if len(out) >= max_comments:
+            break
+        if not isinstance(child, dict) or child.get("kind") != "t1":
+            continue  # 'more' stub / non-comment: stop this branch (v1)
+        data = child.get("data") or {}
+        if date_limit and _before(data.get("created_utc"), date_limit):
+            continue
+        item = parse_comment(data, depth=depth)
+        before = len(out)
+        out.append(item)
+        replies = data.get("replies")
+        flatten_comments(
+            children(replies) if isinstance(replies, dict) else [],
+            max_comments=max_comments,
+            date_limit=date_limit,
+            depth=depth + 1,
+            out=out,
+        )
+        item["numberOfReplies"] = len(out) - before - 1
+    return out

From dd43ab8505f484c506a902c5abc1009eaf6317d9 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:27:33 +0530
Subject: [PATCH 05/16] feat(native-connector): add reddit scraper schemas

---
 .../proprietary/scrapers/reddit/schemas.py    | 127 ++++++++++++++++++
 1 file changed, 127 insertions(+)
 create mode 100644 surfsense_backend/app/proprietary/scrapers/reddit/schemas.py

diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py b/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py
new file mode 100644
index 000000000..28d322936
--- /dev/null
+++ b/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py
@@ -0,0 +1,127 @@
+# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec
+"""Apify-compatible input/output models for the Reddit scraper.
+
+The models mirror the public Apify "Reddit Scraper" actor spec so the endpoint
+can be a drop-in. The MVP populates a reliably-sourced subset; every other field
+is accepted (input) or emitted as ``None``/``[]`` (output) so parity is additive.
+
+**Anonymous only.** There is deliberately **no** authentication field on the
+input (no username/password/token/``login*``) — the scraper holds only Reddit's
+anonymous ``loid`` session cookie and can never log in. Anything auth-shaped a
+caller sends lands in ``extra`` and is ignored.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+RedditSort = Literal["relevance", "hot", "top", "new", "rising", "comments"]
+RedditTime = Literal["all", "hour", "day", "week", "month", "year"]
+RedditDataType = Literal["post", "comment", "community", "user"]
+
+
+class StartUrl(BaseModel):
+    """A single direct URL entry (Apify passes ``{"url": ...}`` objects)."""
+
+    model_config = ConfigDict(extra="allow")
+
+    url: str
+
+
+class RedditScrapeInput(BaseModel):
+    """Full Apify "Reddit Scraper" input surface (anonymous, no auth fields).
+
+    Caps (``maxItems``/``maxPostCount``/...) are collector policy applied by
+    :func:`scrape_reddit`, NOT ceilings baked into the streaming flows. Fields
+    the MVP doesn't act on are still accepted via ``extra="allow"`` for parity.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    # Discovery
+    startUrls: list[StartUrl] = Field(default_factory=list)
+    searches: list[str] = Field(default_factory=list)
+    searchCommunityName: str | None = None
+
+    # Sort / filter
+    sort: RedditSort = "new"
+    time: RedditTime | None = None
+    includeNSFW: bool = True
+
+    # Skips
+    skipComments: bool = False
+    skipUserPosts: bool = False
+    skipCommunity: bool = False
+
+    # Caps (collector policy; enforced by scrape_reddit, not the flows)
+    maxItems: int = Field(default=10, ge=0)
+    maxPostCount: int = Field(default=10, ge=0)
+    maxComments: int = Field(default=10, ge=0)
+    maxCommunitiesCount: int = Field(default=2, ge=0)
+    maxUserCount: int = Field(default=2, ge=0)
+
+    # Incremental scraping (ISO dates)
+    postDateLimit: str | None = None
+    commentDateLimit: str | None = None
+
+
+class RedditItem(BaseModel):
+    """Single flat output item, keyed by ``dataType``.
+
+    One model for post/comment/community/user (union of fields, unsourced
+    default ``None``/``[]``), matching the Apify actor's single-dataset shape.
+    ``extra="allow"`` keeps the contract open so added fields never break
+    consumers.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    dataType: RedditDataType | None = None
+
+    # Identity / provenance
+    id: str | None = None
+    parsedId: str | None = None
+    url: str | None = None
+    username: str | None = None
+    userId: str | None = None
+
+    # Content
+    title: str | None = None
+    body: str | None = None
+    html: str | None = None
+    link: str | None = None
+    externalUrl: str | None = None
+
+    # Community
+    communityName: str | None = None
+    parsedCommunityName: str | None = None
+    numberOfMembers: int | None = None
+
+    # Engagement
+    numberOfComments: int | None = None
+    numberOfReplies: int | None = None
+    upVotes: int | None = None
+    upVoteRatio: float | None = None
+
+    # Flags / media
+    over18: bool | None = None
+    isVideo: bool | None = None
+    flair: str | None = None
+    authorFlair: str | None = None
+    thumbnailUrl: str | None = None
+    imageUrls: list[str] = Field(default_factory=list)
+    videoUrls: list[str] = Field(default_factory=list)
+
+    # Threading
+    postId: str | None = None
+    parentId: str | None = None
+
+    # Timestamps
+    createdAt: str | None = None
+    scrapedAt: str | None = None
+
+    def to_output(self) -> dict[str, Any]:
+        """Serialize to the flat dict shape Apify emits (keeps extras)."""
+        return self.model_dump(exclude_none=False)

From eaa8239e3668ca63301314087a553232d9a69db6 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:27:38 +0530
Subject: [PATCH 06/16] feat(native-connector): add reddit scraper orchestrator

---
 .../proprietary/scrapers/reddit/scraper.py    | 411 ++++++++++++++++++
 1 file changed, 411 insertions(+)
 create mode 100644 surfsense_backend/app/proprietary/scrapers/reddit/scraper.py

diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py b/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py
new file mode 100644
index 000000000..5fc4e07e9
--- /dev/null
+++ b/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py
@@ -0,0 +1,411 @@
+"""Orchestrator for the Reddit scraper.
+
+The core is the async generator :func:`iter_reddit` (unbounded, ``after``-cursor
+paged); :func:`scrape_reddit` is a thin collector with a caller-supplied
+``limit`` guard. Any cap is caller policy, never baked into flow logic.
+
+Independent targets (one per ``startUrl`` / search) fan out concurrently on a
+pool of warm ``loid`` sessions (sticky IPs); each target's own ``after`` paging
+stays sequential. ``fan_out`` is ported from ``../youtube/scraper.py`` but bound
+to *this* module's proxy holders so every worker warms its own ``loid`` once and
+reuses it — the ~10-50x throughput win over a browser design.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from typing import Any
+
+from .fetch import (
+    RedditAccessBlockedError,
+    bind_proxy_holder,
+    fetch_json,
+    now_iso,
+    open_proxy_holder,
+)
+from .parsers import (
+    _before,
+    after,
+    children,
+    flatten_comments,
+    parse_comment,
+    parse_community,
+    parse_post,
+)
+from .schemas import RedditItem, RedditScrapeInput
+from .url_resolver import ResolvedUrl, resolve_url
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "RedditAccessBlockedError",
+    "iter_reddit",
+    "scrape_reddit",
+]
+
+# Independent jobs run concurrently on a pool of warm proxy sessions. Matches
+# the youtube sibling; 16 workers saturate typical job counts while leaving
+# gateway headroom.
+_FANOUT_CONCURRENCY = 16
+
+# Reddit caps any listing at ~1000 items (100/page => ~10 pages). Stop there so
+# a runaway target can't page forever.
+_LISTING_LIMIT = 100
+_MAX_PAGES = 10
+_EMPTY_STREAK_ABORT = 2
+
+# Search sorts differ from listing sorts; fall back to "new" for a listing path
+# when the input carries a search-only sort.
+_LISTING_SORTS = frozenset({"hot", "new", "top", "rising", "controversial", "best"})
+
+
+async def fan_out(
+    jobs: list[AsyncIterator[dict[str, Any]]], *, concurrency: int = _FANOUT_CONCURRENCY
+) -> AsyncIterator[dict[str, Any]]:
+    """Stream items from independent async-iterator jobs via a warm worker pool.
+
+    Each worker opens ONE proxy session and reuses it across the sequential jobs
+    it pulls, so only the first job per worker pays the proxy handshake + the
+    ``loid`` warm-up. A bad job yields nothing rather than aborting the batch;
+    workers are cancelled and their sessions closed if the consumer stops early.
+    """
+    if not jobs:
+        return
+    job_queue: asyncio.Queue[AsyncIterator[dict[str, Any]]] = asyncio.Queue()
+    for job in jobs:
+        job_queue.put_nowait(job)
+    results: asyncio.Queue[list[dict[str, Any]]] = asyncio.Queue()
+
+    async def worker() -> None:
+        holder = None
+        try:
+            holder = await open_proxy_holder()
+        except Exception as e:  # no session: jobs still run via one-shot fetches
+            logger.warning("[reddit] proxy session open failed: %s", e)
+        try:
+            while True:
+                try:
+                    job = job_queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    return
+                items: list[dict[str, Any]] = []
+                try:
+                    if holder is not None:
+                        async with bind_proxy_holder(holder):
+                            items = [item async for item in job]
+                    else:
+                        items = [item async for item in job]
+                except Exception as e:  # one bad target must not kill the run
+                    logger.warning("[reddit] fan-out job failed: %s", e)
+                await results.put(items)
+        finally:
+            if holder is not None:
+                await holder.close()
+
+    tasks = [asyncio.create_task(worker()) for _ in range(min(concurrency, len(jobs)))]
+    try:
+        for _ in range(len(jobs)):
+            for item in await results.get():
+                yield item
+    finally:
+        for task in tasks:
+            if not task.done():
+                task.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+
+def _emit(partial: dict[str, Any], *, include_nsfw: bool) -> dict[str, Any] | None:
+    """Stamp ``scrapedAt``, apply the NSFW gate, and wrap as an output dict."""
+    if not include_nsfw and partial.get("over18") is True:
+        return None
+    return RedditItem(**{**partial, "scrapedAt": now_iso()}).to_output()
+
+
+async def _paginate_listing(
+    path: str,
+    base_params: dict[str, Any],
+    kinds: frozenset[str],
+    *,
+    max_items: int,
+    include_nsfw: bool,
+    date_limit: str | None = None,
+) -> AsyncIterator[dict[str, Any]]:
+    """Yield raw child ``data`` dicts across pages via the ``after`` cursor.
+
+    Filters by child ``kind`` (``t3``/``t1``), the NSFW gate, and ``date_limit``
+    (drops older items and, since ``date_limit`` forces newest-first, stops once
+    a page crosses the cutoff). Aborts on an empty-streak, a null ``after``, or
+    the ~1000-item page ceiling.
+    """
+    if max_items <= 0:
+        return
+    emitted = 0
+    cursor: str | None = None
+    empty_streak = 0
+    for _page in range(_MAX_PAGES):
+        params = {**base_params, "limit": _LISTING_LIMIT}
+        if cursor:
+            params["after"] = cursor
+        listing = await fetch_json(path, params)
+        kids = children(listing)
+        if not kids:
+            empty_streak += 1
+            if empty_streak >= _EMPTY_STREAK_ABORT:
+                break
+        else:
+            empty_streak = 0
+        crossed_cutoff = False
+        for child in kids:
+            if not isinstance(child, dict) or child.get("kind") not in kinds:
+                continue
+            data = child.get("data") or {}
+            if date_limit and _before(data.get("created_utc"), date_limit):
+                crossed_cutoff = True
+                continue
+            if not include_nsfw and data.get("over_18") is True:
+                continue
+            yield data
+            emitted += 1
+            if emitted >= max_items:
+                return
+        cursor = after(listing)
+        if not cursor or crossed_cutoff:
+            break
+
+
+async def _post_flow(
+    post_id: str,
+    *,
+    input_model: RedditScrapeInput,
+    subreddit: str | None = None,
+    include_post: bool = True,
+) -> AsyncIterator[dict[str, Any]]:
+    """Emit a post (unless ``include_post`` is False) plus its comment tree."""
+    path = f"r/{subreddit}/comments/{post_id}" if subreddit else f"comments/{post_id}"
+    data = await fetch_json(path)
+    if not isinstance(data, list) or not data:
+        return
+    post_children = children(data[0])
+    if include_post and post_children:
+        item = _emit(parse_post(post_children[0]), include_nsfw=input_model.includeNSFW)
+        if item is not None:
+            yield item
+    if input_model.skipComments or len(data) < 2:
+        return
+    flat = flatten_comments(
+        children(data[1]),
+        max_comments=input_model.maxComments,
+        date_limit=input_model.commentDateLimit,
+    )
+    for comment in flat:
+        item = _emit(comment, include_nsfw=input_model.includeNSFW)
+        if item is not None:
+            yield item
+
+
+async def _subreddit_flow(
+    subreddit: str,
+    *,
+    input_model: RedditScrapeInput,
+    sort: str | None = None,
+) -> AsyncIterator[dict[str, Any]]:
+    """Emit the community, then paged posts (descending into comments if asked)."""
+    if not input_model.skipCommunity:
+        about = await fetch_json(f"r/{subreddit}/about")
+        if isinstance(about, dict):
+            item = _emit(parse_community(about), include_nsfw=input_model.includeNSFW)
+            if item is not None:
+                yield item
+
+    # postDateLimit forces newest-first so the early-stop is correct.
+    sort = "new" if input_model.postDateLimit else (sort or input_model.sort)
+    if sort not in _LISTING_SORTS:
+        sort = "new"
+    params: dict[str, Any] = {}
+    if sort == "top" and input_model.time:
+        params["t"] = input_model.time
+
+    async for data in _paginate_listing(
+        f"r/{subreddit}/{sort}",
+        params,
+        frozenset({"t3"}),
+        max_items=input_model.maxPostCount,
+        include_nsfw=input_model.includeNSFW,
+        date_limit=input_model.postDateLimit,
+    ):
+        item = _emit(parse_post(data), include_nsfw=input_model.includeNSFW)
+        if item is not None:
+            yield item
+        if not input_model.skipComments and isinstance(data.get("id"), str):
+            async for comment in _post_flow(
+                data["id"],
+                input_model=input_model,
+                subreddit=subreddit,
+                include_post=False,
+            ):
+                yield comment
+
+
+async def _user_flow(
+    username: str,
+    *,
+    input_model: RedditScrapeInput,
+    content: str | None = None,
+) -> AsyncIterator[dict[str, Any]]:
+    """Page a user's overview/submitted/comments listing (mixed t3 + t1)."""
+    if content == "submitted":
+        path, kinds = f"user/{username}/submitted", frozenset({"t3"})
+    elif content == "comments":
+        path, kinds = f"user/{username}/comments", frozenset({"t1"})
+    else:
+        path = f"user/{username}"
+        kinds = frozenset({"t1"} if input_model.skipUserPosts else {"t3", "t1"})
+
+    async for data in _paginate_listing(
+        path,
+        {},
+        kinds,
+        max_items=input_model.maxItems,
+        include_nsfw=input_model.includeNSFW,
+        date_limit=input_model.postDateLimit,
+    ):
+        # A user listing mixes posts (t3) and comments (t1); a post has a title.
+        parsed = parse_post(data) if data.get("title") is not None else parse_comment(
+            data
+        )
+        item = _emit(parsed, include_nsfw=input_model.includeNSFW)
+        if item is not None:
+            yield item
+
+
+async def _search_flow(
+    query: str,
+    *,
+    input_model: RedditScrapeInput,
+    subreddit: str | None = None,
+) -> AsyncIterator[dict[str, Any]]:
+    """Global search, or in-subreddit when ``subreddit`` is set. De-dupes by id."""
+    params: dict[str, Any] = {"q": query, "sort": input_model.sort}
+    if input_model.time:
+        params["t"] = input_model.time
+    if subreddit:
+        path = f"r/{subreddit}/search"
+        params["restrict_sr"] = "on"
+    else:
+        path = "search"
+
+    seen: set[str] = set()
+    async for data in _paginate_listing(
+        path,
+        params,
+        frozenset({"t3"}),
+        max_items=input_model.maxItems,
+        include_nsfw=input_model.includeNSFW,
+        date_limit=input_model.postDateLimit,
+    ):
+        post_id = data.get("id")
+        if isinstance(post_id, str):
+            if post_id in seen:
+                continue
+            seen.add(post_id)
+        item = _emit(parse_post(data), include_nsfw=input_model.includeNSFW)
+        if item is not None:
+            yield item
+
+
+def _dispatch(
+    resolved: ResolvedUrl, input_model: RedditScrapeInput
+) -> AsyncIterator[dict[str, Any]]:
+    """Route a resolved URL to its flow (returns the flow's async generator)."""
+    if resolved.kind == "post":
+        return _post_flow(
+            resolved.value, input_model=input_model, subreddit=resolved.subreddit
+        )
+    if resolved.kind == "subreddit":
+        return _subreddit_flow(
+            resolved.value, input_model=input_model, sort=resolved.sort
+        )
+    if resolved.kind == "user":
+        return _user_flow(
+            resolved.value, input_model=input_model, content=resolved.content
+        )
+    return _search_flow(
+        resolved.value, input_model=input_model, subreddit=resolved.subreddit
+    )
+
+
+def _capped_targets(
+    resolved: list[ResolvedUrl], input_model: RedditScrapeInput
+) -> list[ResolvedUrl]:
+    """Apply the target-count caps (``maxCommunitiesCount`` / ``maxUserCount``).
+
+    These bound how many subreddit / user *targets* are scraped; per-target item
+    counts are bounded inside each flow (maxPostCount / maxItems / maxComments).
+    """
+    subs = users = 0
+    out: list[ResolvedUrl] = []
+    for r in resolved:
+        if r.kind == "subreddit":
+            if subs >= input_model.maxCommunitiesCount:
+                continue
+            subs += 1
+        elif r.kind == "user":
+            if users >= input_model.maxUserCount:
+                continue
+            users += 1
+        out.append(r)
+    return out
+
+
+async def iter_reddit(
+    input_model: RedditScrapeInput,
+) -> AsyncIterator[dict[str, Any]]:
+    """Yield Apify-shaped Reddit items. ``startUrls`` override ``searches``.
+
+    Independent targets fan out concurrently; each target's ``after`` paging
+    stays sequential.
+    """
+    if input_model.startUrls:
+        resolved: list[ResolvedUrl] = []
+        for entry in input_model.startUrls:
+            r = resolve_url(entry.url)
+            if r is None:
+                logger.warning("[reddit] unrecognized URL: %s", entry.url)
+                continue
+            resolved.append(r)
+        jobs = [
+            _dispatch(r, input_model)
+            for r in _capped_targets(resolved, input_model)
+        ]
+        async for item in fan_out(jobs):
+            yield item
+        return
+
+    jobs = [
+        _search_flow(
+            query,
+            input_model=input_model,
+            subreddit=input_model.searchCommunityName,
+        )
+        for query in input_model.searches
+    ]
+    async for item in fan_out(jobs):
+        yield item
+
+
+async def scrape_reddit(
+    input_model: RedditScrapeInput, *, limit: int | None = None
+) -> list[dict[str, Any]]:
+    """Collect :func:`iter_reddit` into a list, honoring an optional ``limit``.
+
+    ``limit`` is a request-time policy guard, NOT a ceiling in the streaming
+    core.
+    """
+    results: list[dict[str, Any]] = []
+    async for item in iter_reddit(input_model):
+        results.append(item)
+        if limit is not None and len(results) >= limit:
+            break
+    return results

From 3956045221577d1ba10f08bc3c7cf734b3f4140f Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:27:44 +0530
Subject: [PATCH 07/16] feat(native-connector): classify reddit scrape urls

---
 .../scrapers/reddit/url_resolver.py           | 89 +++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py

diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py b/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py
new file mode 100644
index 000000000..3b6d4e442
--- /dev/null
+++ b/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py
@@ -0,0 +1,89 @@
+"""Classify a Reddit URL into a scrape job.
+
+Covers the ``startUrls`` shapes the Apify spec accepts: a post (permalink), a
+subreddit listing, a user profile, and a search-results page. Non-Reddit hosts
+resolve to ``None`` so the orchestrator can skip them. Mirrors the frozen
+``ResolvedUrl`` dataclass shape of ``../youtube/url_resolver.py``, widened with
+the extra context Reddit flows need (subreddit, sort, user content type).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+from urllib.parse import parse_qs, unquote, urlparse
+
+ResolvedKind = Literal["post", "subreddit", "user", "search"]
+UserContent = Literal["overview", "submitted", "comments"]
+
+_REDDIT_HOSTS = frozenset(
+    {
+        "reddit.com",
+        "www.reddit.com",
+        "old.reddit.com",
+        "new.reddit.com",
+        "np.reddit.com",
+        "m.reddit.com",
+    }
+)
+
+# Listing sorts that can appear as a trailing subreddit path segment.
+_SUBREDDIT_SORTS = frozenset(
+    {"hot", "new", "top", "rising", "controversial", "best"}
+)
+_USER_CONTENT = frozenset({"overview", "submitted", "comments"})
+
+
+@dataclass(frozen=True)
+class ResolvedUrl:
+    kind: ResolvedKind
+    value: str  # post id, subreddit, username, or search query
+    url: str
+    subreddit: str | None = None  # carried for posts and in-sub searches
+    sort: str | None = None  # trailing subreddit sort, if present
+    content: UserContent | None = None  # user profile tab
+
+
+def _is_reddit_host(hostname: str | None) -> bool:
+    return bool(hostname) and hostname.lower() in _REDDIT_HOSTS
+
+
+def resolve_url(url: str) -> ResolvedUrl | None:
+    """Classify a Reddit URL into a scrape job, or ``None`` if unrecognized."""
+    parsed = urlparse(url)
+    if not _is_reddit_host(parsed.hostname):
+        return None
+
+    segments = [s for s in (parsed.path or "").split("/") if s]
+
+    # Search: /search?q=... or /r//search?q=...
+    if segments and segments[-1] == "search":
+        query = parse_qs(parsed.query).get("q", [None])[0]
+        if not query:
+            return None
+        sub = segments[1] if len(segments) >= 3 and segments[0] == "r" else None
+        return ResolvedUrl("search", unquote(query), url, subreddit=sub)
+
+    # Subreddit-scoped URLs: /r//...
+    if len(segments) >= 2 and segments[0] == "r":
+        sub = segments[1]
+        # Post permalink: /r//comments/[/slug]
+        if len(segments) >= 4 and segments[2] == "comments":
+            return ResolvedUrl("post", segments[3], url, subreddit=sub)
+        # Subreddit listing, optional trailing sort: /r/[/]
+        sort = (
+            segments[2]
+            if len(segments) >= 3 and segments[2] in _SUBREDDIT_SORTS
+            else None
+        )
+        return ResolvedUrl("subreddit", sub, url, sort=sort)
+
+    # User profile: /user/[/tab] or /u/[/tab]
+    if len(segments) >= 2 and segments[0] in ("user", "u"):
+        name = segments[1]
+        content: UserContent | None = None
+        if len(segments) >= 3 and segments[2] in _USER_CONTENT:
+            content = segments[2]  # type: ignore[assignment]
+        return ResolvedUrl("user", name, url, content=content)
+
+    return None

From 975fdad0ef4cb0e6f9838bcd5fbe9dc6413d2900 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:27:47 +0530
Subject: [PATCH 08/16] test(native-connector): add live reddit scraper e2e
 check

---
 .../scripts/e2e_reddit_scraper.py             | 213 ++++++++++++++++++
 1 file changed, 213 insertions(+)
 create mode 100644 surfsense_backend/scripts/e2e_reddit_scraper.py

diff --git a/surfsense_backend/scripts/e2e_reddit_scraper.py b/surfsense_backend/scripts/e2e_reddit_scraper.py
new file mode 100644
index 000000000..cdb870a03
--- /dev/null
+++ b/surfsense_backend/scripts/e2e_reddit_scraper.py
@@ -0,0 +1,213 @@
+"""Manual functional e2e for the Reddit scraper (app/proprietary/scrapers/reddit).
+
+Run from the backend directory:
+    cd surfsense_backend
+    uv run python scripts/e2e_reddit_scraper.py
+    # or: .venv/bin/python scripts/e2e_reddit_scraper.py
+
+This is NOT a pytest test (it needs live network + a residential/custom proxy).
+It:
+
+  Step 0 — go/no-go probe (folds in the old scripts/reddit_probe.py): open a
+      proxy session, warm a ``loid`` (svc/shreddit first, old.reddit fallback),
+      then do sequential ``.json`` fetches on the SAME sticky IP and assert each
+      returns a Reddit Listing. If this fails the whole approach is invalid —
+      later steps are skipped.
+  Step 1 — scrape a discovered post URL (post + a few comments).
+  Step 2 — scrape a subreddit listing.
+  Step 3 — run a search query.
+  Step 4 — scrape a user profile.
+  Step 5 — dump trimmed raw ``.json`` fixtures into
+      tests/unit/scrapers/reddit/fixtures/ for the offline parser tests.
+"""
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# --- bootstrap: load .env and put the backend root on sys.path before app.* ---
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(_BACKEND_ROOT))
+for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"):
+    if _candidate.exists():
+        load_dotenv(_candidate)
+        break
+
+from app.proprietary.scrapers.reddit import (  # noqa: E402
+    RedditScrapeInput,
+    scrape_reddit,
+)
+from app.proprietary.scrapers.reddit.fetch import (  # noqa: E402
+    fetch_json,
+    proxy_session,
+    warm_session,
+)
+from app.proprietary.scrapers.reddit.parsers import children  # noqa: E402
+
+_SUBREDDIT = "python"
+_SEARCH_TERM = "async web scraping"
+_USER = "spez"
+
+_FIXTURE_DIR = _BACKEND_ROOT / "tests" / "unit" / "scrapers" / "reddit" / "fixtures"
+
+
+def _hr(title: str) -> None:
+    print(f"\n{'=' * 70}\n{title}\n{'=' * 70}")
+
+
+def _check(label: str, ok: bool, detail: str = "") -> bool:
+    print(f"  [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}")
+    return ok
+
+
+def _listing_count(data) -> int | None:
+    """Child count if ``data`` looks like a Reddit Listing (or post+comments)."""
+    try:
+        if isinstance(data, dict) and data.get("kind") == "Listing":
+            return len(data["data"]["children"])
+        if isinstance(data, list):
+            return sum(
+                len(x["data"]["children"])
+                for x in data
+                if isinstance(x, dict) and x.get("kind") == "Listing"
+            )
+    except Exception:
+        return None
+    return None
+
+
+async def _first_post_permalink() -> str | None:
+    """Discover a live post URL from the subreddit's hot listing."""
+    listing = await fetch_json(f"r/{_SUBREDDIT}/hot", {"limit": 5})
+    kids = children(listing)
+    if not kids:
+        return None
+    permalink = (kids[0].get("data") or {}).get("permalink")
+    return f"https://www.reddit.com{permalink}" if permalink else None
+
+
+async def step0_probe() -> bool:
+    _hr("STEP 0 — go/no-go: loid warm-up + sticky .json")
+    async with proxy_session() as holder:
+        if holder.session is None:
+            return _check(
+                "proxy configured", False, "no proxy -> set PROXY_PROVIDER + creds"
+            )
+        minted = await warm_session(holder.session)
+        holder.warmed = True  # don't let fetch_json re-warm; we just warmed it
+        _check("loid warm-up minted a session", minted)
+        oks: list[bool] = []
+        for path in (f"r/{_SUBREDDIT}/hot", "r/programming/new", f"r/{_SUBREDDIT}/hot"):
+            data = await fetch_json(path, {"limit": 5})
+            n = _listing_count(data)
+            print(f"    {path} -> listing_count={n}")
+            oks.append(n is not None and n > 0)
+            await asyncio.sleep(1.0)
+        return _check("sequential .json on sticky IP", minted and all(oks))
+
+
+async def step1_post() -> bool:
+    _hr("STEP 1 — scrape a discovered post (post + comments)")
+    url = await _first_post_permalink()
+    if not url:
+        return _check("discovered a post URL", False)
+    items = await scrape_reddit(
+        RedditScrapeInput(startUrls=[{"url": url}], maxComments=5)
+    )
+    posts = [i for i in items if i.get("dataType") == "post"]
+    comments = [i for i in items if i.get("dataType") == "comment"]
+    print(f"    posts={len(posts)} comments={len(comments)} url={url}")
+    return _check("post scraped", bool(posts) and bool(posts[0].get("id")))
+
+
+async def step2_subreddit() -> bool:
+    _hr("STEP 2 — scrape a subreddit listing")
+    items = await scrape_reddit(
+        RedditScrapeInput(
+            startUrls=[{"url": f"https://www.reddit.com/r/{_SUBREDDIT}/hot"}],
+            maxPostCount=5,
+            skipComments=True,
+        )
+    )
+    posts = [i for i in items if i.get("dataType") == "post"]
+    for it in posts[:5]:
+        print(f"    - {it.get('id')} | {it.get('title')}")
+    return _check("subreddit returned posts", len(posts) > 0, f"{len(posts)} posts")
+
+
+async def step3_search() -> bool:
+    _hr("STEP 3 — search query")
+    items = await scrape_reddit(
+        RedditScrapeInput(searches=[_SEARCH_TERM], sort="relevance", maxItems=5)
+    )
+    for it in items[:5]:
+        print(f"    - {it.get('id')} | {it.get('title')}")
+    return _check("search returned results", len(items) > 0, f"{len(items)} items")
+
+
+async def step4_user() -> bool:
+    _hr("STEP 4 — user profile")
+    items = await scrape_reddit(
+        RedditScrapeInput(
+            startUrls=[{"url": f"https://www.reddit.com/user/{_USER}"}], maxItems=5
+        )
+    )
+    print(f"    {len(items)} items for u/{_USER}")
+    return _check("user returned items", len(items) > 0, f"{len(items)} items")
+
+
+async def step5_dump_fixtures() -> bool:
+    _hr("STEP 5 — dump trimmed .json fixtures for offline tests")
+    listing = await fetch_json(f"r/{_SUBREDDIT}/hot", {"limit": 25})
+    url = await _first_post_permalink()
+    post = None
+    if url:
+        path = url.split("reddit.com/")[-1].strip("/")
+        post = await fetch_json(path, {"limit": 20})
+
+    _FIXTURE_DIR.mkdir(parents=True, exist_ok=True)
+    wrote = []
+    if _listing_count(listing):
+        (_FIXTURE_DIR / "sample_listing.json").write_text(
+            json.dumps(listing), encoding="utf-8"
+        )
+        wrote.append("sample_listing.json")
+    if isinstance(post, list) and post:
+        (_FIXTURE_DIR / "sample_post.json").write_text(
+            json.dumps(post), encoding="utf-8"
+        )
+        wrote.append("sample_post.json")
+        # A single comment thing, for the comment-mapping fixture.
+        comment_kids = children(post[1]) if len(post) > 1 else []
+        first_comment = next(
+            (c for c in comment_kids if c.get("kind") == "t1"), None
+        )
+        if first_comment:
+            (_FIXTURE_DIR / "sample_comment.json").write_text(
+                json.dumps(first_comment), encoding="utf-8"
+            )
+            wrote.append("sample_comment.json")
+    return _check("dumped fixtures", bool(wrote), f"{wrote} -> {_FIXTURE_DIR}")
+
+
+async def main() -> int:
+    results = [await step0_probe()]
+    if not results[-1]:
+        print("\nloid probe failed — the approach is invalid on this IP/proxy.")
+        print("Aborting remaining steps.")
+        return 1
+    results.append(await step1_post())
+    results.append(await step2_subreddit())
+    results.append(await step3_search())
+    results.append(await step4_user())
+    results.append(await step5_dump_fixtures())
+    _hr("SUMMARY")
+    print(f"  {sum(results)}/{len(results)} steps passed")
+    return 0 if all(results) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

From f14bf1f06dee00b9c8eaf284d1218174271e7f1c Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:27:53 +0530
Subject: [PATCH 09/16] test(native-connector): add reddit scraper test package

---
 surfsense_backend/tests/unit/scrapers/reddit/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 surfsense_backend/tests/unit/scrapers/reddit/__init__.py

diff --git a/surfsense_backend/tests/unit/scrapers/reddit/__init__.py b/surfsense_backend/tests/unit/scrapers/reddit/__init__.py
new file mode 100644
index 000000000..e69de29bb

From 83f52b17b1238746c3b48fd2aa609083be37b315 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:27:56 +0530
Subject: [PATCH 10/16] test(native-connector): add reddit comment fixture

---
 .../tests/unit/scrapers/reddit/fixtures/sample_comment.json      | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_comment.json

diff --git a/surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_comment.json b/surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_comment.json
new file mode 100644
index 000000000..e8d1d60aa
--- /dev/null
+++ b/surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_comment.json
@@ -0,0 +1 @@
+{"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "oprc6qa", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "hannah_G_", "can_mod_post": false, "created_utc": 1780597457.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 4, "author_fullname": "t2_gjk10lou", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "**db-git**\u00a0\\- keep your local database in sync with your git branches.\n\n# What My Project Does\n\n`db-git`\u00a0is a developer tool for projects where database state follows code changes: schema migrations, seed data, experimental feature work, and branch switching during reviews. It installs git\u00a0`post-checkout`\u00a0hook and keeps your local database aligned with the branch you are working on.\n\n* Two workflows:\n   * `shared`: one database, saved and restored per branch\n   * `per-branch`: one database per branch\n* PostgreSQL support today, with plans for more database backends\n* Two PostgreSQL snapshot strategies:\n   * `template`: fast database clones using\u00a0`CREATE DATABASE ... TEMPLATE`\n   * `pgdump`: portable snapshots using\u00a0`pg_dump`\u00a0and\u00a0`pg_restore`\n\n# Target Audience\n\nBackend and full-stack developers who run databases locally and switch branches often, especially on projects where migrations or seed data diverge between branches. It's a local development tool.\n\n# Comparison\n\nThe main things that set\u00a0`db-git`\u00a0apart from existing tools are:\n\n1. It lets you choose per project,\u00a0`shared`\u00a0vs\u00a0`per-branch`, and\u00a0`template`\u00a0vs\u00a0`pgdump`.\n2. It ties database state directly to checkout.\n3. It is not tied to a specific database engine. PostgreSQL is the first supported backend, but the design isn't Postgres-specific, and more databases are planned.\n\n`uv tool install db-git`\n\nGitHub:\u00a0[https://github.com/earthcomfy/db-git](https://github.com/earthcomfy/db-git)\n\nAny feedback is very welcome!", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_oprc6qa", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
db-git\u00a0- keep your local database in sync with your git branches.
\n\n
What My Project Does
\n\n
db-git\u00a0is a developer tool for projects where database state follows code changes: schema migrations, seed data, experimental feature work, and branch switching during reviews. It installs git\u00a0post-checkout\u00a0hook and keeps your local database aligned with the branch you are working on.
\n\n
    \n
  • Two workflows:\n\n
      \n
    • shared: one database, saved and restored per branch
      • \n
    • per-branch: one database per branch
      • \n
    • \n
  • PostgreSQL support today, with plans for more database backends
    • \n
  • Two PostgreSQL snapshot strategies:\n\n
      \n
    • template: fast database clones using\u00a0CREATE DATABASE ... TEMPLATE
      • \n
    • pgdump: portable snapshots using\u00a0pg_dump\u00a0and\u00a0pg_restore
      • \n
    • \n
\n\n
Target Audience
\n\n
Backend and full-stack developers who run databases locally and switch branches often, especially on projects where migrations or seed data diverge between branches. It's a local development tool.
\n\n
Comparison
\n\n
The main things that set\u00a0db-git\u00a0apart from existing tools are:
\n\n
    \n
  1. It lets you choose per project,\u00a0shared\u00a0vs\u00a0per-branch, and\u00a0template\u00a0vs\u00a0pgdump.
    2. \n
  2. It ties database state directly to checkout.
    3. \n
  3. It is not tied to a specific database engine. PostgreSQL is the first supported backend, but the design isn't Postgres-specific, and more databases are planned.
    4. \n
\n\n
uv tool install db-git
\n\n
GitHub:\u00a0https://github.com/earthcomfy/db-git
\n\n
Any feedback is very welcome!
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/oprc6qa/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780597457.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 4}}
\ No newline at end of file

From 672f79d620527df546fa1e792b6b753748897fb9 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:28:02 +0530
Subject: [PATCH 11/16] test(native-connector): add reddit listing fixture

---
 .../tests/unit/scrapers/reddit/fixtures/sample_listing.json      | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_listing.json

diff --git a/surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_listing.json b/surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_listing.json
new file mode 100644
index 000000000..d984552a9
--- /dev/null
+++ b/surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_listing.json
@@ -0,0 +1 @@
+{"kind": "Listing", "data": {"after": "t3_1ugsdwl", "dist": 25, "modhash": "", "geo_filter": null, "children": [{"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "Post all of your code/projects/showcases/AI slop here. \n\nRecycles once a month.", "author_fullname": "t2_6l4z3", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Showcase Thread", "link_flair_richtext": [{"e": "text", "t": "Showcase"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "showcase", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1tws1w7", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.89, "author_flair_background_color": null, "subreddit_type": "public", "ups": 29, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Showcase", "can_mod_post": false, "score": 29, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": true, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1780589106.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Post all of your code/projects/showcases/AI slop here. 
\n\n
Recycles once a month.
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "f35fb004-c1ff-11ee-8305-565bc5d0cc73", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#ff66ac", "id": "1tws1w7", "is_robot_indexable": true, "report_reasons": null, "author": "AutoModerator", "discussion_type": null, "num_comments": 207, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/", "stickied": true, "url": "https://www.reddit.com/r/Python/comments/1tws1w7/showcase_thread/", "subreddit_subscribers": 1493425, "created_utc": 1780589106.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "# Weekly Thread: Resource Request and Sharing \ud83d\udcda\n\nStumbled upon a useful Python resource? Or are you looking for a guide on a specific topic? Welcome to the Resource Request and Sharing thread!\n\n## How it Works:\n\n1. **Request**: Can't find a resource on a particular topic? Ask here!\n2. **Share**: Found something useful? Share it with the community.\n3. **Review**: Give or get opinions on Python resources you've used.\n\n## Guidelines:\n\n* Please include the type of resource (e.g., book, video, article) and the topic.\n* Always be respectful when reviewing someone else's shared resource.\n\n## Example Shares:\n\n1. **Book**: [\"Fluent Python\"](https://www.amazon.com/Fluent-Python-Concise-Effective-Programming/dp/1491946008) \\- Great for understanding Pythonic idioms.\n2. **Video**: [Python Data Structures](https://www.youtube.com/watch?v=pkYVOmU3MgA) \\- Excellent overview of Python's built-in data structures.\n3. **Article**: [Understanding Python Decorators](https://realpython.com/primer-on-python-decorators/) \\- A deep dive into decorators.\n\n## Example Requests:\n\n1. **Looking for**: Video tutorials on web scraping with Python.\n2. **Need**: Book recommendations for Python machine learning.\n\nShare the knowledge, enrich the community. Happy learning! \ud83c\udf1f", "author_fullname": "t2_6l4z3", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Saturday Daily Thread: Resource Request and Sharing! Daily Thread", "link_flair_richtext": [{"a": ":pythonLogo:", "e": "emoji", "u": "https://emoji.redditmedia.com/8yxdpg6xxnr71_t5_2qh0y/pythonLogo"}, {"e": "text", "t": " Daily Thread"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "daily-thread", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1umu29k", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 1.0, "author_flair_background_color": null, "subreddit_type": "public", "ups": 7, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": ":pythonLogo: Daily Thread", "can_mod_post": false, "score": 7, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": true, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "post_hint": "self", "content_categories": null, "is_self": true, "mod_note": null, "created": 1783123215.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Weekly Thread: Resource Request and Sharing \ud83d\udcda
\n\n
Stumbled upon a useful Python resource? Or are you looking for a guide on a specific topic? Welcome to the Resource Request and Sharing thread!
\n\n
How it Works:
\n\n
    \n
  1. Request: Can't find a resource on a particular topic? Ask here!
    2. \n
  2. Share: Found something useful? Share it with the community.
    3. \n
  3. Review: Give or get opinions on Python resources you've used.
    4. \n
\n\n
Guidelines:
\n\n
    \n
  • Please include the type of resource (e.g., book, video, article) and the topic.
    • \n
  • Always be respectful when reviewing someone else's shared resource.
    • \n
\n\n
Example Shares:
\n\n
    \n
  1. Book: "Fluent Python" - Great for understanding Pythonic idioms.
    2. \n
  2. Video: Python Data Structures - Excellent overview of Python's built-in data structures.
    3. \n
  3. Article: Understanding Python Decorators - A deep dive into decorators.
    4. \n
\n\n
Example Requests:
\n\n
    \n
  1. Looking for: Video tutorials on web scraping with Python.
    2. \n
  2. Need: Book recommendations for Python machine learning.
    3. \n
\n\n
Share the knowledge, enrich the community. Happy learning! \ud83c\udf1f
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "preview": {"images": [{"source": {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?auto=webp&s=8cdb17f0919f23f3fc3c0bd9dac21cd40118adda", "width": 1910, "height": 1000}, "resolutions": [{"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=108&crop=smart&auto=webp&s=c7ef9713fb4fbf51d0d7da30fb558f95324a395b", "width": 108, "height": 56}, {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=216&crop=smart&auto=webp&s=70f4ef0366eafa569960666b4537977954dc4da4", "width": 216, "height": 113}, {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=320&crop=smart&auto=webp&s=e88e6f574ea2b6abf3644be5140a1ed8ad6d613c", "width": 320, "height": 167}, {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=640&crop=smart&auto=webp&s=290ace7209dd3df0a237ec970a6a8b1662d523e1", "width": 640, "height": 335}, {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=960&crop=smart&auto=webp&s=421952297faebb04d1038184216c053ab1f0bb56", "width": 960, "height": 502}, {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=1080&crop=smart&auto=webp&s=2e3704dd3e397c6dbebe004c6cce33e8cd82d316", "width": 1080, "height": 565}], "variants": {}, "id": "wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI"}], "enabled": false}, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "6c024934-de3f-11ea-a05a-0ea86b2be9a1", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#00a6a5", "id": "1umu29k", "is_robot_indexable": true, "report_reasons": null, "author": "AutoModerator", "discussion_type": null, "num_comments": 2, "send_replies": false, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1umu29k/saturday_daily_thread_resource_request_and/", "stickied": true, "url": "https://www.reddit.com/r/Python/comments/1umu29k/saturday_daily_thread_resource_request_and/", "subreddit_subscribers": 1493425, "created_utc": 1783123215.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "I have this really messy code scattered in 5 files, it s not that long abt 4000 lines.  \nadding a new feature started feeling like a pain,  \nI hate to deliver this tomorrow, and it s not meant to be scalable.  \nshould i refactor or add the features ? \n\nor just add the features and keep the code structure unchanged", "author_fullname": "t2_2h7k9mb406", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "refactor or don't touch", "link_flair_richtext": [{"e": "text", "t": "Discussion"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "discussion", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1umxiti", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.39, "author_flair_background_color": null, "subreddit_type": "public", "ups": 0, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Discussion", "can_mod_post": false, "score": 0, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1783133390.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
I have this really messy code scattered in 5 files, it s not that long abt 4000 lines.
\nadding a new feature started feeling like a pain,
\nI hate to deliver this tomorrow, and it s not meant to be scalable.
\nshould i refactor or add the features ? 
\n\n
or just add the features and keep the code structure unchanged
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": true, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "0df42996-1c5e-11ea-b1a0-0e44e1c5b731", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#f50057", "id": "1umxiti", "is_robot_indexable": true, "report_reasons": null, "author": "Negative_Pay_2940", "discussion_type": null, "num_comments": 19, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1umxiti/refactor_or_dont_touch/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1umxiti/refactor_or_dont_touch/", "subreddit_subscribers": 1493425, "created_utc": 1783133390.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "I wrote [a practical article](https://modernpython.io/serving-a-frontend-with-fastapi-a-practical-guide/) about FastAPI's `app.frontend()` feature.\n\nThe interesting bit is that it serves static frontend build output as low-priority routes, so normal FastAPI API endpoints still win.\n\nThe article covers:\n\n* `app.frontend(\"/\", directory=\"dist\")`\n* SPA fallback with `fallback=\"index.html\"`\n* how it differs from `StaticFiles`\n* serving under a prefix with `APIRouter`\n* a complete mini dashboard example with FastAPI + vanilla JS\n\n", "author_fullname": "t2_2g9w5mkd56", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "FastAPI app.frontend(): serving a frontend build from the same Python app", "link_flair_richtext": [{"e": "text", "t": "News"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "news", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uleil4", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.87, "author_flair_background_color": null, "subreddit_type": "public", "ups": 43, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "News", "can_mod_post": false, "score": 43, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "post_hint": "self", "content_categories": null, "is_self": true, "mod_note": null, "created": 1782988324.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
I wrote a practical article about FastAPI's app.frontend() feature.
\n\n
The interesting bit is that it serves static frontend build output as low-priority routes, so normal FastAPI API endpoints still win.
\n\n
The article covers:
\n\n
    \n
  • app.frontend("/", directory="dist")
    • \n
  • SPA fallback with fallback="index.html"
    • \n
  • how it differs from StaticFiles
    • \n
  • serving under a prefix with APIRouter
    • \n
  • a complete mini dashboard example with FastAPI + vanilla JS
    • \n
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "preview": {"images": [{"source": {"url": "https://external-preview.redd.it/W-WAtZE5DKslM0Jc7ztGVYI5ALKI9kdfdg2SO7Is-Gk.png?auto=webp&s=da9c531b534cd2ce2779c8005b7153f7f653124e", "width": 1200, "height": 800}, "resolutions": [{"url": "https://external-preview.redd.it/W-WAtZE5DKslM0Jc7ztGVYI5ALKI9kdfdg2SO7Is-Gk.png?width=108&crop=smart&auto=webp&s=8eabae27f78ffb0229c98205ef3c572e76984bf4", "width": 108, "height": 72}, {"url": "https://external-preview.redd.it/W-WAtZE5DKslM0Jc7ztGVYI5ALKI9kdfdg2SO7Is-Gk.png?width=216&crop=smart&auto=webp&s=f9e91bf257102fca1a361ceb8a0787edcffcf92c", "width": 216, "height": 144}, {"url": "https://external-preview.redd.it/W-WAtZE5DKslM0Jc7ztGVYI5ALKI9kdfdg2SO7Is-Gk.png?width=320&crop=smart&auto=webp&s=991b2400e6f7a30ba2840c5a2863a0c6b26c11b4", "width": 320, "height": 213}, {"url": "https://external-preview.redd.it/W-WAtZE5DKslM0Jc7ztGVYI5ALKI9kdfdg2SO7Is-Gk.png?width=640&crop=smart&auto=webp&s=6a2d8086326a1d4535d2dceaaf57c682c3937cc7", "width": 640, "height": 426}, {"url": "https://external-preview.redd.it/W-WAtZE5DKslM0Jc7ztGVYI5ALKI9kdfdg2SO7Is-Gk.png?width=960&crop=smart&auto=webp&s=da8e0ad5c59939e0d9056a5c02da7a9c0e67fae0", "width": 960, "height": 640}, {"url": "https://external-preview.redd.it/W-WAtZE5DKslM0Jc7ztGVYI5ALKI9kdfdg2SO7Is-Gk.png?width=1080&crop=smart&auto=webp&s=cdb6a73a729564d7890372379f04bc7928ca2e28", "width": 1080, "height": 720}], "variants": {}, "id": "W-WAtZE5DKslM0Jc7ztGVYI5ALKI9kdfdg2SO7Is-Gk"}], "enabled": false}, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "0ad780a0-1c5e-11ea-978c-0ee7bacb2bff", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#7193ff", "id": "1uleil4", "is_robot_indexable": true, "report_reasons": null, "author": "ModernPython", "discussion_type": null, "num_comments": 13, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uleil4/fastapi_appfrontend_serving_a_frontend_build_from/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uleil4/fastapi_appfrontend_serving_a_frontend_build_from/", "subreddit_subscribers": 1493425, "created_utc": 1782988324.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "# Weekly Thread: Meta Discussions and Free Talk Friday \ud83c\udf99\ufe0f\n\nWelcome to Free Talk Friday on /r/Python! This is the place to discuss the r/Python community (meta discussions), Python news, projects, or anything else Python-related!\n\n## How it Works:\n\n1. **Open Mic**: Share your thoughts, questions, or anything you'd like related to Python or the community.\n2. **Community Pulse**: Discuss what you feel is working well or what could be improved in the /r/python community.\n3. **News & Updates**: Keep up-to-date with the latest in Python and share any news you find interesting.\n\n## Guidelines:\n\n* All topics should be related to Python or the /r/python community.\n* Be respectful and follow Reddit's [Code of Conduct](https://www.redditinc.com/policies/content-policy).\n\n## Example Topics:\n\n1. **New Python Release**: What do you think about the new features in Python 3.11?\n2. **Community Events**: Any Python meetups or webinars coming up?\n3. **Learning Resources**: Found a great Python tutorial? Share it here!\n4. **Job Market**: How has Python impacted your career?\n5. **Hot Takes**: Got a controversial Python opinion? Let's hear it!\n6. **Community Ideas**: Something you'd like to see us do? tell us.\n\nLet's keep the conversation going. Happy discussing! \ud83c\udf1f", "author_fullname": "t2_6l4z3", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Friday Daily Thread: r/Python Meta and Free-Talk Fridays", "link_flair_richtext": [{"a": ":pythonLogo:", "e": "emoji", "u": "https://emoji.redditmedia.com/8yxdpg6xxnr71_t5_2qh0y/pythonLogo"}, {"e": "text", "t": " Daily Thread"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "daily-thread", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1ulyrvh", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.86, "author_flair_background_color": null, "subreddit_type": "public", "ups": 5, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": ":pythonLogo: Daily Thread", "can_mod_post": false, "score": 5, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": true, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1783036824.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Weekly Thread: Meta Discussions and Free Talk Friday \ud83c\udf99\ufe0f
\n\n
Welcome to Free Talk Friday on /r/Python! This is the place to discuss the r/Python community (meta discussions), Python news, projects, or anything else Python-related!
\n\n
How it Works:
\n\n
    \n
  1. Open Mic: Share your thoughts, questions, or anything you'd like related to Python or the community.
    2. \n
  2. Community Pulse: Discuss what you feel is working well or what could be improved in the /r/python community.
    3. \n
  3. News & Updates: Keep up-to-date with the latest in Python and share any news you find interesting.
    4. \n
\n\n
Guidelines:
\n\n
    \n
  • All topics should be related to Python or the /r/python community.
    • \n
  • Be respectful and follow Reddit's Code of Conduct.
    • \n
\n\n
Example Topics:
\n\n
    \n
  1. New Python Release: What do you think about the new features in Python 3.11?
    2. \n
  2. Community Events: Any Python meetups or webinars coming up?
    3. \n
  3. Learning Resources: Found a great Python tutorial? Share it here!
    4. \n
  4. Job Market: How has Python impacted your career?
    5. \n
  5. Hot Takes: Got a controversial Python opinion? Let's hear it!
    6. \n
  6. Community Ideas: Something you'd like to see us do? tell us.
    7. \n
\n\n
Let's keep the conversation going. Happy discussing! \ud83c\udf1f
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "6c024934-de3f-11ea-a05a-0ea86b2be9a1", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#00a6a5", "id": "1ulyrvh", "is_robot_indexable": true, "report_reasons": null, "author": "AutoModerator", "discussion_type": null, "num_comments": 2, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1ulyrvh/friday_daily_thread_rpython_meta_and_freetalk/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1ulyrvh/friday_daily_thread_rpython_meta_and_freetalk/", "subreddit_subscribers": 1493425, "created_utc": 1783036824.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "Running Celery on AWS ECS can be trickier than it seems if you want to avoid lost tasks and ensure all work is completed. Especially if you're frequently deploying to production and using autoscaling.\n\nThere are two main components for reliable processing:\n- Celery configuration updates\n- Structuring tasks\n\nFor Celery, you should update the following settings:\n- task_acks_late -> True: To treat tasks as successfully processed only after processing. Otherwise, tasks are not retried.\n- task_reject_on_worker_lost -> True: To ensure tasks are retried if workers die for any reason (e.g., warm shutdown + SIGKILL).\n- worker_prefetch_multiplier -> 1: To avoid unnecessarily delayed tasks.\n- broker_connection_retry_on_startup -> True: To make startups more reliable.\n- broker_transport_options -> {\"confirm_publish\": True}: To avoid unsubmitted tasks due to message transport issues.\n- Make sure exponential retries are enabled. This way, you ensure that tasks are retried in the event of an interruption.\n\nFor structuring tasks, use the following two approaches:\n- Batching: Instead of doing all the work at once, you split the work into batches. e.g., Process 1000 users, then submit the next job to process the next 1000 users.\n- Fan out: You can split the work between a \"scheduler\" task and \"execution\" tasks. e.g., One task to list all the users and submit email sending tasks, another task to actually send an email for the selected user\n\nThe same applies to other similar services, such as Heroku and Azure App Containers, which use short grace periods during rolling deployments and downscaling.\n\nYou can read a more elaborate tutorial here: https://jangiacomelli.com/blog/celery-on-aws-ecs/\n", "author_fullname": "t2_5uvfj9zf", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Celery on AWS ECS - prevent lost tasks and ensure the work is always done", "link_flair_richtext": [{"e": "text", "t": "Tutorial"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "tutorial", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uleyez", "quarantine": false, "link_flair_text_color": "dark", "upvote_ratio": 0.82, "author_flair_background_color": null, "subreddit_type": "public", "ups": 31, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Tutorial", "can_mod_post": false, "score": 31, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "post_hint": "self", "content_categories": null, "is_self": true, "mod_note": null, "created": 1782989774.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Running Celery on AWS ECS can be trickier than it seems if you want to avoid lost tasks and ensure all work is completed. Especially if you're frequently deploying to production and using autoscaling.
\n\n
There are two main components for reliable processing:\n- Celery configuration updates\n- Structuring tasks
\n\n
For Celery, you should update the following settings:\n- task_acks_late -> True: To treat tasks as successfully processed only after processing. Otherwise, tasks are not retried.\n- task_reject_on_worker_lost -> True: To ensure tasks are retried if workers die for any reason (e.g., warm shutdown + SIGKILL).\n- worker_prefetch_multiplier -> 1: To avoid unnecessarily delayed tasks.\n- broker_connection_retry_on_startup -> True: To make startups more reliable.\n- broker_transport_options -> {"confirm_publish": True}: To avoid unsubmitted tasks due to message transport issues.\n- Make sure exponential retries are enabled. This way, you ensure that tasks are retried in the event of an interruption.
\n\n
For structuring tasks, use the following two approaches:\n- Batching: Instead of doing all the work at once, you split the work into batches. e.g., Process 1000 users, then submit the next job to process the next 1000 users.\n- Fan out: You can split the work between a "scheduler" task and "execution" tasks. e.g., One task to list all the users and submit email sending tasks, another task to actually send an email for the selected user
\n\n
The same applies to other similar services, such as Heroku and Azure App Containers, which use short grace periods during rolling deployments and downscaling.
\n\n
You can read a more elaborate tutorial here: https://jangiacomelli.com/blog/celery-on-aws-ecs/
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "preview": {"images": [{"source": {"url": "https://external-preview.redd.it/mw6L6MkeGzNGpfloYiy2lOpj4iblwUcNtBT7-YQogdc.png?auto=webp&s=4476a039806bd30ea3113713247eeb17736e2a04", "width": 1200, "height": 630}, "resolutions": [{"url": "https://external-preview.redd.it/mw6L6MkeGzNGpfloYiy2lOpj4iblwUcNtBT7-YQogdc.png?width=108&crop=smart&auto=webp&s=e004a2cd64b3e00a25e30cf9334f09216b2013ca", "width": 108, "height": 56}, {"url": "https://external-preview.redd.it/mw6L6MkeGzNGpfloYiy2lOpj4iblwUcNtBT7-YQogdc.png?width=216&crop=smart&auto=webp&s=f8b05628d10153dae539aa93e3d9672de283ce16", "width": 216, "height": 113}, {"url": "https://external-preview.redd.it/mw6L6MkeGzNGpfloYiy2lOpj4iblwUcNtBT7-YQogdc.png?width=320&crop=smart&auto=webp&s=008f5cc3e3a7b2997f9efce55cdfd4f4b761911a", "width": 320, "height": 168}, {"url": "https://external-preview.redd.it/mw6L6MkeGzNGpfloYiy2lOpj4iblwUcNtBT7-YQogdc.png?width=640&crop=smart&auto=webp&s=16910f1218cbb38fa5649ceec04feb8624bca35e", "width": 640, "height": 336}, {"url": "https://external-preview.redd.it/mw6L6MkeGzNGpfloYiy2lOpj4iblwUcNtBT7-YQogdc.png?width=960&crop=smart&auto=webp&s=5dac9bf645784e560b4aefd42968c8377f26656f", "width": 960, "height": 504}, {"url": "https://external-preview.redd.it/mw6L6MkeGzNGpfloYiy2lOpj4iblwUcNtBT7-YQogdc.png?width=1080&crop=smart&auto=webp&s=904e0e68111e9dafe766adb0b7d95fc53e94668b", "width": 1080, "height": 567}], "variants": {}, "id": "mw6L6MkeGzNGpfloYiy2lOpj4iblwUcNtBT7-YQogdc"}], "enabled": false}, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "7987a74c-04d8-11eb-84ca-0e0ac8b5a78f", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#dadada", "id": "1uleyez", "is_robot_indexable": true, "report_reasons": null, "author": "JanGiacomelli", "discussion_type": null, "num_comments": 13, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uleyez/celery_on_aws_ecs_prevent_lost_tasks_and_ensure/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uleyez/celery_on_aws_ecs_prevent_lost_tasks_and_ensure/", "subreddit_subscribers": 1493425, "created_utc": 1782989774.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "# Weekly Thread: Professional Use, Jobs, and Education \ud83c\udfe2\n\nWelcome to this week's discussion on Python in the professional world! This is your spot to talk about job hunting, career growth, and educational resources in Python. Please note, this thread is **not for recruitment**.\n\n---\n\n## How it Works:\n\n1. **Career Talk**: Discuss using Python in your job, or the job market for Python roles.\n2. **Education Q&A**: Ask or answer questions about Python courses, certifications, and educational resources.\n3. **Workplace Chat**: Share your experiences, challenges, or success stories about using Python professionally.\n\n---\n\n## Guidelines:\n\n- This thread is **not for recruitment**. For job postings, please see r/PythonJobs or the recruitment thread in the sidebar.\n- Keep discussions relevant to Python in the professional and educational context.\n  \n---\n\n## Example Topics:\n\n1. **Career Paths**: What kinds of roles are out there for Python developers?\n2. **Certifications**: Are Python certifications worth it?\n3. **Course Recommendations**: Any good advanced Python courses to recommend?\n4. **Workplace Tools**: What Python libraries are indispensable in your professional work?\n5. **Interview Tips**: What types of Python questions are commonly asked in interviews?\n\n---\n\nLet's help each other grow in our careers and education. Happy discussing! \ud83c\udf1f", "author_fullname": "t2_6l4z3", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Thursday Daily Thread: Python Careers, Courses, and Furthering Education!", "link_flair_richtext": [{"a": ":pythonLogo:", "e": "emoji", "u": "https://emoji.redditmedia.com/8yxdpg6xxnr71_t5_2qh0y/pythonLogo"}, {"e": "text", "t": " Daily Thread"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "daily-thread", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1ul2dky", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.73, "author_flair_background_color": null, "subreddit_type": "public", "ups": 8, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": ":pythonLogo: Daily Thread", "can_mod_post": false, "score": 8, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": true, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782950428.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Weekly Thread: Professional Use, Jobs, and Education \ud83c\udfe2
\n\n
Welcome to this week's discussion on Python in the professional world! This is your spot to talk about job hunting, career growth, and educational resources in Python. Please note, this thread is not for recruitment.
\n\n
\n\n
How it Works:
\n\n
    \n
  1. Career Talk: Discuss using Python in your job, or the job market for Python roles.
    2. \n
  2. Education Q&A: Ask or answer questions about Python courses, certifications, and educational resources.
    3. \n
  3. Workplace Chat: Share your experiences, challenges, or success stories about using Python professionally.
    4. \n
\n\n
\n\n
Guidelines:
\n\n
    \n
  • This thread is not for recruitment. For job postings, please see r/PythonJobs or the recruitment thread in the sidebar.
    • \n
  • Keep discussions relevant to Python in the professional and educational context.
    • \n
\n\n
\n\n
Example Topics:
\n\n
    \n
  1. Career Paths: What kinds of roles are out there for Python developers?
    2. \n
  2. Certifications: Are Python certifications worth it?
    3. \n
  3. Course Recommendations: Any good advanced Python courses to recommend?
    4. \n
  4. Workplace Tools: What Python libraries are indispensable in your professional work?
    5. \n
  5. Interview Tips: What types of Python questions are commonly asked in interviews?
    6. \n
\n\n
\n\n
Let's help each other grow in our careers and education. Happy discussing! \ud83c\udf1f
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "6c024934-de3f-11ea-a05a-0ea86b2be9a1", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": "moderator", "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#00a6a5", "id": "1ul2dky", "is_robot_indexable": true, "report_reasons": null, "author": "AutoModerator", "discussion_type": null, "num_comments": 2, "send_replies": false, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1ul2dky/thursday_daily_thread_python_careers_courses_and/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1ul2dky/thursday_daily_thread_python_careers_courses_and/", "subreddit_subscribers": 1493425, "created_utc": 1782950428.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "A tip that's saved us a lot of boilerplate across our Python stack (Litestar, and our document-extraction tooling): stop decoding JSON into `dict[str, Any]` and casting/`.get()`-ing your way through it. Decode straight into your declared type.\n\n`msgspec` validates and decodes directly into your type at C speed. Quick comparison of the usual options on the same payload:\n\n- `json.loads` / `orjson.loads` -> `dict[str, Any]` (cast and pray; orjson just faster)\n- `pydantic` TypeAdapter(...).validate_json -> your model, validated + rich, but heavier\n- `msgspec.json.decode(raw, type=T)` -> your type, validated, C-fast\n\npydantic does far more and its Rust core is fast; for model-heavy code it's still my default. But on hot paths where you just need decode-into-a-struct, a C decoder going straight to the type is hard to beat.\n\nWith PEP 695 generics the whole (de)serialization layer collapses to one function:\n\n```python\ndef deserialize[T](raw: bytes, t: type[T]) -> T:\n    return msgspec.json.decode(raw, type=t, strict=False)\n\ndeserialize(raw, Grant)        # -> Grant\ndeserialize(raw, list[Grant])  # -> list[Grant]\n```\n\nWe landed on this while building Litestar (msgspec is a big reason it's fast) and reuse it across everything now. How do you handle hot-path decoding \u2014 msgspec, orjson + manual validation, or full pydantic?", "author_fullname": "t2_9t15mit", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Tip: use msgspec for JSON decoding \u2014 it decodes straight into your type at C speed", "link_flair_richtext": [{"e": "text", "t": "Discussion"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "discussion", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1ukh3q5", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.87, "author_flair_background_color": "transparent", "subreddit_type": "public", "ups": 106, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": "67d01c9c-537b-11ee-b0d0-7225f76af176", "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Discussion", "can_mod_post": false, "score": 106, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [{"e": "text", "t": "Pythonista"}], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782899155.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "richtext", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
A tip that's saved us a lot of boilerplate across our Python stack (Litestar, and our document-extraction tooling): stop decoding JSON into dict[str, Any] and casting/.get()-ing your way through it. Decode straight into your declared type.
\n\n
msgspec validates and decodes directly into your type at C speed. Quick comparison of the usual options on the same payload:
\n\n
    \n
  • json.loads / orjson.loads -> dict[str, Any] (cast and pray; orjson just faster)
    • \n
  • pydantic TypeAdapter(...).validate_json -> your model, validated + rich, but heavier
    • \n
  • msgspec.json.decode(raw, type=T) -> your type, validated, C-fast
    • \n
\n\n
pydantic does far more and its Rust core is fast; for model-heavy code it's still my default. But on hot paths where you just need decode-into-a-struct, a C decoder going straight to the type is hard to beat.
\n\n
With PEP 695 generics the whole (de)serialization layer collapses to one function:
\n\n
```python\ndef deserialize[T](raw: bytes, t: type[T]) -> T:\n    return msgspec.json.decode(raw, type=t, strict=False)
\n\n
deserialize(raw, Grant)        # -> Grant\ndeserialize(raw, list[Grant])  # -> list[Grant]\n```
\n\n
We landed on this while building Litestar (msgspec is a big reason it's fast) and reuse it across everything now. How do you handle hot-path decoding \u2014 msgspec, orjson + manual validation, or full pydantic?
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "0df42996-1c5e-11ea-b1a0-0e44e1c5b731", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": "Pythonista", "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#f50057", "id": "1ukh3q5", "is_robot_indexable": true, "report_reasons": null, "author": "Goldziher", "discussion_type": null, "num_comments": 26, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": "dark", "permalink": "/r/Python/comments/1ukh3q5/tip_use_msgspec_for_json_decoding_it_decodes/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1ukh3q5/tip_use_msgspec_for_json_decoding_it_decodes/", "subreddit_subscribers": 1493425, "created_utc": 1782899155.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "I\u2019ve been using the Pythonista IDE by Ole Zorn for over 10 years and I\u2019m just amazed at how consistently good it is. It doesn\u2019t have the latest greatest features but I still use it almost daily. Works with IOS Shortcuts as well. This would be a good one to add to the Wiki.", "author_fullname": "t2_q6jo6", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Pythonista IDE for IOS should be added to the Wiki", "link_flair_richtext": [{"e": "text", "t": "Discussion"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "discussion", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1ul4j9h", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.65, "author_flair_background_color": null, "subreddit_type": "public", "ups": 5, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Discussion", "can_mod_post": false, "score": 5, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782956290.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
I\u2019ve been using the Pythonista IDE by Ole Zorn for over 10 years and I\u2019m just amazed at how consistently good it is. It doesn\u2019t have the latest greatest features but I still use it almost daily. Works with IOS Shortcuts as well. This would be a good one to add to the Wiki.
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "0df42996-1c5e-11ea-b1a0-0e44e1c5b731", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#f50057", "id": "1ul4j9h", "is_robot_indexable": true, "report_reasons": null, "author": "TutorialDoctor", "discussion_type": null, "num_comments": 8, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1ul4j9h/pythonista_ide_for_ios_should_be_added_to_the_wiki/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1ul4j9h/pythonista_ide_for_ios_should_be_added_to_the_wiki/", "subreddit_subscribers": 1493425, "created_utc": 1782956290.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "**The clustering problem with correlated signals**\n\nMy system scores ~40 macro signals (Fed funds rate, yield curve, M2, insider buying, short interest, etc.) and generates a composite \"confluence score\" for a given ticker. The naive approach is to just average the signals. Problem: many signals are correlated \u2014 yield curve and credit spreads move together, insider buying and short interest are often inversely related. Averaging them inflates apparent confidence.\n\nFix I landed on: pairwise Pearson correlation matrix using pandas + numpy on 3 years of weekly signal history. Then `scipy.cluster.hierarchy.linkage` with single-linkage at a 0.6 threshold groups correlated signals into clusters. Each cluster gets one vote, weighted by the cluster member with the best out-of-sample Sharpe ratio on that ticker's 60-day forward returns.\n\n**Streamlit caching gotchas**\n\n`@st.cache_data` is great but has a subtle memory issue: it keeps ALL cached versions until max_entries is hit. For a function that fetches 40 signals with 5 time-period variations, you can end up caching 200+ DataFrames. Added `max_entries=1` to the main signals cache \u2014 memory dropped from ~1.1GB to ~200MB under concurrent load.\n\nAlso: calling `ThreadPoolExecutor` inside a cached function is fine for pure data fetching. But if the cached function spawns threads that themselves call other cached functions, you can hit Streamlit's session state lock. Solution: only parallelize at the outermost uncached layer.\n\n**SEC EDGAR Form 4 XML parsing**\n\nEDGAR serves Form 4 filings as XML, but namespace handling is inconsistent across filings. Some have explicit xmlns declarations, some don't. I strip namespaces with a regex before parsing:\n\n    xml_str = re.sub(r'\\s*xmlns[^\"]*\"[^\"]*\"', '', raw_xml)\n    tree = ET.fromstring(xml_str)\n\nFor insider cluster detection (flagging when 2+ insiders buy within 21 days), I group by issuer CIK, filter for `transactionCode == 'P'` (open-market purchase), then use a rolling window on sorted transaction dates.\n\n**SQLAlchemy Core schema**\n\nUsing SQLAlchemy Core (not ORM) for the main tables: users, signal_snapshots, watchlist_items, alerts. One thing I'm glad I did: a single DATABASE_URL env var that switches between Postgres (prod) and SQLite (local dev). Same schema DDL works for both \u2014 keeps the local dev loop fast.\n\nHappy to answer questions on any of the above.", "author_fullname": "t2_d021y6fm", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Learned a lot building a macro signal scoring system in Python - sharing architecture decisions", "link_flair_richtext": [{"e": "text", "t": "Discussion"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "discussion", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1ul7qiv", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.56, "author_flair_background_color": null, "subreddit_type": "public", "ups": 1, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Discussion", "can_mod_post": false, "score": 1, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782965413.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
The clustering problem with correlated signals
\n\n
My system scores ~40 macro signals (Fed funds rate, yield curve, M2, insider buying, short interest, etc.) and generates a composite "confluence score" for a given ticker. The naive approach is to just average the signals. Problem: many signals are correlated \u2014 yield curve and credit spreads move together, insider buying and short interest are often inversely related. Averaging them inflates apparent confidence.
\n\n
Fix I landed on: pairwise Pearson correlation matrix using pandas + numpy on 3 years of weekly signal history. Then scipy.cluster.hierarchy.linkage with single-linkage at a 0.6 threshold groups correlated signals into clusters. Each cluster gets one vote, weighted by the cluster member with the best out-of-sample Sharpe ratio on that ticker's 60-day forward returns.
\n\n
Streamlit caching gotchas
\n\n
@st.cache_data is great but has a subtle memory issue: it keeps ALL cached versions until max_entries is hit. For a function that fetches 40 signals with 5 time-period variations, you can end up caching 200+ DataFrames. Added max_entries=1 to the main signals cache \u2014 memory dropped from ~1.1GB to ~200MB under concurrent load.
\n\n
Also: calling ThreadPoolExecutor inside a cached function is fine for pure data fetching. But if the cached function spawns threads that themselves call other cached functions, you can hit Streamlit's session state lock. Solution: only parallelize at the outermost uncached layer.
\n\n
SEC EDGAR Form 4 XML parsing
\n\n
EDGAR serves Form 4 filings as XML, but namespace handling is inconsistent across filings. Some have explicit xmlns declarations, some don't. I strip namespaces with a regex before parsing:
\n\n
xml_str = re.sub(r'\\s*xmlns[^"]*"[^"]*"', '', raw_xml)\ntree = ET.fromstring(xml_str)\n
\n\n
For insider cluster detection (flagging when 2+ insiders buy within 21 days), I group by issuer CIK, filter for transactionCode == 'P' (open-market purchase), then use a rolling window on sorted transaction dates.
\n\n
SQLAlchemy Core schema
\n\n
Using SQLAlchemy Core (not ORM) for the main tables: users, signal_snapshots, watchlist_items, alerts. One thing I'm glad I did: a single DATABASE_URL env var that switches between Postgres (prod) and SQLite (local dev). Same schema DDL works for both \u2014 keeps the local dev loop fast.
\n\n
Happy to answer questions on any of the above.
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": true, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "0df42996-1c5e-11ea-b1a0-0e44e1c5b731", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#f50057", "id": "1ul7qiv", "is_robot_indexable": true, "report_reasons": null, "author": "Historical_Ad9654", "discussion_type": null, "num_comments": 2, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1ul7qiv/learned_a_lot_building_a_macro_signal_scoring/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1ul7qiv/learned_a_lot_building_a_macro_signal_scoring/", "subreddit_subscribers": 1493425, "created_utc": 1782965413.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "I saw one yesterday during an interview and it really confused me at first since the feature has been deprecated for so long.\n\n  \nAre there still code bases out there running Python 2? I used Python 3.8 at my last job and that made me feel like a dinosaur. ", "author_fullname": "t2_1q305nm7", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "When's the last time you saw Python 2 Super() syntax?", "link_flair_richtext": [{"e": "text", "t": "Discussion"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "discussion", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uk3ym6", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.87, "author_flair_background_color": null, "subreddit_type": "public", "ups": 103, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Discussion", "can_mod_post": false, "score": 103, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782859080.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
I saw one yesterday during an interview and it really confused me at first since the feature has been deprecated for so long.
\n\n
Are there still code bases out there running Python 2? I used Python 3.8 at my last job and that made me feel like a dinosaur. 
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "0df42996-1c5e-11ea-b1a0-0e44e1c5b731", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#f50057", "id": "1uk3ym6", "is_robot_indexable": true, "report_reasons": null, "author": "GongtingLover", "discussion_type": null, "num_comments": 86, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uk3ym6/whens_the_last_time_you_saw_python_2_super_syntax/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uk3ym6/whens_the_last_time_you_saw_python_2_super_syntax/", "subreddit_subscribers": 1493425, "created_utc": 1782859080.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "The\u00a0Triple Product Property *(*TPP) algorithm is an obscure matmul algorithm that uses group theory (instead of linear algebra) to find matrix products.\n\nOne may summarize it as a fast fourier transform for multiplying matrices. The algorithm was published by Microsoft and Caltech researchers in 2003 but the original paper's math-heavy. I coded the paper in Python to make matrix multiplication research accessible to everyone.\n\nGitHub: [https://github.com/MurageKibicho/The-Annotated-Triple-Product-Property-Matrix-Multiplication-Algorithm/tree/main](https://github.com/MurageKibicho/The-Annotated-Triple-Product-Property-Matrix-Multiplication-Algorithm/tree/main)\n\nWritten Guide: [https://leetarxiv.substack.com/p/triple-product-property-matrix-multiplication](https://leetarxiv.substack.com/p/triple-product-property-matrix-multiplication)\n\n", "author_fullname": "t2_3xvmpjwh9", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Annotated Triple Product Property Matrix Multiplication Algorithm In Python", "link_flair_richtext": [{"e": "text", "t": "Tutorial"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "tutorial", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uknaq3", "quarantine": false, "link_flair_text_color": "dark", "upvote_ratio": 0.67, "author_flair_background_color": null, "subreddit_type": "public", "ups": 1, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Tutorial", "can_mod_post": false, "score": 1, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782916193.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
The\u00a0Triple Product Property (TPP) algorithm is an obscure matmul algorithm that uses group theory (instead of linear algebra) to find matrix products.
\n\n
One may summarize it as a fast fourier transform for multiplying matrices. The algorithm was published by Microsoft and Caltech researchers in 2003 but the original paper's math-heavy. I coded the paper in Python to make matrix multiplication research accessible to everyone.
\n\n
GitHub: https://github.com/MurageKibicho/The-Annotated-Triple-Product-Property-Matrix-Multiplication-Algorithm/tree/main
\n\n
Written Guide: https://leetarxiv.substack.com/p/triple-product-property-matrix-multiplication
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": true, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "7987a74c-04d8-11eb-84ca-0e0ac8b5a78f", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#dadada", "id": "1uknaq3", "is_robot_indexable": true, "report_reasons": null, "author": "DataBaeBee", "discussion_type": null, "num_comments": 0, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uknaq3/annotated_triple_product_property_matrix/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uknaq3/annotated_triple_product_property_matrix/", "subreddit_subscribers": 1493425, "created_utc": 1782916193.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "I've been experimenting with AI agents and autocomplete platforms for a greenfield FastAPI project. In the first few weeks, it felt incredibly fast. But now that we've scaled to multiple routers, complex Pydantic schemas, and SQLAlchemy models, the structural debt is piling up.\n\nThe AI writes code that functions, but it constantly violates our architecture. It'll put complex business logic inside a route handler instead of the service layer, or it'll mess up async database sessions across modules. I find myself spending more time refactoring the structure of what it built than it would have taken to write the logic myself.\n\nIs anyone else hitting this scaling wall where AI utility drops off as codebase complexity grows? How are you keeping your system architecture clean?", "author_fullname": "t2_4xeh5jtc", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Mitigating \"architectural drift\" in large Python backend codebases using AI tools", "link_flair_richtext": [{"e": "text", "t": "Discussion"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "discussion", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1ujy1a2", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.73, "author_flair_background_color": null, "subreddit_type": "public", "ups": 32, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Discussion", "can_mod_post": false, "score": 32, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782845636.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
I've been experimenting with AI agents and autocomplete platforms for a greenfield FastAPI project. In the first few weeks, it felt incredibly fast. But now that we've scaled to multiple routers, complex Pydantic schemas, and SQLAlchemy models, the structural debt is piling up.
\n\n
The AI writes code that functions, but it constantly violates our architecture. It'll put complex business logic inside a route handler instead of the service layer, or it'll mess up async database sessions across modules. I find myself spending more time refactoring the structure of what it built than it would have taken to write the logic myself.
\n\n
Is anyone else hitting this scaling wall where AI utility drops off as codebase complexity grows? How are you keeping your system architecture clean?
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "0df42996-1c5e-11ea-b1a0-0e44e1c5b731", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#f50057", "id": "1ujy1a2", "is_robot_indexable": true, "report_reasons": null, "author": "CrazyGeek7", "discussion_type": null, "num_comments": 42, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1ujy1a2/mitigating_architectural_drift_in_large_python/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1ujy1a2/mitigating_architectural_drift_in_large_python/", "subreddit_subscribers": 1493425, "created_utc": 1782845636.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "Please read this and contribute your opinions so that this becomes basic foundation for spring devs who adapt to python .\n\nhttps://bunny-learner.github.io/Python-Handbook-for-Spring-Devs/", "author_fullname": "t2_1gpwvz9l5s", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Python handbook for spring devs", "link_flair_richtext": [{"e": "text", "t": "Resource"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "resource", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1ukn0wi", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.25, "author_flair_background_color": null, "subreddit_type": "public", "ups": 0, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Resource", "can_mod_post": false, "score": 0, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782915570.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Please read this and contribute your opinions so that this becomes basic foundation for spring devs who adapt to python .
\n\n
https://bunny-learner.github.io/Python-Handbook-for-Spring-Devs/
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": true, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "f9716fb2-4113-11ea-a3f1-0ef51f60f757", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#ddbd37", "id": "1ukn0wi", "is_robot_indexable": true, "report_reasons": null, "author": "External-Wait-2583", "discussion_type": null, "num_comments": 2, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1ukn0wi/python_handbook_for_spring_devs/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1ukn0wi/python_handbook_for_spring_devs/", "subreddit_subscribers": 1493425, "created_utc": 1782915570.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "I just spent three hours trying to get a basic python cron script to send out a weekly web scraping summary. used to just use `smtplib` and a random gmail app password but google basically killed that workflow  \n  \nTried installing the official python sdk for one of the big email providers and it pulled in like 6 different async dependencies just to send a plain text string. It is genuinely insane how bloated the modern python ecosystem has gotten for the most basic tasks  \n  \nI ended up just writing a simple `requests.post()` webhook over to yaplet to handle the actual subscriber list and formatting because I absolutely refuse to fight with another bloated `__init__.py` or dns auth protocol this month  \n  \nsometimes it really feels like we spend 10% of our time writing actual python logic and 90% fighting with enterprise api wrappers tbh", "author_fullname": "t2_81nfn6e5d", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Why is sending an automated email with python still a nightmare in 2026", "link_flair_richtext": [{"e": "text", "t": "Discussion"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "discussion", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1ukljb6", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.25, "author_flair_background_color": null, "subreddit_type": "public", "ups": 0, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Discussion", "can_mod_post": false, "score": 0, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782912026.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
I just spent three hours trying to get a basic python cron script to send out a weekly web scraping summary. used to just use smtplib and a random gmail app password but google basically killed that workflow  
\n\n
Tried installing the official python sdk for one of the big email providers and it pulled in like 6 different async dependencies just to send a plain text string. It is genuinely insane how bloated the modern python ecosystem has gotten for the most basic tasks  
\n\n
I ended up just writing a simple requests.post() webhook over to yaplet to handle the actual subscriber list and formatting because I absolutely refuse to fight with another bloated __init__.py or dns auth protocol this month  
\n\n
sometimes it really feels like we spend 10% of our time writing actual python logic and 90% fighting with enterprise api wrappers tbh
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": true, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "0df42996-1c5e-11ea-b1a0-0e44e1c5b731", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#f50057", "id": "1ukljb6", "is_robot_indexable": true, "report_reasons": null, "author": "Crystallover1991", "discussion_type": null, "num_comments": 20, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1ukljb6/why_is_sending_an_automated_email_with_python/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1ukljb6/why_is_sending_an_automated_email_with_python/", "subreddit_subscribers": 1493425, "created_utc": 1782912026.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "This is the first part of a multi-part series exploring why `async/await` might not be the best concurrency pattern for most use cases, and what alternative models you should consider instead. Using Python for our practical examples, this opening post digs into the roots of `async/await`, guiding you through building a custom event loop from scratch using generators.\n\n[https://theblog.info/posts/asyncawait-is-a-plague-part-1-roots](https://theblog.info/posts/asyncawait-is-a-plague-part-1-roots)\n\n\n**Note:** This is Part 1 of a multi-part series. Instead of diving straight into why `async/await` can be problematic, this post explores the original motivations behind the pattern. Understanding how it works under the hood will provide the essential context for the issues we'll discuss in upcoming parts.", "author_fullname": "t2_rkfihkkzk", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Async/Await is a Plague: Part 1 Roots", "link_flair_richtext": [{"e": "text", "t": "Discussion"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "discussion", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uisy46", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.7, "author_flair_background_color": null, "subreddit_type": "public", "ups": 73, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Discussion", "can_mod_post": false, "score": 73, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": 1782753438.0, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "post_hint": "self", "content_categories": null, "is_self": true, "mod_note": null, "created": 1782740712.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
This is the first part of a multi-part series exploring why async/await might not be the best concurrency pattern for most use cases, and what alternative models you should consider instead. Using Python for our practical examples, this opening post digs into the roots of async/await, guiding you through building a custom event loop from scratch using generators.
\n\n
https://theblog.info/posts/asyncawait-is-a-plague-part-1-roots
\n\n
Note: This is Part 1 of a multi-part series. Instead of diving straight into why async/await can be problematic, this post explores the original motivations behind the pattern. Understanding how it works under the hood will provide the essential context for the issues we'll discuss in upcoming parts.
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "preview": {"images": [{"source": {"url": "https://external-preview.redd.it/o5iToZglarhA_St0cCPapqc_aoL8yr3wvMq-nTIXbBY.jpeg?auto=webp&s=8c085a9706bd5c46bf2669a92aec2fe9a968b931", "width": 640, "height": 640}, "resolutions": [{"url": "https://external-preview.redd.it/o5iToZglarhA_St0cCPapqc_aoL8yr3wvMq-nTIXbBY.jpeg?width=108&crop=smart&auto=webp&s=2ee3e5583e9679af89b4127ff8ef801ad96d7c21", "width": 108, "height": 108}, {"url": "https://external-preview.redd.it/o5iToZglarhA_St0cCPapqc_aoL8yr3wvMq-nTIXbBY.jpeg?width=216&crop=smart&auto=webp&s=6855cb906f7b5e61cd39e971a6813b92089c9ddc", "width": 216, "height": 216}, {"url": "https://external-preview.redd.it/o5iToZglarhA_St0cCPapqc_aoL8yr3wvMq-nTIXbBY.jpeg?width=320&crop=smart&auto=webp&s=6cc8885184c29b6a2bf5b3cda04f8e48f141681b", "width": 320, "height": 320}, {"url": "https://external-preview.redd.it/o5iToZglarhA_St0cCPapqc_aoL8yr3wvMq-nTIXbBY.jpeg?width=640&crop=smart&auto=webp&s=ed1a0969eb982fc4f33d3b725973b3d4a9165394", "width": 640, "height": 640}], "variants": {}, "id": "o5iToZglarhA_St0cCPapqc_aoL8yr3wvMq-nTIXbBY"}], "enabled": false}, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "0df42996-1c5e-11ea-b1a0-0e44e1c5b731", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#f50057", "id": "1uisy46", "is_robot_indexable": true, "report_reasons": null, "author": "EntryNo8040", "discussion_type": null, "num_comments": 71, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uisy46/asyncawait_is_a_plague_part_1_roots/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uisy46/asyncawait_is_a_plague_part_1_roots/", "subreddit_subscribers": 1493425, "created_utc": 1782740712.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "# Weekly Wednesday Thread: Advanced Questions \ud83d\udc0d\n\nDive deep into Python with our Advanced Questions thread! This space is reserved for questions about more advanced Python topics, frameworks, and best practices.\n\n## How it Works:\n\n1. **Ask Away**: Post your advanced Python questions here.\n2. **Expert Insights**: Get answers from experienced developers.\n3. **Resource Pool**: Share or discover tutorials, articles, and tips.\n\n## Guidelines:\n\n* This thread is for **advanced questions only**. Beginner questions are welcome in our [Daily Beginner Thread](#daily-beginner-thread-link) every Thursday.\n* Questions that are not advanced may be removed and redirected to the appropriate thread.\n\n## Recommended Resources:\n\n* If you don't receive a response, consider exploring r/LearnPython or join the [Python Discord Server](https://discord.gg/python) for quicker assistance.\n\n## Example Questions:\n\n1. **How can you implement a custom memory allocator in Python?**\n2. **What are the best practices for optimizing Cython code for heavy numerical computations?**\n3. **How do you set up a multi-threaded architecture using Python's Global Interpreter Lock (GIL)?**\n4. **Can you explain the intricacies of metaclasses and how they influence object-oriented design in Python?**\n5. **How would you go about implementing a distributed task queue using Celery and RabbitMQ?**\n6. **What are some advanced use-cases for Python's decorators?**\n7. **How can you achieve real-time data streaming in Python with WebSockets?**\n8. **What are the performance implications of using native Python data structures vs NumPy arrays for large-scale data?**\n9. **Best practices for securing a Flask (or similar) REST API with OAuth 2.0?**\n10. **What are the best practices for using Python in a microservices architecture? (..and more generally, should I even use microservices?)**\n\nLet's deepen our Python knowledge together. Happy coding! \ud83c\udf1f", "author_fullname": "t2_6l4z3", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Tuesday Daily Thread: Advanced questions", "link_flair_richtext": [{"a": ":pythonLogo:", "e": "emoji", "u": "https://emoji.redditmedia.com/8yxdpg6xxnr71_t5_2qh0y/pythonLogo"}, {"e": "text", "t": " Daily Thread"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "daily-thread", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uj99ws", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.67, "author_flair_background_color": null, "subreddit_type": "public", "ups": 3, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": ":pythonLogo: Daily Thread", "can_mod_post": false, "score": 3, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": true, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "post_hint": "self", "content_categories": null, "is_self": true, "mod_note": null, "created": 1782777606.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Weekly Wednesday Thread: Advanced Questions \ud83d\udc0d
\n\n
Dive deep into Python with our Advanced Questions thread! This space is reserved for questions about more advanced Python topics, frameworks, and best practices.
\n\n
How it Works:
\n\n
    \n
  1. Ask Away: Post your advanced Python questions here.
    2. \n
  2. Expert Insights: Get answers from experienced developers.
    3. \n
  3. Resource Pool: Share or discover tutorials, articles, and tips.
    4. \n
\n\n
Guidelines:
\n\n
    \n
  • This thread is for advanced questions only. Beginner questions are welcome in our Daily Beginner Thread every Thursday.
    • \n
  • Questions that are not advanced may be removed and redirected to the appropriate thread.
    • \n
\n\n
Recommended Resources:
\n\n\n\n
Example Questions:
\n\n
    \n
  1. How can you implement a custom memory allocator in Python?
    2. \n
  2. What are the best practices for optimizing Cython code for heavy numerical computations?
    3. \n
  3. How do you set up a multi-threaded architecture using Python's Global Interpreter Lock (GIL)?
    4. \n
  4. Can you explain the intricacies of metaclasses and how they influence object-oriented design in Python?
    5. \n
  5. How would you go about implementing a distributed task queue using Celery and RabbitMQ?
    6. \n
  6. What are some advanced use-cases for Python's decorators?
    7. \n
  7. How can you achieve real-time data streaming in Python with WebSockets?
    8. \n
  8. What are the performance implications of using native Python data structures vs NumPy arrays for large-scale data?
    9. \n
  9. Best practices for securing a Flask (or similar) REST API with OAuth 2.0?
    10. \n
  10. What are the best practices for using Python in a microservices architecture? (..and more generally, should I even use microservices?)
    11. \n
\n\n
Let's deepen our Python knowledge together. Happy coding! \ud83c\udf1f
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "preview": {"images": [{"source": {"url": "https://external-preview.redd.it/Qr0qqr0jX1TjZaLE6_zaVcWJHZQGJPofEJEedmb9MtI.jpeg?auto=webp&s=f0e96c458bf454a41999b79e0f2b4170f1e1c85f", "width": 512, "height": 288}, "resolutions": [{"url": "https://external-preview.redd.it/Qr0qqr0jX1TjZaLE6_zaVcWJHZQGJPofEJEedmb9MtI.jpeg?width=108&crop=smart&auto=webp&s=7726f1bd96bb293ee00f58330b9db8277d3a4e97", "width": 108, "height": 60}, {"url": "https://external-preview.redd.it/Qr0qqr0jX1TjZaLE6_zaVcWJHZQGJPofEJEedmb9MtI.jpeg?width=216&crop=smart&auto=webp&s=7592cc25c340779fdbd514118a98258c5def8dce", "width": 216, "height": 121}, {"url": "https://external-preview.redd.it/Qr0qqr0jX1TjZaLE6_zaVcWJHZQGJPofEJEedmb9MtI.jpeg?width=320&crop=smart&auto=webp&s=6dca793160bdafbbdac9126b927e2b64871738a5", "width": 320, "height": 180}], "variants": {}, "id": "Qr0qqr0jX1TjZaLE6_zaVcWJHZQGJPofEJEedmb9MtI"}], "enabled": false}, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "6c024934-de3f-11ea-a05a-0ea86b2be9a1", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": "moderator", "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#00a6a5", "id": "1uj99ws", "is_robot_indexable": true, "report_reasons": null, "author": "AutoModerator", "discussion_type": null, "num_comments": 4, "send_replies": false, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uj99ws/tuesday_daily_thread_advanced_questions/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uj99ws/tuesday_daily_thread_advanced_questions/", "subreddit_subscribers": 1493425, "created_utc": 1782777606.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "We compared the 3 pip-installable mesh boolean libraries on the task of pairwise mesh booleans at industry scale.\n\nFull write up: [https://polydera.com/algorithms/python-mesh-boolean-libraries-2026](https://polydera.com/algorithms/python-mesh-boolean-libraries-2026)\n\n#### Libraries tested\n\n- **[MeshLib](https://meshlib.io/) 3.1** \u2014 `pip install meshlib`. Simulation of Simplicity for degeneracy handling.\n- **[Manifold](https://github.com/elalish/manifold) 3.5** \u2014 `pip install manifold3d`. Deterministic floating point with symbolic perturbation.\n- **[trueform](https://polydera.com/trueform) 0.9.8** \u2014 `pip install trueform`. Topologically-exact arrangements via a bounded integer kernel. Ships native, python and WebAssembly.\n\nAll three are native cores (C++/Rust) with Python bindings.\n\n#### Protocol\n\nEach library is timed from input arrays (vertices, triangles) to output arrays of the same shape. Native-object construction \u2014 acceleration structures, trees, topology \u2014 plus the boolean, all in the timer. Only file I/O is outside. Best of 5, fresh objects every run; nothing amortised across calls.\n\n**Result agreement.** On every pair the three produced the same solid \u2014 signed volumes agree within floating-point tolerance. trueform and Manifold returned a closed, manifold mesh on all 1000 pairs; MeshLib on 999. The comparison is wall-clock only.\n\n**Corpus.** Random sets of solid, manifold, non-self-intersecting Thingi10K meshes, 200K to 1.5M polygons per operand. Each operand is normalised to unit extent, randomly rotated, and translated so the bounding boxes overlap; each pair takes the union. Thingi10K IDs and per-operand counts for every case are published: [pairwise corpus](https://github.com/polydera/trueform/blob/main/research/uncertainty-aware-mesh-csg/data/pairwise-corpus-ids.json).\n\n**Environment.** Apple M4 Max (arm64), macOS, CPython 3.13. Installed from PyPI: trueform 0.9.8, meshlib 3.1.0.75, manifold3d 3.5.1 \u2014 default builds, default thread count. On Apple Silicon the wheel's compiled architecture matters; all three ship native arm64 builds.\n\n#### Results\n\n**Of the libraries you can `pip install`, trueform was the fastest mesh boolean in Python** \u2014 fastest on every one of the 1000 pairwise pairs.\n\n**Pairwise** \u2014 one boolean per pair across the 1000-pair corpus.\n\n| library | median (ms) | geomean \u00d7 vs trueform | valid / 1000 |\n|---|---:|---:|---:|\n| trueform 0.9.8 | 18.0 | 1.0\u00d7 | 1000 |\n| MeshLib 3.1 | 87.6 | 4.9\u00d7 | 999 |\n| Manifold 3.5 | 120.3 | 6.9\u00d7 | 1000 |\n\n---\n\n*Disclosure: I'm one of the authors of trueform.*", "author_fullname": "t2_81ze3aps", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Comparison and Benchmarks of Python Mesh Boolean Libraries at Industry Scale", "link_flair_richtext": [{"e": "text", "t": "Resource"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "resource", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uins5r", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.79, "author_flair_background_color": null, "subreddit_type": "public", "ups": 8, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Resource", "can_mod_post": false, "score": 8, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": 1782753231.0, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "post_hint": "self", "content_categories": null, "is_self": true, "mod_note": null, "created": 1782725887.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
We compared the 3 pip-installable mesh boolean libraries on the task of pairwise mesh booleans at industry scale.
\n\n
Full write up: https://polydera.com/algorithms/python-mesh-boolean-libraries-2026
\n\n
Libraries tested
\n\n
    \n
  • MeshLib 3.1 \u2014 pip install meshlib. Simulation of Simplicity for degeneracy handling.
    • \n
  • Manifold 3.5 \u2014 pip install manifold3d. Deterministic floating point with symbolic perturbation.
    • \n
  • trueform 0.9.8 \u2014 pip install trueform. Topologically-exact arrangements via a bounded integer kernel. Ships native, python and WebAssembly.
    • \n
\n\n
All three are native cores (C++/Rust) with Python bindings.
\n\n
Protocol
\n\n
Each library is timed from input arrays (vertices, triangles) to output arrays of the same shape. Native-object construction \u2014 acceleration structures, trees, topology \u2014 plus the boolean, all in the timer. Only file I/O is outside. Best of 5, fresh objects every run; nothing amortised across calls.
\n\n
Result agreement. On every pair the three produced the same solid \u2014 signed volumes agree within floating-point tolerance. trueform and Manifold returned a closed, manifold mesh on all 1000 pairs; MeshLib on 999. The comparison is wall-clock only.
\n\n
Corpus. Random sets of solid, manifold, non-self-intersecting Thingi10K meshes, 200K to 1.5M polygons per operand. Each operand is normalised to unit extent, randomly rotated, and translated so the bounding boxes overlap; each pair takes the union. Thingi10K IDs and per-operand counts for every case are published: pairwise corpus.
\n\n
Environment. Apple M4 Max (arm64), macOS, CPython 3.13. Installed from PyPI: trueform 0.9.8, meshlib 3.1.0.75, manifold3d 3.5.1 \u2014 default builds, default thread count. On Apple Silicon the wheel's compiled architecture matters; all three ship native arm64 builds.
\n\n
Results
\n\n
Of the libraries you can pip install, trueform was the fastest mesh boolean in Python \u2014 fastest on every one of the 1000 pairwise pairs.
\n\n
Pairwise \u2014 one boolean per pair across the 1000-pair corpus.
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
librarymedian (ms)geomean \u00d7 vs trueformvalid / 1000
trueform 0.9.818.01.0\u00d71000
MeshLib 3.187.64.9\u00d7999
Manifold 3.5120.36.9\u00d71000
\n\n
\n\n
Disclosure: I'm one of the authors of trueform.
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "preview": {"images": [{"source": {"url": "https://external-preview.redd.it/uP9ziF1Ybs-odwNAz2IxyEjZSvHb6KCbsUD3bZsW7Rk.png?auto=webp&s=9872ff4503eed4216dbc3d09833ceb2ce9893e9d", "width": 1200, "height": 600}, "resolutions": [{"url": "https://external-preview.redd.it/uP9ziF1Ybs-odwNAz2IxyEjZSvHb6KCbsUD3bZsW7Rk.png?width=108&crop=smart&auto=webp&s=48a6c26427e9bcc0b3e033694a4ad7864ae8aec6", "width": 108, "height": 54}, {"url": "https://external-preview.redd.it/uP9ziF1Ybs-odwNAz2IxyEjZSvHb6KCbsUD3bZsW7Rk.png?width=216&crop=smart&auto=webp&s=3448012d2ca17920431ff0b9af75287cb7ddb127", "width": 216, "height": 108}, {"url": "https://external-preview.redd.it/uP9ziF1Ybs-odwNAz2IxyEjZSvHb6KCbsUD3bZsW7Rk.png?width=320&crop=smart&auto=webp&s=e44c7768721ffdd42c421ee7ded2390eeee2dcff", "width": 320, "height": 160}, {"url": "https://external-preview.redd.it/uP9ziF1Ybs-odwNAz2IxyEjZSvHb6KCbsUD3bZsW7Rk.png?width=640&crop=smart&auto=webp&s=bd31638b30f1d7b12f29c5eab21fed8a1aa66bb6", "width": 640, "height": 320}, {"url": "https://external-preview.redd.it/uP9ziF1Ybs-odwNAz2IxyEjZSvHb6KCbsUD3bZsW7Rk.png?width=960&crop=smart&auto=webp&s=ec326eb5a02b7424897a25e5ef64b39c6b20123b", "width": 960, "height": 480}, {"url": "https://external-preview.redd.it/uP9ziF1Ybs-odwNAz2IxyEjZSvHb6KCbsUD3bZsW7Rk.png?width=1080&crop=smart&auto=webp&s=bd5479c4f871219c0a41d93d1ba87ae6d8c6e9c2", "width": 1080, "height": 540}], "variants": {}, "id": "uP9ziF1Ybs-odwNAz2IxyEjZSvHb6KCbsUD3bZsW7Rk"}], "enabled": false}, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "f9716fb2-4113-11ea-a3f1-0ef51f60f757", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#ddbd37", "id": "1uins5r", "is_robot_indexable": true, "report_reasons": null, "author": "Separate-Summer-6027", "discussion_type": null, "num_comments": 4, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uins5r/comparison_and_benchmarks_of_python_mesh_boolean/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uins5r/comparison_and_benchmarks_of_python_mesh_boolean/", "subreddit_subscribers": 1493425, "created_utc": 1782725887.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "# Weekly Thread: Project Ideas \ud83d\udca1\n\nWelcome to our weekly Project Ideas thread! Whether you're a newbie looking for a first project or an expert seeking a new challenge, this is the place for you.\n\n## How it Works:\n\n1. **Suggest a Project**: Comment your project idea\u2014be it beginner-friendly or advanced.\n2. **Build & Share**: If you complete a project, reply to the original comment, share your experience, and attach your source code.\n3. **Explore**: Looking for ideas? Check out Al Sweigart's [\"The Big Book of Small Python Projects\"](https://www.amazon.com/Big-Book-Small-Python-Programming/dp/1718501242) for inspiration.\n\n## Guidelines:\n\n* Clearly state the difficulty level.\n* Provide a brief description and, if possible, outline the tech stack.\n* Feel free to link to tutorials or resources that might help.\n\n# Example Submissions:\n\n## Project Idea: Chatbot\n\n**Difficulty**: Intermediate\n\n**Tech Stack**: Python, NLP, Flask/FastAPI/Litestar \n\n**Description**: Create a chatbot that can answer FAQs for a website.\n\n**Resources**: [Building a Chatbot with Python](https://www.youtube.com/watch?v=a37BL0stIuM)\n\n# Project Idea: Weather Dashboard\n\n**Difficulty**: Beginner\n\n**Tech Stack**: HTML, CSS, JavaScript, API\n\n**Description**: Build a dashboard that displays real-time weather information using a weather API.\n\n**Resources**: [Weather API Tutorial](https://www.youtube.com/watch?v=9P5MY_2i7K8)\n\n## Project Idea: File Organizer\n\n**Difficulty**: Beginner\n\n**Tech Stack**: Python, File I/O\n\n**Description**: Create a script that organizes files in a directory into sub-folders based on file type.\n\n**Resources**: [Automate the Boring Stuff: Organizing Files](https://automatetheboringstuff.com/2e/chapter9/)\n\nLet's help each other grow. Happy coding! \ud83c\udf1f", "author_fullname": "t2_6l4z3", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Monday Daily Thread: Project ideas!", "link_flair_richtext": [{"a": ":pythonLogo:", "e": "emoji", "u": "https://emoji.redditmedia.com/8yxdpg6xxnr71_t5_2qh0y/pythonLogo"}, {"e": "text", "t": " Daily Thread"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "daily-thread", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uictw2", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.67, "author_flair_background_color": null, "subreddit_type": "public", "ups": 4, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": ":pythonLogo: Daily Thread", "can_mod_post": false, "score": 4, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": true, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "post_hint": "self", "content_categories": null, "is_self": true, "mod_note": null, "created": 1782691206.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Weekly Thread: Project Ideas \ud83d\udca1
\n\n
Welcome to our weekly Project Ideas thread! Whether you're a newbie looking for a first project or an expert seeking a new challenge, this is the place for you.
\n\n
How it Works:
\n\n
    \n
  1. Suggest a Project: Comment your project idea\u2014be it beginner-friendly or advanced.
    2. \n
  2. Build & Share: If you complete a project, reply to the original comment, share your experience, and attach your source code.
    3. \n
  3. Explore: Looking for ideas? Check out Al Sweigart's "The Big Book of Small Python Projects" for inspiration.
    4. \n
\n\n
Guidelines:
\n\n
    \n
  • Clearly state the difficulty level.
    • \n
  • Provide a brief description and, if possible, outline the tech stack.
    • \n
  • Feel free to link to tutorials or resources that might help.
    • \n
\n\n
Example Submissions:
\n\n
Project Idea: Chatbot
\n\n
Difficulty: Intermediate
\n\n
Tech Stack: Python, NLP, Flask/FastAPI/Litestar 
\n\n
Description: Create a chatbot that can answer FAQs for a website.
\n\n
Resources: Building a Chatbot with Python
\n\n
Project Idea: Weather Dashboard
\n\n
Difficulty: Beginner
\n\n
Tech Stack: HTML, CSS, JavaScript, API
\n\n
Description: Build a dashboard that displays real-time weather information using a weather API.
\n\n
Resources: Weather API Tutorial
\n\n
Project Idea: File Organizer
\n\n
Difficulty: Beginner
\n\n
Tech Stack: Python, File I/O
\n\n
Description: Create a script that organizes files in a directory into sub-folders based on file type.
\n\n
Resources: Automate the Boring Stuff: Organizing Files
\n\n
Let's help each other grow. Happy coding! \ud83c\udf1f
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "preview": {"images": [{"source": {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?auto=webp&s=8cdb17f0919f23f3fc3c0bd9dac21cd40118adda", "width": 1910, "height": 1000}, "resolutions": [{"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=108&crop=smart&auto=webp&s=c7ef9713fb4fbf51d0d7da30fb558f95324a395b", "width": 108, "height": 56}, {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=216&crop=smart&auto=webp&s=70f4ef0366eafa569960666b4537977954dc4da4", "width": 216, "height": 113}, {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=320&crop=smart&auto=webp&s=e88e6f574ea2b6abf3644be5140a1ed8ad6d613c", "width": 320, "height": 167}, {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=640&crop=smart&auto=webp&s=290ace7209dd3df0a237ec970a6a8b1662d523e1", "width": 640, "height": 335}, {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=960&crop=smart&auto=webp&s=421952297faebb04d1038184216c053ab1f0bb56", "width": 960, "height": 502}, {"url": "https://external-preview.redd.it/wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI.png?width=1080&crop=smart&auto=webp&s=2e3704dd3e397c6dbebe004c6cce33e8cd82d316", "width": 1080, "height": 565}], "variants": {}, "id": "wyRlnnC4nIHWRfWMUIBnHvHMsP98N9mROJtKXbnwKWI"}], "enabled": false}, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "6c024934-de3f-11ea-a05a-0ea86b2be9a1", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": "moderator", "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#00a6a5", "id": "1uictw2", "is_robot_indexable": true, "report_reasons": null, "author": "AutoModerator", "discussion_type": null, "num_comments": 0, "send_replies": false, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uictw2/monday_daily_thread_project_ideas/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uictw2/monday_daily_thread_project_ideas/", "subreddit_subscribers": 1493425, "created_utc": 1782691206.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "Hey r/learnpython \ud83d\udc4b\n\nBuilt PyRun (pyrun.in) \u2014 a browser-based Python learning platform \n\npowered by Pyodide. Write and run Python directly in your browser, \n\nzero setup needed.\n\nWhat's included:\n\n\u2022 Interactive Python editor (Monaco-based)\n\n\u2022 Structured lessons from beginner to advanced\n\n\u2022 Instant output \u2014 no backend, runs locally in your browser\n\n\u2022 Free to use\n\nWould love feedback from this community \u2014 what topics or features \n\nwould make this more useful for you?\n\nLink: https://pyrun.in", "author_fullname": "t2_2hinuthhdk", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "I built a free in-browser Python learning platform \u2013 no installs, just open and code", "link_flair_richtext": [{"e": "text", "t": "Tutorial"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "tutorial", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uiw602", "quarantine": false, "link_flair_text_color": "dark", "upvote_ratio": 0.25, "author_flair_background_color": null, "subreddit_type": "public", "ups": 0, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Tutorial", "can_mod_post": false, "score": 0, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "post_hint": "self", "content_categories": null, "is_self": true, "mod_note": null, "created": 1782747928.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Hey r/learnpython \ud83d\udc4b
\n\n
Built PyRun (pyrun.in) \u2014 a browser-based Python learning platform 
\n\n
powered by Pyodide. Write and run Python directly in your browser, 
\n\n
zero setup needed.
\n\n
What's included:
\n\n
\u2022 Interactive Python editor (Monaco-based)
\n\n
\u2022 Structured lessons from beginner to advanced
\n\n
\u2022 Instant output \u2014 no backend, runs locally in your browser
\n\n
\u2022 Free to use
\n\n
Would love feedback from this community \u2014 what topics or features 
\n\n
would make this more useful for you?
\n\n
Link: https://pyrun.in
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": true, "is_crosspostable": false, "pinned": false, "over_18": false, "preview": {"images": [{"source": {"url": "https://external-preview.redd.it/Zbc0eUv1jXytp6GEqwGvRsh-PTjb-xLlLDl976BYG_g.png?auto=webp&s=30a4651979e078369eee9b8f8707a356f8a1cc11", "width": 1200, "height": 630}, "resolutions": [{"url": "https://external-preview.redd.it/Zbc0eUv1jXytp6GEqwGvRsh-PTjb-xLlLDl976BYG_g.png?width=108&crop=smart&auto=webp&s=963a21f143c1243528c655e9506c1d7c98703ade", "width": 108, "height": 56}, {"url": "https://external-preview.redd.it/Zbc0eUv1jXytp6GEqwGvRsh-PTjb-xLlLDl976BYG_g.png?width=216&crop=smart&auto=webp&s=fb89b2dda88c5cf5963a2e6ae03251a188ae0826", "width": 216, "height": 113}, {"url": "https://external-preview.redd.it/Zbc0eUv1jXytp6GEqwGvRsh-PTjb-xLlLDl976BYG_g.png?width=320&crop=smart&auto=webp&s=e38c05daf7fe3afd90e786b07c97f0bff68f0f67", "width": 320, "height": 168}, {"url": "https://external-preview.redd.it/Zbc0eUv1jXytp6GEqwGvRsh-PTjb-xLlLDl976BYG_g.png?width=640&crop=smart&auto=webp&s=29b6d153f034b3b0117e66c68b8ae6a92f4ae98a", "width": 640, "height": 336}, {"url": "https://external-preview.redd.it/Zbc0eUv1jXytp6GEqwGvRsh-PTjb-xLlLDl976BYG_g.png?width=960&crop=smart&auto=webp&s=279a2855c24fae439dd2cc3a0d9b7011039fed27", "width": 960, "height": 504}, {"url": "https://external-preview.redd.it/Zbc0eUv1jXytp6GEqwGvRsh-PTjb-xLlLDl976BYG_g.png?width=1080&crop=smart&auto=webp&s=09560fe60b9e09b9e38403956c12f60d4437efc5", "width": 1080, "height": 567}], "variants": {}, "id": "Zbc0eUv1jXytp6GEqwGvRsh-PTjb-xLlLDl976BYG_g"}], "enabled": false}, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "7987a74c-04d8-11eb-84ca-0e0ac8b5a78f", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#dadada", "id": "1uiw602", "is_robot_indexable": true, "report_reasons": null, "author": "No_Monitor3155", "discussion_type": null, "num_comments": 2, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uiw602/i_built_a_free_inbrowser_python_learning_platform/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uiw602/i_built_a_free_inbrowser_python_learning_platform/", "subreddit_subscribers": 1493425, "created_utc": 1782747928.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "I'm a coding beginner and after i updated windows the layout started looking different (i can't add pics), i want it to go back to what it used to be as I'm not sure how to use it. ", "author_fullname": "t2_7pzenp33", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "why is pycharm doing this after a windows update", "link_flair_richtext": [{"e": "text", "t": "Discussion"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "discussion", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uj092r", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.08, "author_flair_background_color": null, "subreddit_type": "public", "ups": 0, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Discussion", "can_mod_post": false, "score": 0, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782756782.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
I'm a coding beginner and after i updated windows the layout started looking different (i can't add pics), i want it to go back to what it used to be as I'm not sure how to use it. 
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": true, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "0df42996-1c5e-11ea-b1a0-0e44e1c5b731", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#f50057", "id": "1uj092r", "is_robot_indexable": true, "report_reasons": null, "author": "Regular_Philosophy_9", "discussion_type": null, "num_comments": 12, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uj092r/why_is_pycharm_doing_this_after_a_windows_update/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uj092r/why_is_pycharm_doing_this_after_a_windows_update/", "subreddit_subscribers": 1493425, "created_utc": 1782756782.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "Hey everyone and all space enthusiasts,\n\n  \nOn April 13, 2029, the asteroid **99942 Apophis** will fly past Earth at roughly 38,000 km. Closer than our geostationary satellites! Its discovery history was pretty interesting, because in the beginning there was a 2.7 % chance of an impact (don't worry, it is almost null now). So I decided to use NASA/JPL's SPICE toolkit using Python (`spiceypy`) to calculate the encounter's properties.\n\n  \nI documented the entire project in a Jupyter Notebook and put together a complete video walk-through tutorial. Code: [https://github.com/ThomasAlbin/Space-Science-With-Python/blob/main/2026/04\\_SPICE\\_Apohis\\_2029\\_Flyby.ipynb](https://github.com/ThomasAlbin/Space-Science-With-Python/blob/main/2026/04_SPICE_Apohis_2029_Flyby.ipynb)\n\nVideo: [https://www.youtube.com/watch?v=j4mJTR-BTto](https://www.youtube.com/watch?v=j4mJTR-BTto)\n\nWhat the tutorial does (and why it's 30 minutes long \ud83d\ude05):\n\n* Computing the time when Apophis enters Earth's gravitational vicinity (the Sphere of Influece)\n* Computing the closest appraoch (distance and time)\n* Computing how the orbital elements (the dynamical properties to describe orbits in space) change after the encounter with Earth.\n\nBy the way: I used also Python + NASA's Cosmographia to create a nice 3D animation of the encounter :). But I will post it soon on my YouTube channel. Since the corresponding code is a complete mess I won't post it here, because I will only post the animation. I have to clean it up...\n\n  \nBest,\n\nThomas (your Cassini/Huygens scientist)", "author_fullname": "t2_67yyoriy", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Computing the close encounter between Apophis and Earth in 2029 (tutorial)", "link_flair_richtext": [{"e": "text", "t": "Tutorial"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "tutorial", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uhutaj", "quarantine": false, "link_flair_text_color": "dark", "upvote_ratio": 0.9, "author_flair_background_color": "#b8001f", "subreddit_type": "public", "ups": 23, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": "01f57bbe-537c-11ee-bb0d-6ef63b2ae5b9", "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Tutorial", "can_mod_post": false, "score": 23, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [{"e": "text", "t": "git push -f"}], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782645910.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "richtext", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Hey everyone and all space enthusiasts,
\n\n
On April 13, 2029, the asteroid 99942 Apophis will fly past Earth at roughly 38,000 km. Closer than our geostationary satellites! Its discovery history was pretty interesting, because in the beginning there was a 2.7 % chance of an impact (don't worry, it is almost null now). So I decided to use NASA/JPL's SPICE toolkit using Python (spiceypy) to calculate the encounter's properties.
\n\n
I documented the entire project in a Jupyter Notebook and put together a complete video walk-through tutorial. Code: https://github.com/ThomasAlbin/Space-Science-With-Python/blob/main/2026/04_SPICE_Apohis_2029_Flyby.ipynb
\n\n
Video: https://www.youtube.com/watch?v=j4mJTR-BTto
\n\n
What the tutorial does (and why it's 30 minutes long \ud83d\ude05):
\n\n
    \n
  • Computing the time when Apophis enters Earth's gravitational vicinity (the Sphere of Influece)
    • \n
  • Computing the closest appraoch (distance and time)
    • \n
  • Computing how the orbital elements (the dynamical properties to describe orbits in space) change after the encounter with Earth.
    • \n
\n\n
By the way: I used also Python + NASA's Cosmographia to create a nice 3D animation of the encounter :). But I will post it soon on my YouTube channel. Since the corresponding code is a complete mess I won't post it here, because I will only post the animation. I have to clean it up...
\n\n
Best,
\n\n
Thomas (your Cassini/Huygens scientist)
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "7987a74c-04d8-11eb-84ca-0e0ac8b5a78f", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": "git push -f", "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#dadada", "id": "1uhutaj", "is_robot_indexable": true, "report_reasons": null, "author": "MrAstroThomas", "discussion_type": null, "num_comments": 10, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": "light", "permalink": "/r/Python/comments/1uhutaj/computing_the_close_encounter_between_apophis_and/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uhutaj/computing_the_close_encounter_between_apophis_and/", "subreddit_subscribers": 1493425, "created_utc": 1782645910.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "Hi guys this tutorial is about training a neural network in Python to sort lists of numbers using the Gumbel-Sinkhorn architecture from the original 2018 paper.\n\n\n\nGithub: [https://github.com/MurageKibicho/Neural-Sorting-Algorithms-Gumbel-Sinkhorn-Networks/tree/main](https://github.com/MurageKibicho/Neural-Sorting-Algorithms-Gumbel-Sinkhorn-Networks/tree/main)\n\n\n\nWriteup: [https://leetarxiv.substack.com/p/gumbel-sinkhorn-neural-sort](https://leetarxiv.substack.com/p/gumbel-sinkhorn-neural-sort)", "author_fullname": "t2_3xvmpjwh9", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Neural Sorting Algorithms: Gumbel-Sinkhorn Networks", "link_flair_richtext": [{"e": "text", "t": "Tutorial"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "tutorial", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uhkicg", "quarantine": false, "link_flair_text_color": "dark", "upvote_ratio": 0.7, "author_flair_background_color": null, "subreddit_type": "public", "ups": 10, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Tutorial", "can_mod_post": false, "score": 10, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782611904.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Hi guys this tutorial is about training a neural network in Python to sort lists of numbers using the Gumbel-Sinkhorn architecture from the original 2018 paper.
\n\n
Github: https://github.com/MurageKibicho/Neural-Sorting-Algorithms-Gumbel-Sinkhorn-Networks/tree/main
\n\n
Writeup: https://leetarxiv.substack.com/p/gumbel-sinkhorn-neural-sort
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "7987a74c-04d8-11eb-84ca-0e0ac8b5a78f", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#dadada", "id": "1uhkicg", "is_robot_indexable": true, "report_reasons": null, "author": "DataBaeBee", "discussion_type": null, "num_comments": 8, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uhkicg/neural_sorting_algorithms_gumbelsinkhorn_networks/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uhkicg/neural_sorting_algorithms_gumbelsinkhorn_networks/", "subreddit_subscribers": 1493425, "created_utc": 1782611904.0, "num_crossposts": 2, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "# Weekly Thread: What's Everyone Working On This Week? \ud83d\udee0\ufe0f\n\nHello r/Python! It's time to share what you've been working on! Whether it's a work-in-progress, a completed masterpiece, or just a rough idea, let us know what you're up to!\n\n# How it Works:\n\n1. **Show & Tell**: Share your current projects, completed works, or future ideas.\n2. **Discuss**: Get feedback, find collaborators, or just chat about your project.\n3. **Inspire**: Your project might inspire someone else, just as you might get inspired here.\n\n# Guidelines:\n\n* Feel free to include as many details as you'd like. Code snippets, screenshots, and links are all welcome.\n* Whether it's your job, your hobby, or your passion project, all Python-related work is welcome here.\n\n# Example Shares:\n\n1. **Machine Learning Model**: Working on a ML model to predict stock prices. Just cracked a 90% accuracy rate!\n2. **Web Scraping**: Built a script to scrape and analyze news articles. It's helped me understand media bias better.\n3. **Automation**: Automated my home lighting with Python and Raspberry Pi. My life has never been easier!\n\nLet's build and grow together! Share your journey and learn from others. Happy coding! \ud83c\udf1f", "author_fullname": "t2_6l4z3", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Sunday Daily Thread: What's everyone working on this week?", "link_flair_richtext": [{"a": ":pythonLogo:", "e": "emoji", "u": "https://emoji.redditmedia.com/8yxdpg6xxnr71_t5_2qh0y/pythonLogo"}, {"e": "text", "t": " Daily Thread"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "daily-thread", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1uhhza7", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.74, "author_flair_background_color": null, "subreddit_type": "public", "ups": 5, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": ":pythonLogo: Daily Thread", "can_mod_post": false, "score": 5, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": true, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1782604808.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Weekly Thread: What's Everyone Working On This Week? \ud83d\udee0\ufe0f
\n\n
Hello r/Python! It's time to share what you've been working on! Whether it's a work-in-progress, a completed masterpiece, or just a rough idea, let us know what you're up to!
\n\n
How it Works:
\n\n
    \n
  1. Show & Tell: Share your current projects, completed works, or future ideas.
    2. \n
  2. Discuss: Get feedback, find collaborators, or just chat about your project.
    3. \n
  3. Inspire: Your project might inspire someone else, just as you might get inspired here.
    4. \n
\n\n
Guidelines:
\n\n
    \n
  • Feel free to include as many details as you'd like. Code snippets, screenshots, and links are all welcome.
    • \n
  • Whether it's your job, your hobby, or your passion project, all Python-related work is welcome here.
    • \n
\n\n
Example Shares:
\n\n
    \n
  1. Machine Learning Model: Working on a ML model to predict stock prices. Just cracked a 90% accuracy rate!
    2. \n
  2. Web Scraping: Built a script to scrape and analyze news articles. It's helped me understand media bias better.
    3. \n
  3. Automation: Automated my home lighting with Python and Raspberry Pi. My life has never been easier!
    4. \n
\n\n
Let's build and grow together! Share your journey and learn from others. Happy coding! \ud83c\udf1f
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "6c024934-de3f-11ea-a05a-0ea86b2be9a1", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": "moderator", "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#00a6a5", "id": "1uhhza7", "is_robot_indexable": true, "report_reasons": null, "author": "AutoModerator", "discussion_type": null, "num_comments": 5, "send_replies": false, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1uhhza7/sunday_daily_thread_whats_everyone_working_on/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1uhhza7/sunday_daily_thread_whats_everyone_working_on/", "subreddit_subscribers": 1493425, "created_utc": 1782604808.0, "num_crossposts": 0, "media": null, "is_video": false}}, {"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "Hi guys this tutorial is about handling errors as value in Python or in nerd language monadic error handling pattern in Python using [katharos](https://github.com/kamalfarahani/katharos) library:\n\n[https://theblog.info/posts/handling-errors-as-values-in-python-with-katharos](https://theblog.info/posts/handling-errors-as-values-in-python-with-katharos)", "author_fullname": "t2_rkfihkkzk", "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Handling Errors as Values in Python with Katharos", "link_flair_richtext": [{"e": "text", "t": "Tutorial"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "tutorial", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1ugsdwl", "quarantine": false, "link_flair_text_color": "dark", "upvote_ratio": 0.8, "author_flair_background_color": null, "subreddit_type": "public", "ups": 32, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "user_reports": [], "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Tutorial", "can_mod_post": false, "score": 32, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": false, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "post_hint": "self", "content_categories": null, "is_self": true, "mod_note": null, "created": 1782532952.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Hi guys this tutorial is about handling errors as value in Python or in nerd language monadic error handling pattern in Python using katharos library:
\n\n
https://theblog.info/posts/handling-errors-as-values-in-python-with-katharos
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "preview": {"images": [{"source": {"url": "https://external-preview.redd.it/2Eh5ntKTVDMhEisUeP53EhuqwJJIX0ynlVvClZhlpoc.png?auto=webp&s=e1b494e288c1332109f2282e0eda050c121c7f94", "width": 1200, "height": 600}, "resolutions": [{"url": "https://external-preview.redd.it/2Eh5ntKTVDMhEisUeP53EhuqwJJIX0ynlVvClZhlpoc.png?width=108&crop=smart&auto=webp&s=ce16fd58cc96586a6f4e09bca243f41637f64836", "width": 108, "height": 54}, {"url": "https://external-preview.redd.it/2Eh5ntKTVDMhEisUeP53EhuqwJJIX0ynlVvClZhlpoc.png?width=216&crop=smart&auto=webp&s=c3fb804da74b957b5369749521851ac924019025", "width": 216, "height": 108}, {"url": "https://external-preview.redd.it/2Eh5ntKTVDMhEisUeP53EhuqwJJIX0ynlVvClZhlpoc.png?width=320&crop=smart&auto=webp&s=f77c73faba71e3855e963638b6e281ae8ba9326b", "width": 320, "height": 160}, {"url": "https://external-preview.redd.it/2Eh5ntKTVDMhEisUeP53EhuqwJJIX0ynlVvClZhlpoc.png?width=640&crop=smart&auto=webp&s=12389bfb3289b1adbe89c931c1868cfb229f14cd", "width": 640, "height": 320}, {"url": "https://external-preview.redd.it/2Eh5ntKTVDMhEisUeP53EhuqwJJIX0ynlVvClZhlpoc.png?width=960&crop=smart&auto=webp&s=d129a03ff5a59c64480e409707497af6c212c840", "width": 960, "height": 480}, {"url": "https://external-preview.redd.it/2Eh5ntKTVDMhEisUeP53EhuqwJJIX0ynlVvClZhlpoc.png?width=1080&crop=smart&auto=webp&s=db5f489c84a30dd6dbc69accc931aebd60168702", "width": 1080, "height": 540}], "variants": {}, "id": "2Eh5ntKTVDMhEisUeP53EhuqwJJIX0ynlVvClZhlpoc"}], "enabled": false}, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "7987a74c-04d8-11eb-84ca-0e0ac8b5a78f", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#dadada", "id": "1ugsdwl", "is_robot_indexable": true, "report_reasons": null, "author": "EntryNo8040", "discussion_type": null, "num_comments": 9, "send_replies": true, "contest_mode": false, "mod_reports": [], "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1ugsdwl/handling_errors_as_values_in_python_with_katharos/", "stickied": false, "url": "https://www.reddit.com/r/Python/comments/1ugsdwl/handling_errors_as_values_in_python_with_katharos/", "subreddit_subscribers": 1493425, "created_utc": 1782532952.0, "num_crossposts": 2, "media": null, "is_video": false}}], "before": null}}
\ No newline at end of file

From 4d950ddcb50c3b7fe7fbd65cd6f43f758803c09f Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:28:08 +0530
Subject: [PATCH 12/16] test(native-connector): add reddit post fixture

---
 .../tests/unit/scrapers/reddit/fixtures/sample_post.json         | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_post.json

diff --git a/surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_post.json b/surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_post.json
new file mode 100644
index 000000000..d2627e6e9
--- /dev/null
+++ b/surfsense_backend/tests/unit/scrapers/reddit/fixtures/sample_post.json
@@ -0,0 +1 @@
+[{"kind": "Listing", "data": {"after": null, "dist": 1, "modhash": "", "geo_filter": "", "children": [{"kind": "t3", "data": {"approved_at_utc": null, "subreddit": "Python", "selftext": "Post all of your code/projects/showcases/AI slop here. \n\nRecycles once a month.", "user_reports": [], "saved": false, "mod_reason_title": null, "gilded": 0, "clicked": false, "title": "Showcase Thread", "link_flair_richtext": [{"e": "text", "t": "Showcase"}], "subreddit_name_prefixed": "r/Python", "hidden": false, "pwls": 6, "link_flair_css_class": "showcase", "downs": 0, "thumbnail_height": null, "top_awarded_type": null, "hide_score": false, "name": "t3_1tws1w7", "quarantine": false, "link_flair_text_color": "light", "upvote_ratio": 0.87, "author_flair_background_color": null, "subreddit_type": "public", "ups": 27, "total_awards_received": 0, "media_embed": {}, "thumbnail_width": null, "author_flair_template_id": null, "is_original_content": false, "author_fullname": "t2_6l4z3", "secure_media": null, "is_reddit_media_domain": false, "is_meta": false, "category": null, "secure_media_embed": {}, "link_flair_text": "Showcase", "can_mod_post": false, "score": 27, "approved_by": null, "is_created_from_ads_ui": false, "author_premium": true, "thumbnail": "self", "edited": false, "author_flair_css_class": null, "author_flair_richtext": [], "gildings": {}, "content_categories": null, "is_self": true, "mod_note": null, "created": 1780589106.0, "link_flair_type": "richtext", "wls": 6, "removed_by_category": null, "banned_by": null, "author_flair_type": "text", "domain": "self.Python", "allow_live_comments": false, "selftext_html": "
Post all of your code/projects/showcases/AI slop here. 
\n\n
Recycles once a month.
\n
", "likes": null, "suggested_sort": null, "banned_at_utc": null, "view_count": null, "archived": false, "no_follow": false, "is_crosspostable": false, "pinned": false, "over_18": false, "all_awardings": [], "awarders": [], "media_only": false, "link_flair_template_id": "f35fb004-c1ff-11ee-8305-565bc5d0cc73", "can_gild": false, "spoiler": false, "locked": false, "author_flair_text": null, "treatment_tags": [], "visited": false, "removed_by": null, "num_reports": null, "distinguished": null, "subreddit_id": "t5_2qh0y", "author_is_blocked": false, "mod_reason_by": null, "removal_reason": null, "link_flair_background_color": "#ff66ac", "id": "1tws1w7", "is_robot_indexable": true, "num_duplicates": 0, "report_reasons": null, "author": "AutoModerator", "discussion_type": null, "num_comments": 207, "send_replies": true, "media": null, "contest_mode": false, "author_patreon_flair": false, "author_flair_text_color": null, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/", "stickied": true, "url": "https://www.reddit.com/r/Python/comments/1tws1w7/showcase_thread/", "subreddit_subscribers": 1493425, "created_utc": 1780589106.0, "num_crossposts": 0, "mod_reports": [], "is_video": false}}], "before": null}}, {"kind": "Listing", "data": {"after": null, "dist": null, "modhash": "", "geo_filter": "", "children": [{"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "oprc6qa", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "hannah_G_", "can_mod_post": false, "created_utc": 1780597457.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 4, "author_fullname": "t2_gjk10lou", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "**db-git**\u00a0\\- keep your local database in sync with your git branches.\n\n# What My Project Does\n\n`db-git`\u00a0is a developer tool for projects where database state follows code changes: schema migrations, seed data, experimental feature work, and branch switching during reviews. It installs git\u00a0`post-checkout`\u00a0hook and keeps your local database aligned with the branch you are working on.\n\n* Two workflows:\n   * `shared`: one database, saved and restored per branch\n   * `per-branch`: one database per branch\n* PostgreSQL support today, with plans for more database backends\n* Two PostgreSQL snapshot strategies:\n   * `template`: fast database clones using\u00a0`CREATE DATABASE ... TEMPLATE`\n   * `pgdump`: portable snapshots using\u00a0`pg_dump`\u00a0and\u00a0`pg_restore`\n\n# Target Audience\n\nBackend and full-stack developers who run databases locally and switch branches often, especially on projects where migrations or seed data diverge between branches. It's a local development tool.\n\n# Comparison\n\nThe main things that set\u00a0`db-git`\u00a0apart from existing tools are:\n\n1. It lets you choose per project,\u00a0`shared`\u00a0vs\u00a0`per-branch`, and\u00a0`template`\u00a0vs\u00a0`pgdump`.\n2. It ties database state directly to checkout.\n3. It is not tied to a specific database engine. PostgreSQL is the first supported backend, but the design isn't Postgres-specific, and more databases are planned.\n\n`uv tool install db-git`\n\nGitHub:\u00a0[https://github.com/earthcomfy/db-git](https://github.com/earthcomfy/db-git)\n\nAny feedback is very welcome!", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_oprc6qa", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
db-git\u00a0- keep your local database in sync with your git branches.
\n\n
What My Project Does
\n\n
db-git\u00a0is a developer tool for projects where database state follows code changes: schema migrations, seed data, experimental feature work, and branch switching during reviews. It installs git\u00a0post-checkout\u00a0hook and keeps your local database aligned with the branch you are working on.
\n\n
    \n
  • Two workflows:\n\n
      \n
    • shared: one database, saved and restored per branch
      • \n
    • per-branch: one database per branch
      • \n
    • \n
  • PostgreSQL support today, with plans for more database backends
    • \n
  • Two PostgreSQL snapshot strategies:\n\n
      \n
    • template: fast database clones using\u00a0CREATE DATABASE ... TEMPLATE
      • \n
    • pgdump: portable snapshots using\u00a0pg_dump\u00a0and\u00a0pg_restore
      • \n
    • \n
\n\n
Target Audience
\n\n
Backend and full-stack developers who run databases locally and switch branches often, especially on projects where migrations or seed data diverge between branches. It's a local development tool.
\n\n
Comparison
\n\n
The main things that set\u00a0db-git\u00a0apart from existing tools are:
\n\n
    \n
  1. It lets you choose per project,\u00a0shared\u00a0vs\u00a0per-branch, and\u00a0template\u00a0vs\u00a0pgdump.
    2. \n
  2. It ties database state directly to checkout.
    3. \n
  3. It is not tied to a specific database engine. PostgreSQL is the first supported backend, but the design isn't Postgres-specific, and more databases are planned.
    4. \n
\n\n
uv tool install db-git
\n\n
GitHub:\u00a0https://github.com/earthcomfy/db-git
\n\n
Any feedback is very welcome!
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/oprc6qa/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780597457.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 4}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "opt03ob", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "emnoleg", "can_mod_post": false, "created_utc": 1780615070.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 3, "author_fullname": "t2_25q8p7gsgg", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "# Tired of screen-sharing 3D models in meetings, so I made them float on my webcam instead\n\n  \nDrop any\u00a0*.glb*\u00a0file in a folder, pick it from the tray icon, and it appears on your webcam. **Move it, rotate it, scale it**. Your meeting app sees it as a normal camera. Nothing else to open or manage.\n\nGood for product demos, design reviews, showing off Blender exports, branded content on calls. Anything where you want the model visible **without screen-sharing**.\n\nWorks in Zoom, Meet, Teams, Discord, pretty much anything that accepts a webcam input.\n\nRenders the 3D layer **offscreen** and caches it. The model only re-renders when you move it, so the webcam stays smooth on any machine. Python,\u00a0*pyrender*,\u00a0*pyvirtualcam*\u00a0under the hood.\u00a0Open source, setup is one script.\n\n[Project Link](https://github.com/aadi-joshi/cam3)\n\n\n\n*Demo model: V1 from ULTRAKILL on Sketchfab (CC BY). Character by Hakita / New Blood Interactive. Not included in the repo.*\u00a0[*https://sketchfab.com/3d-models/v1-ultrakill-d951a08a8f50412d84e262bad887b285*](https://sketchfab.com/3d-models/v1-ultrakill-d951a08a8f50412d84e262bad887b285)", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_opt03ob", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
Tired of screen-sharing 3D models in meetings, so I made them float on my webcam instead
\n\n
Drop any\u00a0.glb\u00a0file in a folder, pick it from the tray icon, and it appears on your webcam. Move it, rotate it, scale it. Your meeting app sees it as a normal camera. Nothing else to open or manage.
\n\n
Good for product demos, design reviews, showing off Blender exports, branded content on calls. Anything where you want the model visible without screen-sharing.
\n\n
Works in Zoom, Meet, Teams, Discord, pretty much anything that accepts a webcam input.
\n\n
Renders the 3D layer offscreen and caches it. The model only re-renders when you move it, so the webcam stays smooth on any machine. Python,\u00a0pyrender,\u00a0pyvirtualcam\u00a0under the hood.\u00a0Open source, setup is one script.
\n\n
Project Link
\n\n
Demo model: V1 from ULTRAKILL on Sketchfab (CC BY). Character by Hakita / New Blood Interactive. Not included in the repo.\u00a0https://sketchfab.com/3d-models/v1-ultrakill-d951a08a8f50412d84e262bad887b285
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/opt03ob/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780615070.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 3}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "oqcmdwf", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "Comfortable-Noise144", "can_mod_post": false, "created_utc": 1780872157.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 3, "author_fullname": "t2_7qhbipym", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "Hi,\n\n**I built a VS Code extension to helt prevent losing track of complex math expressions in Python**\n\nWhen you're writing something like\u00a0`(a * np.sqrt(x**2 + y**2)) / (2 * np.pi * sigma**2)`\u00a0it gets really hard to mentally parse what that formula actually is. In Maple or Mathematica you see a clean equation. In Python you just see a wall of parentheses.\n\nSo I built a VS Code extension that lets you select any expression and instantly renders it as a proper math equation. It supports NumPy, SymPy, SciPy, matrices, integrals, derivatives and more.\n\nThis extension is intended for everyone, beginner programmers as well as experienced programmers, it just meant to give a better overview of math expressions. \n\nFree on the marketplace, search\u00a0**\"Python Expression Visualizer\"**\u00a0in VS Code extensions.\n\nThis is a link to the Github repository:  \n[https://github.com/NickG-DK/python-expression-visualizer](https://github.com/NickG-DK/python-expression-visualizer)\n\n  \nWould love feedback, as this is my first extension ever.", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_oqcmdwf", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
Hi,
\n\n
I built a VS Code extension to helt prevent losing track of complex math expressions in Python
\n\n
When you're writing something like\u00a0(a * np.sqrt(x**2 + y**2)) / (2 * np.pi * sigma**2)\u00a0it gets really hard to mentally parse what that formula actually is. In Maple or Mathematica you see a clean equation. In Python you just see a wall of parentheses.
\n\n
So I built a VS Code extension that lets you select any expression and instantly renders it as a proper math equation. It supports NumPy, SymPy, SciPy, matrices, integrals, derivatives and more.
\n\n
This extension is intended for everyone, beginner programmers as well as experienced programmers, it just meant to give a better overview of math expressions. 
\n\n
Free on the marketplace, search\u00a0"Python Expression Visualizer"\u00a0in VS Code extensions.
\n\n
This is a link to the Github repository:
\nhttps://github.com/NickG-DK/python-expression-visualizer
\n\n
Would love feedback, as this is my first extension ever.
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/oqcmdwf/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780872157.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 3}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "oprmzm5", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "anton273", "can_mod_post": false, "created_utc": 1780600389.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 2, "author_fullname": "t2_5n55nn0l", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "I built watchpoints (data breakpoints) for PyCharm - lets you break on a value change, not a line.   \n  \nFree and open source, every developer deserves a great debugging experience.  \nLink: [https://plugins.jetbrains.com/plugin/32087-python-watchpoint](https://plugins.jetbrains.com/plugin/32087-python-watchpoint)", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_oprmzm5", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
I built watchpoints (data breakpoints) for PyCharm - lets you break on a value change, not a line.   
\n\n
Free and open source, every developer deserves a great debugging experience.
\nLink: https://plugins.jetbrains.com/plugin/32087-python-watchpoint
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/oprmzm5/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780600389.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 2}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "opro849", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "hmoein", "can_mod_post": false, "created_utc": 1780600729.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 2, "author_fullname": "t2_jeyhnly", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "[Grizzlars](https://github.com/NavodPeiris/grizzlars)\u00a0is Python bindings for the C++ DataFrame with an interface almost identical to Pandas. It is a bit of work in progress. It depends on an older version of C++ DataFrame than the latest release. But it is moving along surely.", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_opro849", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
Grizzlars\u00a0is Python bindings for the C++ DataFrame with an interface almost identical to Pandas. It is a bit of work in progress. It depends on an older version of C++ DataFrame than the latest release. But it is moving along surely.
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/opro849/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780600729.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 2}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": {"kind": "Listing", "data": {"after": null, "dist": null, "modhash": "", "geo_filter": "", "children": [{"kind": "more", "data": {"count": 2, "name": "t1_oqtbh24", "id": "oqtbh24", "parent_id": "t1_oprt22n", "depth": 1, "children": ["oqtbh24"]}}], "before": null}}, "user_reports": [], "saved": false, "id": "oprt22n", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "em_el_k0b01101011", "can_mod_post": false, "created_utc": 1780602067.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 2, "author_fullname": "t2_v8e339ps", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "**HyperWeave** is a Python library that generates self-contained SVGs: profile cards, star-history charts, small dashboards, receipts from AI coding sessions, and more. No JS, no external deps. Each one is a single SVG that renders anywhere an image does (GitHub READMEs, Slack, Notion, docs).\n\nDrive it from a CLI, an HTTP API, or an MCP server. It can also pull live data into the artifact, so a card can show a package's real download trend or latest version (PyPI, npm, crates, Hugging Face, arXiv, and a few others) without fetching anything yourself.\n\nThere's a hook for AI coding sessions too: each one ends with a receipt of tokens, cost, and tool calls.\n\nFastAPI + Pydantic + Jinja2 + Typer under the hood.\n\n`pip install hyperweave`\n\nGitHub: https://github.com/InnerAura/hyperweave\n\nPyPI: https://pypi.org/project/hyperweave/\n\nStill early, any feedback welcome. Cheers.", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_oprt22n", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
HyperWeave is a Python library that generates self-contained SVGs: profile cards, star-history charts, small dashboards, receipts from AI coding sessions, and more. No JS, no external deps. Each one is a single SVG that renders anywhere an image does (GitHub READMEs, Slack, Notion, docs).
\n\n
Drive it from a CLI, an HTTP API, or an MCP server. It can also pull live data into the artifact, so a card can show a package's real download trend or latest version (PyPI, npm, crates, Hugging Face, arXiv, and a few others) without fetching anything yourself.
\n\n
There's a hook for AI coding sessions too: each one ends with a receipt of tokens, cost, and tool calls.
\n\n
FastAPI + Pydantic + Jinja2 + Typer under the hood.
\n\n
pip install hyperweave
\n\n
GitHub: https://github.com/InnerAura/hyperweave
\n\n
PyPI: https://pypi.org/project/hyperweave/
\n\n
Still early, any feedback welcome. Cheers.
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/oprt22n/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780602067.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 2}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "opseoev", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "Beginning-Fruit-1397", "can_mod_post": false, "created_utc": 1780608167.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 2, "author_fullname": "t2_ndboruet", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "If you like itertools, fluent interfaces, Rust Result and Option, or the libraries like returns or more-itertools, check out my project pyochain!\u00a0\n\nSince the last showcase, more code have been moved to Rust which led to substantial performance improvements, and new constructs have been added, notably:\n\n- SliceView, zero copy, efficient view/slice of a Sequence\n- StableSet, a set that keep the original Iterable ordering when created.\n- Deque, pyochain version of collections.deque\u00a0\n\nIt's now as well dependency-free!\n\n\u00a0All the code coming from cytoolz (cython toolz version) has been replaced by the same functionality ported in Rust.\n\nLinks:\n\nGithub:\nhttps://github.com/OutSquareCapital/pyochain\n\nPypi:\nhttps://pypi.org/project/pyochain/\n\nMy last showcase on this sub:\nhttps://www.reddit.com/r/Python/comments/1q61bzg/pyochain_rustlike_iterator_result_and_option_in/\n\nComparison vs similar libraries, where's it's been ranked best choice (the post is NOT from me):\nhttps://www.reddit.com/r/Python/comments/1rj3ct7/comment/o8aordo/?context=3\n", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_opseoev", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
If you like itertools, fluent interfaces, Rust Result and Option, or the libraries like returns or more-itertools, check out my project pyochain!\u00a0
\n\n
Since the last showcase, more code have been moved to Rust which led to substantial performance improvements, and new constructs have been added, notably:
\n\n
    \n
  • SliceView, zero copy, efficient view/slice of a Sequence
    • \n
  • StableSet, a set that keep the original Iterable ordering when created.
    • \n
  • Deque, pyochain version of collections.deque\u00a0
    • \n
\n\n
It's now as well dependency-free!
\n\n
\u00a0All the code coming from cytoolz (cython toolz version) has been replaced by the same functionality ported in Rust.
\n\n
Links:
\n\n
Github:\nhttps://github.com/OutSquareCapital/pyochain
\n\n
Pypi:\nhttps://pypi.org/project/pyochain/
\n\n
My last showcase on this sub:\nhttps://www.reddit.com/r/Python/comments/1q61bzg/pyochain_rustlike_iterator_result_and_option_in/
\n\n
Comparison vs similar libraries, where's it's been ranked best choice (the post is NOT from me):\nhttps://www.reddit.com/r/Python/comments/1rj3ct7/comment/o8aordo/?context=3
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/opseoev/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780608167.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 2}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "oq8egkc", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "iliketrains166", "can_mod_post": false, "created_utc": 1780820743.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 2, "author_fullname": "t2_6i4840c2", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "**peeq: A CLI to investigate Python package metadata, dependencies, vulnerabilities and more**\n\nI work with Python packages as published artifacts a lot: building them from source, inspecting source, applying patches, debugging resolver issues, and checking what actually shipped in an sdist or wheel.\n\nThe information is usually available, but it\u2019s scattered across PyPI metadata, source repos, distribution archives, OSV, resolver output, and other places. Source repos are especially tricky because the main branch often does not match the release I\u2019m investigating.\n\nThat led me to build **peeq** \\- a CLI for inspecting Python package metadata, dependencies, files, and versions from PyPI or private indexes, and known vulnerabilities via OSV, without installing or executing the package being inspected.\n\n\n\n**Links:**\n\nGitHub: [https://github.com/MichaelYochpaz/peeq](https://github.com/MichaelYochpaz/peeq)\n\nDocs: [https://peeq.michaelyo.dev](https://peeq.michaelyo.dev)\n\nPyPI: [https://pypi.org/project/peeq](https://pypi.org/project/peeq)\n\n\n\nExample commands you can try (requires uv):\n\n`uvx peeq info requests --full`\n\n`uvx peeq deps docling --version 2.20.0 --diff 2.30.0`\n\n`uvx peeq cat requests pyproject.toml`\n\n`uvx peeq vulns requests --version 2.31.0`\n\n`uvx peeq conflicts kubernetes==35.0.0a1 kfp==2.16.0`\n\n`uvx peeq why \"flask>=3.0\" -d markupsafe`\n\n\n\npeeq is written for both humans and agents, and ships [with a native Agent Skill](https://peeq.michaelyo.dev/ai-agents/skill/).\n\nWith the skill installed, coding agents can answer package questions by inspecting the published metadata/artifacts themselves instead of guessing from memory or installing packages.\n\nWith peeq's skill installed, you can ask your agent questions in natural language like:\n\n\\- \"Does \\`docling\\` depend on \\`pydantic\\`, and through which path?\"\n\n\\- \"Why can't \\`kubernetes==35.0.0a1\\` be installed together with \\`kfp==2.16.0\\`?\"\n\n\\- \"What changed in \\`docling\\` dependencies between versions 2.20.0 and 2.30.0?\"\n\n\\- \"Which build backend does the \\`httpx\\` Python package use?\"\n\nTransparency note: The project relies heavily on AI coding agents for development. I\u2019m a Python software engineer and the maintainer of the project; design choices, code changes, docs, and releases are all manually reviewed and tested by me.\n\n\n\nFeedback (either here or on GitHub) is welcome!\n\n", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_oq8egkc", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
peeq: A CLI to investigate Python package metadata, dependencies, vulnerabilities and more
\n\n
I work with Python packages as published artifacts a lot: building them from source, inspecting source, applying patches, debugging resolver issues, and checking what actually shipped in an sdist or wheel.
\n\n
The information is usually available, but it\u2019s scattered across PyPI metadata, source repos, distribution archives, OSV, resolver output, and other places. Source repos are especially tricky because the main branch often does not match the release I\u2019m investigating.
\n\n
That led me to build peeq - a CLI for inspecting Python package metadata, dependencies, files, and versions from PyPI or private indexes, and known vulnerabilities via OSV, without installing or executing the package being inspected.
\n\n
Links:
\n\n
GitHub: https://github.com/MichaelYochpaz/peeq
\n\n
Docs: https://peeq.michaelyo.dev
\n\n
PyPI: https://pypi.org/project/peeq
\n\n
Example commands you can try (requires uv):
\n\n
uvx peeq info requests --full
\n\n
uvx peeq deps docling --version 2.20.0 --diff 2.30.0
\n\n
uvx peeq cat requests pyproject.toml
\n\n
uvx peeq vulns requests --version 2.31.0
\n\n
uvx peeq conflicts kubernetes==35.0.0a1 kfp==2.16.0
\n\n
uvx peeq why "flask>=3.0" -d markupsafe
\n\n
peeq is written for both humans and agents, and ships with a native Agent Skill.
\n\n
With the skill installed, coding agents can answer package questions by inspecting the published metadata/artifacts themselves instead of guessing from memory or installing packages.
\n\n
With peeq's skill installed, you can ask your agent questions in natural language like:
\n\n
- "Does `docling` depend on `pydantic`, and through which path?"
\n\n
- "Why can't `kubernetes==35.0.0a1` be installed together with `kfp==2.16.0`?"
\n\n
- "What changed in `docling` dependencies between versions 2.20.0 and 2.30.0?"
\n\n
- "Which build backend does the `httpx` Python package use?"
\n\n
Transparency note: The project relies heavily on AI coding agents for development. I\u2019m a Python software engineer and the maintainer of the project; design choices, code changes, docs, and releases are all manually reviewed and tested by me.
\n\n
Feedback (either here or on GitHub) is welcome!
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/oq8egkc/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780820743.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 2}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "ot36qxq", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "WompTitanium", "can_mod_post": false, "created_utc": 1782115044.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 2, "author_fullname": "t2_2agi8wikb3", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "Built a CLI tool that maps any codebase instantly, no API keys, fully offline\nJust run pip install codemappr then codemappr scan inside any project folder.\nIt detects 20+ project types (React, Django, Rust, Flutter, etc.) and gives you a full architecture breakdown in seconds. Outputs to terminal, Markdown, or HTML.\nNo setup, no API keys, no internet needed.\nGitHub: https://github.com/erensh27/CodeMappr[codemappr](https://github.com/erensh27/CodeMappr)", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_ot36qxq", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
Built a CLI tool that maps any codebase instantly, no API keys, fully offline\nJust run pip install codemappr then codemappr scan inside any project folder.\nIt detects 20+ project types (React, Django, Rust, Flutter, etc.) and gives you a full architecture breakdown in seconds. Outputs to terminal, Markdown, or HTML.\nNo setup, no API keys, no internet needed.\nGitHub: https://github.com/erensh27/CodeMappr[codemappr](https://github.com/erensh27/CodeMappr)
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/ot36qxq/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1782115044.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 2}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "ops17j6", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "iamnotafermiparadox", "can_mod_post": false, "created_utc": 1780604316.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 1, "author_fullname": "t2_fd5z8ffe", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "MailTriage is a local, batch-oriented IMAP email triage tool. \n\nThis tool allows me to generate an html page of email received over the last 24 hours and see the subject and snippet in a customized order. Priority senders are first, others are next, followed by a list of email sent by systems. Senders that I don't want to see are not listed.   \n  \nI added the ability to send certain email to a llm to be summarized. Those emails roll into a md file for todos. Once I mark the todo as done, the tool removes that entry and archives it.\n\nBitwarden CLI  pull email credentials. Other methods could be used.\n\n[https://github.com/rogdooley/MailTriage](https://github.com/rogdooley/MailTriage)\n\nCodeGuage: CodeGauge is a deterministic, local-first code quality and security analysis platform. I had this built so I could monitor the \"quality\" of LLM generated code based on linters and static analysis tools.\n\n  \n[https://github.com/rogdooley/CodeGauge](https://github.com/rogdooley/CodeGauge)", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_ops17j6", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
MailTriage is a local, batch-oriented IMAP email triage tool. 
\n\n
This tool allows me to generate an html page of email received over the last 24 hours and see the subject and snippet in a customized order. Priority senders are first, others are next, followed by a list of email sent by systems. Senders that I don't want to see are not listed.   
\n\n
I added the ability to send certain email to a llm to be summarized. Those emails roll into a md file for todos. Once I mark the todo as done, the tool removes that entry and archives it.
\n\n
Bitwarden CLI  pull email credentials. Other methods could be used.
\n\n
https://github.com/rogdooley/MailTriage
\n\n
CodeGuage: CodeGauge is a deterministic, local-first code quality and security analysis platform. I had this built so I could monitor the "quality" of LLM generated code based on linters and static analysis tools.
\n\n
https://github.com/rogdooley/CodeGauge
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/ops17j6/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780604316.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 1}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": {"kind": "Listing", "data": {"after": null, "dist": null, "modhash": "", "geo_filter": "", "children": [{"kind": "more", "data": {"count": 6, "name": "t1_opuqgy8", "id": "opuqgy8", "parent_id": "t1_opt5ogy", "depth": 1, "children": ["opuqgy8"]}}], "before": null}}, "user_reports": [], "saved": false, "id": "opt5ogy", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "Charming_Guidance_76", "can_mod_post": false, "created_utc": 1780617021.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 1, "author_fullname": "t2_7t4cf11b", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "**EDOF, the document toolkit I wish had existed when I started**\n\nThis is one of those \"got fed up, built my own thing\" stories. I make card games, and at work I deal with a lot of document stuff (invoices, certificates, QR labels), and every tool I tried was missing exactly one thing I needed. One had no auto-shrink text. One had no curves. One had nice Photoshop-style effects but you couldn't script it. One couldn't even center text vertically in a box (still bitter about that one). I was tired of duct-taping three tools plus glue code together for every job, so I built EDOF. It took a while. It's on PyPI now.\n\n**What My Project Does**\n\nYou build a document in plain Python, or you open the same file in a visual editor (PyQt6) and drag things around. Both sides read and write the exact same file, so nothing gets lost going back and forth. The part I'm happiest with is that it's literally just `import edof`, so you can drop the whole engine into your own app. The editor I ship is just one program that happens to use it.\n\n    import edof\n    doc  = edof.new(width=210, height=297, title=\"Hello\")\n    page = doc.add_page(dpi=300)\n    page.add_textbox(15, 15, 180, 12, \"Hello world!\")\n    doc.export_pdf(\"hello.pdf\")   # vector PDF, no extra deps\n    \n\nI mostly use it for batch stuff: feed it a CSV, get a few hundred finished PDFs out. That's how I print my own card games now instead of fighting Photoshop layers at 2am. It does text, images, shapes, paths, QR codes, variables with {placeholders}, the auto-shrink and centering I kept missing elsewhere, and it exports to PDF, PNG, SVG and RTF. It can also read and write Word files now, which I'll get to.\n\n**Target Audience**\n\nMe first, honestly. But also anyone in Python who has to crank out documents (labels, invoices, certificates, cards, batch jobs from data), or who wants a document engine living inside their own app instead of bolting on a separate tool. It's MIT, it's on PyPI, I use it for real work. Fair warning on maturity: the core and the editor are solid, the Word part is new and basic, and tables are half-built, so don't lean on those yet.\n\n**Comparison**\n\nReportLab is great but it's code-only and you place everything by coordinates, there's no visual side to hand anyone. python-docx is perfect if you only ever touch Word. LaTeX is its own universe. And none of the visual tools (InDesign and friends) let you just import them into a Python script. EDOF is the in-between I wanted: write it in code or edit it by hand, same file, and embed it anywhere.\n\nThe Word support nearly broke me, by the way. New and genuine respect for whoever maintains .docx tooling for a living. It's basic both directions for now, and it straight up tells you what it can't carry over instead of silently mangling your document.\n\nRepo's here if you want to poke at it: [https://github.com/DavidSchobl/edof](https://github.com/DavidSchobl/edof) . Happy to answer anything, and I'm genuinely after ideas for what to add next.", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_opt5ogy", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
EDOF, the document toolkit I wish had existed when I started
\n\n
This is one of those "got fed up, built my own thing" stories. I make card games, and at work I deal with a lot of document stuff (invoices, certificates, QR labels), and every tool I tried was missing exactly one thing I needed. One had no auto-shrink text. One had no curves. One had nice Photoshop-style effects but you couldn't script it. One couldn't even center text vertically in a box (still bitter about that one). I was tired of duct-taping three tools plus glue code together for every job, so I built EDOF. It took a while. It's on PyPI now.
\n\n
What My Project Does
\n\n
You build a document in plain Python, or you open the same file in a visual editor (PyQt6) and drag things around. Both sides read and write the exact same file, so nothing gets lost going back and forth. The part I'm happiest with is that it's literally just import edof, so you can drop the whole engine into your own app. The editor I ship is just one program that happens to use it.
\n\n
import edof\ndoc  = edof.new(width=210, height=297, title="Hello")\npage = doc.add_page(dpi=300)\npage.add_textbox(15, 15, 180, 12, "Hello world!")\ndoc.export_pdf("hello.pdf")   # vector PDF, no extra deps\n
\n\n
I mostly use it for batch stuff: feed it a CSV, get a few hundred finished PDFs out. That's how I print my own card games now instead of fighting Photoshop layers at 2am. It does text, images, shapes, paths, QR codes, variables with {placeholders}, the auto-shrink and centering I kept missing elsewhere, and it exports to PDF, PNG, SVG and RTF. It can also read and write Word files now, which I'll get to.
\n\n
Target Audience
\n\n
Me first, honestly. But also anyone in Python who has to crank out documents (labels, invoices, certificates, cards, batch jobs from data), or who wants a document engine living inside their own app instead of bolting on a separate tool. It's MIT, it's on PyPI, I use it for real work. Fair warning on maturity: the core and the editor are solid, the Word part is new and basic, and tables are half-built, so don't lean on those yet.
\n\n
Comparison
\n\n
ReportLab is great but it's code-only and you place everything by coordinates, there's no visual side to hand anyone. python-docx is perfect if you only ever touch Word. LaTeX is its own universe. And none of the visual tools (InDesign and friends) let you just import them into a Python script. EDOF is the in-between I wanted: write it in code or edit it by hand, same file, and embed it anywhere.
\n\n
The Word support nearly broke me, by the way. New and genuine respect for whoever maintains .docx tooling for a living. It's basic both directions for now, and it straight up tells you what it can't carry over instead of silently mangling your document.
\n\n
Repo's here if you want to poke at it: https://github.com/DavidSchobl/edof . Happy to answer anything, and I'm genuinely after ideas for what to add next.
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/opt5ogy/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780617021.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 1}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "opul15z", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "Pytrithon", "can_mod_post": false, "created_utc": 1780636699.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 1, "author_fullname": "t2_yzvpcs5sb", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "## Introduction\n\nI have already introduced Pytrithon three times on Reddit.\nSee:\n\nhttps://www.reddit.com/r/Python/comments/1q8dwsm/pytrithon_v119_graphical_petri_net_inspired_agent/\nhttps://www.reddit.com/r/Python/comments/1nr3qvm/pytrithon_graphical_petrinet_inspired_agent/\nhttps://www.reddit.com/r/Python/comments/1mx9w5r/graphical_petrinet_inspired_agent_oriented/\n\n## What My Project Does\n\nPytrithon is a graphical Petri net inspired agent oriented programming language based on Python.\nIt allows writing code as a two dimensional graph of interconnected elements and separates data as Places and code as Transitions. Inter Agent communication and GUI widgets are first class components of the language. Through the Monipulator, Agents can be monitored and manipulated.\n\n## Target Audience\n\nThe target audience is both experienced and novice programmers who want to try something new.\n\n## Why I Built It\n\nI realized the power of Petri net inspired programming and the joy of having a more expressive way to specify control flow.\n\n## Comparison\n\nThere are no other visual programming languages which embed actual code into their graphs.\n\n## How To Explore\n\nTo run all included example Agents you need at least Python 3.10 installed. To install all dependencies, run the 'install' script. Then you can start up a Nexus with a Monipulator by running the 'pytrithon' script, where you can start Agents through opening them with 'crtl-o' twice and hitting the 'Open Agent' button. You can also directly specify which Agents to run through the command line by starting a Nexus, Monipulator, and Agents in one single command: 'python nexus -m \\ \\'.\n\nRecommended example Agents to run are: 'basic', 'prodcons', 'address', 'kata', 'calculator', 'kniffel', 'guess', 'yahtzeeserver' + multiple 'yahtzee', 'pokerserver' + multiple 'poker', 'chatserver' + multiple 'chat', 'image', 'jobapplic', and 'nethods'. As a proof of concept, I created a whole Pygame game, TMWOTY2, which is choreographed by 6 Agents as their own processes, which runs at a solid 60 frames per second. To start or open TMWOTY2 in the Monipulator, run the 'tmwoty2' or 'edittmwoty2' script. Your focus should on the 'workbench' folder, which contains all Agents and their respective Python modules; the 'Pytrithon' folder is just the backstage where the magic happens.\n\n## What Is New\n\nSince my last post the whole system now handles Agents, Monipulators, and Nexi terminating from the network. Bookkeeping is performed, cleansing the internal structures handling all process types, making the prototype more resilient. The 'chatserver' and 'chat' Agents now show a list of Agents currently connected. This is enabled through the new 'Event' Transition, which pushes Nexus Events to all listening Agents.\n\nSince my penultimate post I have added a distributed Yahtzee game which you should try out. In order to setup a server on a reachable machine and connect other machines, you need to do the following:\nOn the machine meant to be the server, run 'python nexus yahtzeeserver' first. Then on the machines meant to be the clients through which users play, run 'python nexus -x \\ yahtzee'. The clients probe the interconnected Nexi for a server and start with a lobby mask where you can select your name and start a game with all players signed up.\n\n## GitHub Link\n\nhttps://github.com/JochenSimon/pytrithon\n\n-------\n\nThis is the fifth post about Pytrithon on Reddit. There is a plethora of example Agents to view and run included in the repository.\nPlease check it out and send feedback to the E-Mail address stated in the Monipulator About blurb.\nI plan on putting Pytrithon onto the next level soon. Be sure to check for new happenings.", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_opul15z", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
Introduction
\n\n
I have already introduced Pytrithon three times on Reddit.\nSee:
\n\n
https://www.reddit.com/r/Python/comments/1q8dwsm/pytrithon_v119_graphical_petri_net_inspired_agent/\nhttps://www.reddit.com/r/Python/comments/1nr3qvm/pytrithon_graphical_petrinet_inspired_agent/\nhttps://www.reddit.com/r/Python/comments/1mx9w5r/graphical_petrinet_inspired_agent_oriented/
\n\n
What My Project Does
\n\n
Pytrithon is a graphical Petri net inspired agent oriented programming language based on Python.\nIt allows writing code as a two dimensional graph of interconnected elements and separates data as Places and code as Transitions. Inter Agent communication and GUI widgets are first class components of the language. Through the Monipulator, Agents can be monitored and manipulated.
\n\n
Target Audience
\n\n
The target audience is both experienced and novice programmers who want to try something new.
\n\n
Why I Built It
\n\n
I realized the power of Petri net inspired programming and the joy of having a more expressive way to specify control flow.
\n\n
Comparison
\n\n
There are no other visual programming languages which embed actual code into their graphs.
\n\n
How To Explore
\n\n
To run all included example Agents you need at least Python 3.10 installed. To install all dependencies, run the 'install' script. Then you can start up a Nexus with a Monipulator by running the 'pytrithon' script, where you can start Agents through opening them with 'crtl-o' twice and hitting the 'Open Agent' button. You can also directly specify which Agents to run through the command line by starting a Nexus, Monipulator, and Agents in one single command: 'python nexus -m <agent1> <agent2>'.
\n\n
Recommended example Agents to run are: 'basic', 'prodcons', 'address', 'kata', 'calculator', 'kniffel', 'guess', 'yahtzeeserver' + multiple 'yahtzee', 'pokerserver' + multiple 'poker', 'chatserver' + multiple 'chat', 'image', 'jobapplic', and 'nethods'. As a proof of concept, I created a whole Pygame game, TMWOTY2, which is choreographed by 6 Agents as their own processes, which runs at a solid 60 frames per second. To start or open TMWOTY2 in the Monipulator, run the 'tmwoty2' or 'edittmwoty2' script. Your focus should on the 'workbench' folder, which contains all Agents and their respective Python modules; the 'Pytrithon' folder is just the backstage where the magic happens.
\n\n
What Is New
\n\n
Since my last post the whole system now handles Agents, Monipulators, and Nexi terminating from the network. Bookkeeping is performed, cleansing the internal structures handling all process types, making the prototype more resilient. The 'chatserver' and 'chat' Agents now show a list of Agents currently connected. This is enabled through the new 'Event' Transition, which pushes Nexus Events to all listening Agents.
\n\n
Since my penultimate post I have added a distributed Yahtzee game which you should try out. In order to setup a server on a reachable machine and connect other machines, you need to do the following:\nOn the machine meant to be the server, run 'python nexus yahtzeeserver' first. Then on the machines meant to be the clients through which users play, run 'python nexus -x <serveraddress> yahtzee'. The clients probe the interconnected Nexi for a server and start with a lobby mask where you can select your name and start a game with all players signed up.
\n\n
GitHub Link
\n\n
https://github.com/JochenSimon/pytrithon
\n\n
\n\n
This is the fifth post about Pytrithon on Reddit. There is a plethora of example Agents to view and run included in the repository.\nPlease check it out and send feedback to the E-Mail address stated in the Monipulator About blurb.\nI plan on putting Pytrithon onto the next level soon. Be sure to check for new happenings.
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/opul15z/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780636699.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 1}}, {"kind": "t1", "data": {"subreddit_id": "t5_2qh0y", "approved_at_utc": null, "author_is_blocked": false, "comment_type": null, "awarders": [], "mod_reason_by": null, "banned_by": null, "author_flair_type": "text", "total_awards_received": 0, "subreddit": "Python", "author_flair_template_id": null, "likes": null, "replies": "", "user_reports": [], "saved": false, "id": "opuw6vu", "banned_at_utc": null, "mod_reason_title": null, "gilded": 0, "archived": false, "collapsed_reason_code": null, "no_follow": true, "author": "AdHot6282", "can_mod_post": false, "created_utc": 1780642079.0, "send_replies": true, "parent_id": "t3_1tws1w7", "score": 1, "author_fullname": "t2_p6h1483kx", "approved_by": null, "mod_note": null, "all_awardings": [], "collapsed": false, "body": "LocalClicky is a menubar app that lets you control your Mac with your voice, completely offline.  \n  \n  \nSay \"Computer\" to start a session. It stays active - chain commands without repeating the wake word. Say \"bye\" to end. It auto-stops recording when you stop talking (webrtcvad), so there's no fixed timeout.  \n  \n  \nWhat it can do: click things on your screen by name, open/quit apps, control Spotify and volume, create reminders from natural language, run shell commands, inject JS into Chrome. Vision is on-demand \u2014 the model calls look\\_at\\_screen itself when it needs to see something.\n\nOne thing that pushed me to build this: I noticed most people don't think twice before enabling cloud based AI assistants on their machines. But these tools are taking full screenshots of your screen, your code, your emails, your Figma files, your bank statements, your personal moment and sending them to a server. I don't like that at all. LocalClicky's vision model runs locally; screenshots never leave your machine.\n\n  \nStack: Python, Whisper.cpp, Ollama (qwen3:8b + gemma4:e4b), webrtcvad, PyAutoGUI, rumps.  \n  \n  \nNothing leaves your machine. MIT licensed, open source.  \n  \n  \nGitHub: [https://github.com/dikshantrajput/LocalClicky](https://github.com/dikshantrajput/LocalClicky)  \nDemo: [https://www.youtube.com/watch?v=i8QpFR6nEY4](https://www.youtube.com/watch?v=i8QpFR6nEY4)", "edited": false, "top_awarded_type": null, "author_flair_css_class": null, "name": "t1_opuw6vu", "is_submitter": false, "downs": 0, "author_flair_richtext": [], "author_patreon_flair": false, "body_html": "
LocalClicky is a menubar app that lets you control your Mac with your voice, completely offline.  
\n\n
Say "Computer" to start a session. It stays active - chain commands without repeating the wake word. Say "bye" to end. It auto-stops recording when you stop talking (webrtcvad), so there's no fixed timeout.  
\n\n
What it can do: click things on your screen by name, open/quit apps, control Spotify and volume, create reminders from natural language, run shell commands, inject JS into Chrome. Vision is on-demand \u2014 the model calls look_at_screen itself when it needs to see something.
\n\n
One thing that pushed me to build this: I noticed most people don't think twice before enabling cloud based AI assistants on their machines. But these tools are taking full screenshots of your screen, your code, your emails, your Figma files, your bank statements, your personal moment and sending them to a server. I don't like that at all. LocalClicky's vision model runs locally; screenshots never leave your machine.
\n\n
Stack: Python, Whisper.cpp, Ollama (qwen3:8b + gemma4:e4b), webrtcvad, PyAutoGUI, rumps.  
\n\n
Nothing leaves your machine. MIT licensed, open source.  
\n\n
GitHub: https://github.com/dikshantrajput/LocalClicky
\nDemo: https://www.youtube.com/watch?v=i8QpFR6nEY4
\n
", "removal_reason": null, "collapsed_reason": null, "distinguished": null, "associated_award": null, "stickied": false, "author_premium": false, "can_gild": false, "gildings": {}, "unrepliable_reason": null, "author_flair_text_color": null, "score_hidden": false, "permalink": "/r/Python/comments/1tws1w7/showcase_thread/opuw6vu/", "subreddit_type": "public", "locked": false, "report_reasons": null, "created": 1780642079.0, "author_flair_text": null, "treatment_tags": [], "link_id": "t3_1tws1w7", "subreddit_name_prefixed": "r/Python", "controversiality": 0, "depth": 0, "author_flair_background_color": null, "collapsed_because_crowd_control": null, "mod_reports": [], "num_reports": null, "ups": 1}}, {"kind": "more", "data": {"count": 193, "name": "t1_opwgwoi", "id": "opwgwoi", "parent_id": "t3_1tws1w7", "depth": 0, "children": ["opwgwoi", "opwtnqv", "opyqlyr", "oq3bs7r", "opzzs23", "oq1whth", "oq1v3y9", "oq8v60l", "oq8i924", "oq6mjf1", "oq2jzij", "oqg5gfa", "oq3ycth", "oqonmb5", "oqh241n", "oqlbezo", "oqfz79y", "oqeqodc", "oqlraf9", "oqn45re", "oq9f2ol", "oq351b0", "oq82llc", "oqm1yhi", "oqgccc2", "oqm2wyz", "oqmhoy5", "orfatfq", "os57st2", "oqq7go0", "os3iesd", "oqz1f8j", "oqv3m6f", "ordqhi0", "orggusr", "oqg3sxh", "os8fu61", "oqmux4u", "oqsiroa", "orj2t6c", "osdrfxd", "orcrbot", "orl3amm", "os1bujj", "os1bslg", "oq88l8r", "oqa89aj", "oqm3sah", "or1h1i1", "ornjldf", "os1bjka", "ot45yd0", "otcaqvg", "oskwuov", "otm5dr8", "orqkqzh", "oskcn73", "ou05zjc", "osdrrsv", "or2o0hc", "or2wz4i", "ose4mwn", "otik76y", "ou3jtfi", "orayagm", "or48pdm", "or4mymi", "osy66rd", "ouvq47j", "ot2dwjl", "orstatp", "orswnog", "oqgxkc5", "otejpd7", "otzyfkb", "otye4hz", "oqqvklr", "or4fva1", "oryyg7m", "os6vy45", "ormojhi", "osk8x4i", "oslgjd1", "oswij3d", "ounqnaf", "oswi1ha", "os0emx6", "osay6i5", "ou4qsxk", "osjsmlw", "oujmvmv", "ouzdhfb", "orhhvzi", "oqa9y6c", "or780cu", "orou62e", "orz37b3", "oruz6fi", "otay7sy", "otptzzv", "os6yr4u", "os1bvxu", "oth1uto", "ova1vbj", "ou6v9t9", "otcgma0", "ouhufmq", "ouktsbw", "ouus4d8", "ou4vz0t", "otrro6f", "ouj22j2", "oukezil", "otnkxz0", "ouwrnax", "ouo4rtt", "ouaa4g9", "osqbpid", "ostr6h1", "ot2qz8l", "ory8egy", "otikk0d", "oul28wi", "otibuj8", "otx91u2", "oucx7sc", "ouzhogi", "ov32np3", "oubafil", "orsrjeo", "orlwbju", "orm9g8l", "os1o62h", "os6k48f", "ouq2dja", "otx9qa4", "ouz7ekk", "opskei3", "ovf2cnm", "otr48db", "oupjpg4", "oukhre1", "ovhhem5", "ospxx2g", "orrxvux", "orcjorb", "ouxhgeu", "ouzgjhg", "ouagbv9", "ouiiv9w", "oukwgbi", "ov4z1m4", "oqzmh06", "oqs64e4", "orrzi6i", "ork3l7k", "oshwp5h", "otc4w9v", "otl0juz", "ovajat5", "or9m71i", "osg9pq0", "oube17h", "ouu84r1", "ouvs7p4", "ovc3kpi", "oup0ukf", "oprodsc", "oprsgod", "ot4i9w4", "otcj6nf", "ot44xc7", "os6ph8i", "oslz59n", "osln80c", "ouqxogi", "ou59piq"]}}], "before": null}}]
\ No newline at end of file

From 1b1e612e2048cc07b1e0cc67b8a3fa434fe9b76e Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:28:13 +0530
Subject: [PATCH 13/16] test(native-connector): cover reddit fetch resilience

---
 .../scrapers/reddit/test_fetch_resilience.py  | 268 ++++++++++++++++++
 1 file changed, 268 insertions(+)
 create mode 100644 surfsense_backend/tests/unit/scrapers/reddit/test_fetch_resilience.py

diff --git a/surfsense_backend/tests/unit/scrapers/reddit/test_fetch_resilience.py b/surfsense_backend/tests/unit/scrapers/reddit/test_fetch_resilience.py
new file mode 100644
index 000000000..cfe304d6a
--- /dev/null
+++ b/surfsense_backend/tests/unit/scrapers/reddit/test_fetch_resilience.py
@@ -0,0 +1,268 @@
+"""Offline resilience tests for the Reddit fetch seam and fan-out worker pool.
+
+No network. Fake sessions drive the ``loid`` warm-up + rotate-on-block + backoff
+paths deterministically (in live runs the first IP warms and returns 200s, so
+these branches rarely fire). Mirrors the youtube sibling's
+``test_fetch_resilience.py`` shape, extended with a fake warm-up.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import AsyncIterator
+
+from app.proprietary.scrapers.reddit import fetch, scraper
+from app.proprietary.scrapers.reddit.fetch import (
+    RedditAccessBlockedError,
+    _current_session,
+    fetch_json,
+)
+
+_LISTING = {"kind": "Listing", "data": {"children": [], "after": None}}
+
+
+class _FakePage:
+    def __init__(self, status: int, *, cookies: dict | None = None, payload=None):
+        self.status = status
+        self.cookies = cookies or {}
+        self._payload = payload if payload is not None else _LISTING
+
+    def json(self):
+        return self._payload
+
+    @property
+    def body(self) -> str:
+        return json.dumps(self._payload)
+
+
+class _FakeSession:
+    """One 'IP': warm-up mints loid per flags; ``.json`` GETs return ``status``."""
+
+    def __init__(
+        self,
+        status: int = 200,
+        *,
+        shreddit_loid: bool = True,
+        old_loid: bool = False,
+        payload=None,
+    ) -> None:
+        self.status = status
+        self.shreddit_loid = shreddit_loid
+        self.old_loid = old_loid
+        self.payload = payload
+        self.json_calls = 0
+        self.warm_calls = 0
+
+    async def get(self, url, headers=None, cookies=None):
+        if "svc/shreddit" in url:
+            self.warm_calls += 1
+            ck = {"loid": "x", "session_tracker": "y"} if self.shreddit_loid else {}
+            return _FakePage(200, cookies=ck)
+        if "old.reddit.com" in url:
+            self.warm_calls += 1
+            return _FakePage(200, cookies={"loid": "x"} if self.old_loid else {})
+        self.json_calls += 1
+        return _FakePage(self.status, payload=self.payload)
+
+
+class _FakeHolder:
+    """Holder whose ``rotate()`` advances to the next fake session (a new IP)."""
+
+    def __init__(self, sessions: list[_FakeSession]) -> None:
+        self._sessions = sessions
+        self.session = sessions[0]
+        self.rotations = 0
+        self.warmed = False
+
+    async def rotate(self):
+        self.rotations += 1
+        self.session = self._sessions[min(self.rotations, len(self._sessions) - 1)]
+        self.warmed = False  # loid binds to the IP: re-warm on the fresh one
+        return self.session
+
+    async def pace(self) -> None:
+        return None
+
+    async def close(self) -> None:
+        return None
+
+
+def _no_sleep(monkeypatch) -> None:
+    async def _noop(_seconds):
+        return None
+
+    monkeypatch.setattr(fetch.asyncio, "sleep", _noop)
+
+
+async def test_warms_then_returns_json():
+    # old.reddit is tried first and mints loid -> a single warm call.
+    holder = _FakeHolder([_FakeSession(200, old_loid=True)])
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/hot")
+    finally:
+        _current_session.reset(token)
+    assert result == _LISTING
+    assert holder.rotations == 0
+    assert holder.session.warm_calls == 1  # warmed exactly once
+
+
+async def test_warm_falls_back_to_shreddit():
+    # old.reddit doesn't mint loid, shreddit does -> still warms on the same IP.
+    holder = _FakeHolder([_FakeSession(200, shreddit_loid=True, old_loid=False)])
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/hot")
+    finally:
+        _current_session.reset(token)
+    assert result == _LISTING
+    assert holder.rotations == 0
+
+
+async def test_rotates_when_warm_fails_then_succeeds():
+    # IP0 can't mint loid at all -> rotate; IP1 warms fine.
+    holder = _FakeHolder(
+        [
+            _FakeSession(200, shreddit_loid=False, old_loid=False),
+            _FakeSession(200, shreddit_loid=True),
+        ]
+    )
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/hot")
+    finally:
+        _current_session.reset(token)
+    assert result == _LISTING
+    assert holder.rotations == 1
+
+
+async def test_raises_when_no_ip_can_warm():
+    holder = _FakeHolder(
+        [
+            _FakeSession(200, shreddit_loid=False, old_loid=False)
+            for _ in range(fetch._MAX_ROTATIONS + 1)
+        ]
+    )
+    token = _current_session.set(holder)
+    try:
+        raised = False
+        try:
+            await fetch_json("r/python/hot")
+        except RedditAccessBlockedError:
+            raised = True
+    finally:
+        _current_session.reset(token)
+    assert raised
+    assert holder.rotations == fetch._MAX_ROTATIONS
+
+
+async def test_rotates_and_rewarms_on_403():
+    holder = _FakeHolder([_FakeSession(403), _FakeSession(200, old_loid=True)])
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/hot")
+    finally:
+        _current_session.reset(token)
+    assert result == _LISTING
+    assert holder.rotations == 1
+    assert holder.session.warm_calls == 1  # re-warmed on the fresh IP
+
+
+async def test_404_returns_none_without_rotating():
+    holder = _FakeHolder([_FakeSession(404), _FakeSession(200)])
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/comments/missing")
+    finally:
+        _current_session.reset(token)
+    assert result is None
+    assert holder.rotations == 0
+
+
+async def test_429_backs_off_without_rotating(monkeypatch):
+    _no_sleep(monkeypatch)
+    # Same IP: 429 first, then a healthy 200 on retry (no rotation).
+    session = _FakeSession(429)
+
+    async def _get(url, headers=None, cookies=None):
+        if "svc/shreddit" in url or "old.reddit.com" in url:
+            session.warm_calls += 1
+            return _FakePage(200, cookies={"loid": "x"})
+        session.json_calls += 1
+        return _FakePage(429 if session.json_calls == 1 else 200)
+
+    session.get = _get  # type: ignore[method-assign]
+    holder = _FakeHolder([session])
+    token = _current_session.set(holder)
+    try:
+        result = await fetch_json("r/python/hot")
+    finally:
+        _current_session.reset(token)
+    assert result == _LISTING
+    assert holder.rotations == 0
+
+
+async def test_persistent_403_raises_blocked(monkeypatch):
+    _no_sleep(monkeypatch)
+    holder = _FakeHolder([_FakeSession(403) for _ in range(fetch._MAX_ROTATIONS + 1)])
+    token = _current_session.set(holder)
+    try:
+        raised = False
+        try:
+            await fetch_json("r/python/hot")
+        except RedditAccessBlockedError:
+            raised = True
+    finally:
+        _current_session.reset(token)
+    assert raised
+    assert holder.rotations == fetch._MAX_ROTATIONS
+
+
+class _TrackingHolder:
+    """Fake fan-out session holder that records whether it was closed."""
+
+    def __init__(self) -> None:
+        self.closed = False
+
+    async def close(self) -> None:
+        self.closed = True
+
+
+async def test_fan_out_closes_all_sessions_on_early_stop(monkeypatch):
+    holders: list[_TrackingHolder] = []
+
+    async def _fake_open():
+        h = _TrackingHolder()
+        holders.append(h)
+        return h
+
+    from contextlib import asynccontextmanager
+
+    @asynccontextmanager
+    async def _fake_bind(_holder):
+        yield _holder
+
+    monkeypatch.setattr(scraper, "open_proxy_holder", _fake_open)
+    monkeypatch.setattr(scraper, "bind_proxy_holder", _fake_bind)
+
+    async def _job(n: int) -> AsyncIterator[dict]:
+        for i in range(5):
+            yield {"job": n, "i": i}
+
+    jobs = [_job(n) for n in range(20)]
+    gen = scraper.fan_out(jobs, concurrency=4)
+    collected = []
+    async for item in gen:
+        collected.append(item)
+        if len(collected) >= 3:
+            break
+    await gen.aclose()
+
+    assert len(collected) >= 3
+    assert holders, "workers should have opened sessions"
+    assert all(h.closed for h in holders), "a worker leaked its proxy session"
+
+
+async def test_fan_out_empty_jobs_is_noop():
+    out = [x async for x in scraper.fan_out([])]
+    assert out == []

From 0ef02c43cc16d675fb54f75a31f0119c29e020f9 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:28:17 +0530
Subject: [PATCH 14/16] test(native-connector): cover reddit parser mappings

---
 .../unit/scrapers/reddit/test_parsers.py      | 165 ++++++++++++++++++
 1 file changed, 165 insertions(+)
 create mode 100644 surfsense_backend/tests/unit/scrapers/reddit/test_parsers.py

diff --git a/surfsense_backend/tests/unit/scrapers/reddit/test_parsers.py b/surfsense_backend/tests/unit/scrapers/reddit/test_parsers.py
new file mode 100644
index 000000000..bcf92e57d
--- /dev/null
+++ b/surfsense_backend/tests/unit/scrapers/reddit/test_parsers.py
@@ -0,0 +1,165 @@
+"""Offline parser tests for the Reddit scraper.
+
+Two layers:
+- Synthetic, deterministic checks of the JSON->item mapping (hand-built minimal
+  "things" — no live Reddit shapes), which run always.
+- Fixture-pinned checks against real ``.json`` captured by
+  ``scripts/e2e_reddit_scraper.py`` into ``fixtures/``; these ``skip`` when the
+  fixtures are absent (mirrors the youtube sibling). Fill in richer assertions
+  against the captured shapes during implementation.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from app.proprietary.scrapers.reddit.parsers import (
+    after,
+    children,
+    flatten_comments,
+    parse_comment,
+    parse_community,
+    parse_post,
+)
+
+_FIXTURE_DIR = Path(__file__).parent / "fixtures"
+
+
+# --- synthetic mapping (always runs) ---------------------------------------
+
+
+def test_parse_post_maps_core_fields():
+    thing = {
+        "kind": "t3",
+        "data": {
+            "name": "t3_x",
+            "id": "x",
+            "author": "alice",
+            "author_fullname": "t2_a",
+            "title": "Hello",
+            "selftext": "body text",
+            "permalink": "/r/py/comments/x/hello/",
+            "subreddit": "py",
+            "subreddit_name_prefixed": "r/py",
+            "num_comments": 3,
+            "score": 42,
+            "upvote_ratio": 0.97,
+            "over_18": False,
+            "is_self": True,
+            "created_utc": 1_700_000_000,
+            "thumbnail": "self",
+        },
+    }
+    item = parse_post(thing)
+    assert item["dataType"] == "post"
+    assert item["id"] == "t3_x"
+    assert item["parsedId"] == "x"
+    assert item["url"] == "https://www.reddit.com/r/py/comments/x/hello/"
+    assert item["upVotes"] == 42
+    assert item["numberOfComments"] == 3
+    assert item["thumbnailUrl"] is None  # 'self' sentinel is not a URL
+    assert item["createdAt"] == "2023-11-14T22:13:20.000Z"
+
+
+def test_parse_comment_strips_link_prefix():
+    thing = {
+        "kind": "t1",
+        "data": {
+            "name": "t1_c",
+            "id": "c",
+            "body": "a comment",
+            "link_id": "t3_x",
+            "parent_id": "t3_x",
+            "created_utc": 1_700_000_000,
+        },
+    }
+    item = parse_comment(thing)
+    assert item["dataType"] == "comment"
+    assert item["postId"] == "x"  # t3_ prefix stripped
+    assert item["parentId"] == "t3_x"
+
+
+def test_flatten_comments_counts_replies_and_stops_at_more():
+    tree = [
+        {
+            "kind": "t1",
+            "data": {
+                "name": "t1_1",
+                "id": "1",
+                "body": "top",
+                "created_utc": 1,
+                "replies": {
+                    "kind": "Listing",
+                    "data": {
+                        "children": [
+                            {"kind": "t1", "data": {"name": "t1_2", "id": "2",
+                                                    "body": "reply", "replies": ""}},
+                            {"kind": "more", "data": {}},  # stub -> ignored
+                        ]
+                    },
+                },
+            },
+        }
+    ]
+    flat = flatten_comments(tree, max_comments=10)
+    assert len(flat) == 2  # the 'more' stub is skipped
+    assert flat[0]["numberOfReplies"] == 1
+    assert [c["depth"] for c in flat] == [0, 1]
+
+
+def test_flatten_comments_honors_max():
+    tree = [
+        {"kind": "t1", "data": {"name": f"t1_{i}", "id": str(i), "body": "x",
+                                "replies": ""}}
+        for i in range(5)
+    ]
+    assert len(flatten_comments(tree, max_comments=2)) == 2
+
+
+def test_children_and_after():
+    listing = {"kind": "Listing", "data": {"children": [1, 2, 3], "after": "t3_next"}}
+    assert children(listing) == [1, 2, 3]
+    assert after(listing) == "t3_next"
+    assert children({}) == []
+    assert after({"data": {"after": None}}) is None
+
+
+def test_parse_community_maps_members():
+    thing = {"kind": "t5", "data": {"name": "t5_s", "id": "s",
+                                    "display_name": "py", "display_name_prefixed": "r/py",
+                                    "subscribers": 1234, "url": "/r/py/"}}
+    item = parse_community(thing)
+    assert item["dataType"] == "community"
+    assert item["numberOfMembers"] == 1234
+    assert item["url"] == "https://www.reddit.com/r/py/"
+
+
+# --- fixture-pinned (skips until e2e captures real .json) ------------------
+
+
+def _load(name: str):
+    path = _FIXTURE_DIR / name
+    if not path.exists():
+        pytest.skip(f"fixture {name} not captured yet (run e2e_reddit_scraper.py)")
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def test_parse_captured_post_fixture_if_present():
+    data = _load("sample_post.json")
+    # sample_post.json is the [postListing, commentsListing] .json shape.
+    post_children = children(data[0]) if isinstance(data, list) else []
+    assert post_children, "captured post fixture has no post child"
+    item = parse_post(post_children[0])
+    assert item["dataType"] == "post"
+    assert item["id"]
+
+
+def test_parse_captured_listing_fixture_if_present():
+    listing = _load("sample_listing.json")
+    kids = children(listing)
+    assert kids, "captured listing fixture is empty"
+    posts = [parse_post(c) for c in kids if c.get("kind") == "t3"]
+    assert posts, "no t3 posts in captured listing"

From 0a83bf61929acf4e02e3e1926880f09451347903 Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:28:21 +0530
Subject: [PATCH 15/16] test(native-connector): cover reddit schemas and url
 resolver

---
 .../unit/scrapers/reddit/test_skeleton.py     | 73 +++++++++++++++++++
 1 file changed, 73 insertions(+)
 create mode 100644 surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py

diff --git a/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py b/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py
new file mode 100644
index 000000000..7c6829540
--- /dev/null
+++ b/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py
@@ -0,0 +1,73 @@
+"""Offline schema + URL-resolver tests for the Reddit scraper.
+
+Deterministic (no network, no live Reddit shapes): asserts the anonymous-only
+input surface, the flat item serialization contract, and URL classification.
+"""
+
+from __future__ import annotations
+
+from app.proprietary.scrapers.reddit.schemas import RedditItem, RedditScrapeInput
+from app.proprietary.scrapers.reddit.url_resolver import resolve_url
+
+
+def test_input_has_no_auth_fields():
+    # Anonymous-only: no auth-shaped field may exist on the input surface.
+    forbidden = {"username", "password", "token", "login", "auth", "credentials"}
+    assert forbidden.isdisjoint(RedditScrapeInput.model_fields)
+
+
+def test_input_defaults():
+    model = RedditScrapeInput()
+    assert model.sort == "new"
+    assert model.includeNSFW is True
+    assert model.maxItems == 10
+    assert model.startUrls == []
+    assert model.searches == []
+
+
+def test_input_allows_extra_inert_fields():
+    # extra="allow": Apify fields we don't act on are accepted, not rejected.
+    model = RedditScrapeInput(debugMode=True, proxy={"useApifyProxy": True})
+    assert model.model_dump().get("debugMode") is True
+
+
+def test_item_to_output_keeps_none_keys():
+    out = RedditItem(dataType="post", id="t3_x").to_output()
+    assert out["dataType"] == "post"
+    assert out["id"] == "t3_x"
+    assert "numberOfComments" in out  # unsourced fields still present (None)
+    assert out["numberOfComments"] is None
+
+
+def test_resolve_post():
+    r = resolve_url("https://www.reddit.com/r/python/comments/abc123/some_title/")
+    assert r is not None
+    assert r.kind == "post"
+    assert r.value == "abc123"
+    assert r.subreddit == "python"
+
+
+def test_resolve_subreddit_with_and_without_sort():
+    bare = resolve_url("https://www.reddit.com/r/python")
+    assert bare is not None and bare.kind == "subreddit" and bare.sort is None
+    sorted_ = resolve_url("https://www.reddit.com/r/python/top")
+    assert sorted_ is not None and sorted_.sort == "top"
+
+
+def test_resolve_user_tabs():
+    overview = resolve_url("https://www.reddit.com/user/spez")
+    assert overview is not None and overview.kind == "user" and overview.content is None
+    comments = resolve_url("https://www.reddit.com/u/spez/comments")
+    assert comments is not None and comments.content == "comments"
+
+
+def test_resolve_search_global_and_in_sub():
+    global_ = resolve_url("https://www.reddit.com/search/?q=hello")
+    assert global_ is not None and global_.kind == "search" and global_.value == "hello"
+    in_sub = resolve_url("https://www.reddit.com/r/python/search/?q=async")
+    assert in_sub is not None and in_sub.subreddit == "python"
+
+
+def test_resolve_rejects_non_reddit_host():
+    assert resolve_url("https://example.com/r/python") is None
+    assert resolve_url("https://notreddit.com/user/x") is None

From 82141d804754b0c2f53e8036f0417bcf0f1d40ea Mon Sep 17 00:00:00 2001
From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com>
Date: Sat, 4 Jul 2026 17:45:43 +0530
Subject: [PATCH 16/16] chore: fix docs

---
 .../app/proprietary/scrapers/reddit/README.md   |  4 ++--
 .../app/proprietary/scrapers/reddit/__init__.py |  2 +-
 .../app/proprietary/scrapers/reddit/fetch.py    |  2 +-
 .../app/proprietary/scrapers/reddit/schemas.py  | 17 ++++++++---------
 .../app/proprietary/scrapers/reddit/scraper.py  |  2 +-
 .../proprietary/scrapers/reddit/url_resolver.py |  2 +-
 .../tests/unit/scrapers/reddit/test_skeleton.py |  4 ++--
 7 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/README.md b/surfsense_backend/app/proprietary/scrapers/reddit/README.md
index 9f4cff5f2..58c601cf5 100644
--- a/surfsense_backend/app/proprietary/scrapers/reddit/README.md
+++ b/surfsense_backend/app/proprietary/scrapers/reddit/README.md
@@ -1,6 +1,6 @@
 # Reddit scraper (anonymous, no browser)
 
-Platform-native Reddit scraper, Apify "Reddit Scraper"-compatible. Standalone
+Platform-native Reddit scraper (anonymous, no browser). Standalone
 module: it depends only on `app.utils.proxy` + `scrapling` and exposes a stable
 public API. It is **not** wired into routes, `connector_service.py`, ingestion,
 or Celery — integration later is a thin `reddit_routes.py` + one `include_router`
@@ -38,7 +38,7 @@ rotation surfaces as `RedditAccessBlockedError` (mirrors google_maps'
 | File | Responsibility |
 |---|---|
 | `__init__.py` | Public exports: `RedditScrapeInput`, `RedditItem`, `iter_reddit`, `scrape_reddit`, `RedditAccessBlockedError`. |
-| `schemas.py` | Apify-shaped `RedditScrapeInput` (`extra="allow"`, no auth fields) + single flat `RedditItem` keyed by `dataType` + `StartUrl`. |
+| `schemas.py` | `RedditScrapeInput` (`extra="allow"`, no auth fields) + single flat `RedditItem` keyed by `dataType` + `StartUrl`. |
 | `fetch.py` | The core. Rotate-on-block sticky `_RotatingSession` + `_current_session` ContextVar + `warm_session` (loid) + `fetch_json`. No browser imports. |
 | `url_resolver.py` | Classify a Reddit URL → `post`/`subreddit`/`user`/`search`; non-Reddit → `None`. |
 | `parsers.py` | Pure JSON→item mapping (`parse_post`, `parse_comment`, `parse_community`, `flatten_comments`, `children`/`after`). I/O-free. |
diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py b/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py
index 8b8c75f11..94a17ed75 100644
--- a/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py
+++ b/surfsense_backend/app/proprietary/scrapers/reddit/__init__.py
@@ -1,4 +1,4 @@
-"""Platform-native Reddit scraper (Apify Reddit Scraper-compatible, anonymous)."""
+"""Platform-native Reddit scraper (anonymous, no browser)."""
 
 from .fetch import RedditAccessBlockedError
 from .schemas import RedditItem, RedditScrapeInput
diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py b/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py
index 02ed3ac88..ce45795e7 100644
--- a/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py
+++ b/surfsense_backend/app/proprietary/scrapers/reddit/fetch.py
@@ -96,7 +96,7 @@ _LOID_COOKIE = "loid"
 
 
 def now_iso() -> str:
-    """UTC timestamp in the millisecond ISO shape the Apify actor stamps."""
+    """UTC timestamp in the millisecond ISO shape used by scraper output."""
     return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
 
 
diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py b/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py
index 28d322936..74560ae97 100644
--- a/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py
+++ b/surfsense_backend/app/proprietary/scrapers/reddit/schemas.py
@@ -1,9 +1,8 @@
-# ruff: noqa: N815 - field names intentionally mirror the Apify camelCase spec
-"""Apify-compatible input/output models for the Reddit scraper.
+# ruff: noqa: N815 - field names intentionally use the public camelCase API
+"""Input/output models for the Reddit scraper.
 
-The models mirror the public Apify "Reddit Scraper" actor spec so the endpoint
-can be a drop-in. The MVP populates a reliably-sourced subset; every other field
-is accepted (input) or emitted as ``None``/``[]`` (output) so parity is additive.
+The MVP populates a reliably-sourced subset of fields; every other output field
+is emitted as ``None``/``[]`` so the contract can expand additively.
 
 **Anonymous only.** There is deliberately **no** authentication field on the
 input (no username/password/token/``login*``) — the scraper holds only Reddit's
@@ -23,7 +22,7 @@ RedditDataType = Literal["post", "comment", "community", "user"]
 
 
 class StartUrl(BaseModel):
-    """A single direct URL entry (Apify passes ``{"url": ...}`` objects)."""
+    """A single direct URL entry (``{"url": ...}``; extra keys ignored)."""
 
     model_config = ConfigDict(extra="allow")
 
@@ -31,7 +30,7 @@ class StartUrl(BaseModel):
 
 
 class RedditScrapeInput(BaseModel):
-    """Full Apify "Reddit Scraper" input surface (anonymous, no auth fields).
+    """Reddit scraper input surface (anonymous, no auth fields).
 
     Caps (``maxItems``/``maxPostCount``/...) are collector policy applied by
     :func:`scrape_reddit`, NOT ceilings baked into the streaming flows. Fields
@@ -71,7 +70,7 @@ class RedditItem(BaseModel):
     """Single flat output item, keyed by ``dataType``.
 
     One model for post/comment/community/user (union of fields, unsourced
-    default ``None``/``[]``), matching the Apify actor's single-dataset shape.
+    default ``None``/``[]``), using a single flat-dataset shape.
     ``extra="allow"`` keeps the contract open so added fields never break
     consumers.
     """
@@ -123,5 +122,5 @@ class RedditItem(BaseModel):
     scrapedAt: str | None = None
 
     def to_output(self) -> dict[str, Any]:
-        """Serialize to the flat dict shape Apify emits (keeps extras)."""
+        """Serialize to the flat output dict shape (keeps extras)."""
         return self.model_dump(exclude_none=False)
diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py b/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py
index 5fc4e07e9..e97cda024 100644
--- a/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py
+++ b/surfsense_backend/app/proprietary/scrapers/reddit/scraper.py
@@ -362,7 +362,7 @@ def _capped_targets(
 async def iter_reddit(
     input_model: RedditScrapeInput,
 ) -> AsyncIterator[dict[str, Any]]:
-    """Yield Apify-shaped Reddit items. ``startUrls`` override ``searches``.
+    """Yield flat Reddit items. ``startUrls`` override ``searches``.
 
     Independent targets fan out concurrently; each target's ``after`` paging
     stays sequential.
diff --git a/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py b/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py
index 3b6d4e442..0af80b524 100644
--- a/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py
+++ b/surfsense_backend/app/proprietary/scrapers/reddit/url_resolver.py
@@ -1,6 +1,6 @@
 """Classify a Reddit URL into a scrape job.
 
-Covers the ``startUrls`` shapes the Apify spec accepts: a post (permalink), a
+Covers the supported ``startUrls`` shapes: a post (permalink), a
 subreddit listing, a user profile, and a search-results page. Non-Reddit hosts
 resolve to ``None`` so the orchestrator can skip them. Mirrors the frozen
 ``ResolvedUrl`` dataclass shape of ``../youtube/url_resolver.py``, widened with
diff --git a/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py b/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py
index 7c6829540..696944b0a 100644
--- a/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py
+++ b/surfsense_backend/tests/unit/scrapers/reddit/test_skeleton.py
@@ -26,8 +26,8 @@ def test_input_defaults():
 
 
 def test_input_allows_extra_inert_fields():
-    # extra="allow": Apify fields we don't act on are accepted, not rejected.
-    model = RedditScrapeInput(debugMode=True, proxy={"useApifyProxy": True})
+    # extra="allow": unknown inert fields are accepted, not rejected.
+    model = RedditScrapeInput(debugMode=True, proxy={"provider": "custom"})
     assert model.model_dump().get("debugMode") is True