chore: bumped version to 0.0.31

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/agent.py

@@ -50,9 +50,7 @@ def _augment_allowlist_for_collision_prefixes(
             continue
         original = meta.get("mcp_original_tool_name")
         if original in trusted_names and tool.name not in trusted_names:
-            alias_rules.append(
-                Rule(permission=tool.name, pattern="*", action="allow")
-            )
+            alias_rules.append(Rule(permission=tool.name, pattern="*", action="allow"))
 
     if not alias_rules:
         return dependencies

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/get_connected_accounts.py

@@ -85,8 +85,7 @@ def create_get_connected_accounts_tool(*, workspace_id: int) -> BaseTool:
                     # ``server_config`` presence == the connector produces agent
                     # tools. Native rows without it are connected for indexing
                     # only and need a reconnect via MCP.
-                    "usable_in_chat": isinstance(cfg, dict)
-                    and "server_config" in cfg,
+                    "usable_in_chat": isinstance(cfg, dict) and "server_config" in cfg,
                     "account": account_meta,
                 }
             )

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py

@@ -83,7 +83,9 @@ def resolve_tool_name_collisions(
             or original_name,
             "mcp_collision_prefixed": True,
         }
-        resolved.append(tool.model_copy(update={"name": new_name, "metadata": new_meta}))
+        resolved.append(
+            tool.model_copy(update={"name": new_name, "metadata": new_meta})
+        )
     return resolved
 
 

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py

@@ -128,7 +128,7 @@ def _cell(value: Any) -> str:
     """Render one CSV cell: scalars as-is, nested structures as compact JSON."""
     if value is None:
         return ""
-    if isinstance(value, (dict, list)):
+    if isinstance(value, dict | list):
         return json.dumps(value, ensure_ascii=False, default=str)
     return str(value)
 
@@ -230,9 +230,7 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]:
         count = min(max(1, limit), _MAX_LIMIT)
         window = lines[start : start + count]
         if not window:
-            return (
-                f"No lines at offset {start} (total {len(lines)} lines in {ref})."
-            )
+            return f"No lines at offset {start} (total {len(lines)} lines in {ref})."
         window_body = "\n".join(window)
         start_char = max(0, char_offset)
         if start_char >= len(window_body) > 0:
@@ -328,10 +326,14 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]:
         records = _rows_from_body(body, rows)
         if include_pattern:
             inc = _build_matcher(include_pattern.strip())
-            records = [r for r in records if inc(" ".join(map(_cell, r.values()))) is not None]
+            records = [
+                r for r in records if inc(" ".join(map(_cell, r.values()))) is not None
+            ]
         if exclude_pattern:
             exc = _build_matcher(exclude_pattern.strip())
-            records = [r for r in records if exc(" ".join(map(_cell, r.values()))) is None]
+            records = [
+                r for r in records if exc(" ".join(map(_cell, r.values()))) is None
+            ]
         if not records:
             return (
                 f"Error: no rows to export from {ref} "
@@ -353,7 +355,9 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]:
 
         preview_lines = csv_text.split("\n")[:4]
         truncated_note = (
-            f" (capped at {_EXPORT_MAX_ROWS} rows)" if row_count >= _EXPORT_MAX_ROWS else ""
+            f" (capped at {_EXPORT_MAX_ROWS} rows)"
+            if row_count >= _EXPORT_MAX_ROWS
+            else ""
         )
         return (
             f"Exported {row_count} rows{truncated_note} to {final_path} "

surfsense_backend/app/app.py

@@ -627,9 +627,13 @@ async def _sweep_stale_scraper_runs() -> None:
         async with async_session_maker() as session:
             swept = await fail_stale_running_runs(session)
         if swept:
-            logger.info("[startup] Marked %d stale running scraper run(s) as error", swept)
+            logger.info(
+                "[startup] Marked %d stale running scraper run(s) as error", swept
+            )
     except Exception:
-        logger.warning("[startup] Stale scraper-run sweep failed (non-fatal)", exc_info=True)
+        logger.warning(
+            "[startup] Stale scraper-run sweep failed (non-fatal)", exc_info=True
+        )
 
 
 @asynccontextmanager

surfsense_backend/app/capabilities/core/access/rest.py

@@ -209,7 +209,10 @@ async def _execute_async_run(
             raise
         except (SurfSenseError, HTTPException) as exc:
             await _finalize_async(
-                run_id, status="error", error=str(exc), started=started,
+                run_id,
+                status="error",
+                error=str(exc),
+                started=started,
                 progress=reporter.coarse,
             )
             _publish_finished(run_id, "error", error=str(exc))

surfsense_backend/app/capabilities/core/runs.py

@@ -291,7 +291,9 @@ async def record_spill(
         return None
 
 
-async def _maybe_cleanup(session: AsyncSession, table: str, retention_days: int) -> None:
+async def _maybe_cleanup(
+    session: AsyncSession, table: str, retention_days: int
+) -> None:
     """Delete a bounded batch of expired rows on ~1% of inserts."""
     if random.random() >= _CLEANUP_SAMPLE_RATE:
         return

surfsense_backend/app/capabilities/google_maps/reviews/executor.py

@@ -30,14 +30,21 @@ def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor:
             reviewsStartDate=payload.start_date,
             language=payload.language,
         )
-        emit_progress("starting", "Fetching Google Maps reviews", total=payload.max_reviews, unit="review")
+        emit_progress(
+            "starting",
+            "Fetching Google Maps reviews",
+            total=payload.max_reviews,
+            unit="review",
+        )
         try:
             items = await scrape_fn(actor_input)
         except SignInRequiredError as exc:
             raise ForbiddenError(
                 f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED"
             ) from exc
-        emit_progress("done", f"Scraped {len(items)} review(s)", current=len(items), unit="review")
+        emit_progress(
+            "done", f"Scraped {len(items)} review(s)", current=len(items), unit="review"
+        )
         return ReviewsOutput(items=items)
 
     return execute

surfsense_backend/app/capabilities/google_maps/scrape/executor.py

@@ -33,14 +33,18 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
             maxReviews=payload.max_reviews,
             maxImages=payload.max_images,
         )
-        emit_progress("starting", "Searching Google Maps", total=payload.max_places, unit="place")
+        emit_progress(
+            "starting", "Searching Google Maps", total=payload.max_places, unit="place"
+        )
         try:
             items = await scrape_fn(actor_input)
         except SignInRequiredError as exc:
             raise ForbiddenError(
                 f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED"
             ) from exc
-        emit_progress("done", f"Scraped {len(items)} place(s)", current=len(items), unit="place")
+        emit_progress(
+            "done", f"Scraped {len(items)} place(s)", current=len(items), unit="place"
+        )
         return ScrapeOutput(items=items)
 
     return execute

surfsense_backend/app/capabilities/google_search/scrape/executor.py

@@ -43,7 +43,12 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
             unit="page",
         )
         items = await scrape_fn(actor_input, limit=_MAX_SERP_ITEMS)
-        emit_progress("done", f"Scraped {len(items)} SERP page(s)", current=len(items), unit="page")
+        emit_progress(
+            "done",
+            f"Scraped {len(items)} SERP page(s)",
+            current=len(items),
+            unit="page",
+        )
         return ScrapeOutput(items=items)
 
     return execute

surfsense_backend/app/capabilities/reddit/scrape/executor.py

@@ -36,7 +36,9 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
             postDateLimit=payload.post_date_limit,
             commentDateLimit=payload.comment_date_limit,
         )
-        emit_progress("starting", "Resolving Reddit targets", total=payload.max_items, unit="item")
+        emit_progress(
+            "starting", "Resolving Reddit targets", total=payload.max_items, unit="item"
+        )
         try:
             items = await scrape_fn(actor_input, limit=payload.max_items)
         except RedditAccessBlockedError as exc:
@@ -46,7 +48,9 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
                 f"Reddit refused anonymous access: {exc}",
                 code="REDDIT_ACCESS_BLOCKED",
             ) from exc
-        emit_progress("done", f"Scraped {len(items)} item(s)", current=len(items), unit="item")
+        emit_progress(
+            "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item"
+        )
         return ScrapeOutput(items=items)
 
     return execute

surfsense_backend/app/capabilities/web/crawl/executor.py

@@ -60,7 +60,9 @@ def build_crawl_executor(engine: WebCrawlerConnector | None = None) -> Executor:
             unit="page",
         )
         items = [_to_item(page, payload.maxLength) for page in pages]
-        emit_progress("done", f"Crawled {len(items)} page(s)", current=len(items), unit="page")
+        emit_progress(
+            "done", f"Crawled {len(items)} page(s)", current=len(items), unit="page"
+        )
         return CrawlOutput(
             items=items,
             contacts=_aggregate_contacts(items),

surfsense_backend/app/capabilities/web/crawl/schemas.py

@@ -165,7 +165,8 @@ class CrawlItem(BaseModel):
         default=None, description="Crawl provenance (loaded URL, depth, referrer)."
     )
     markdown: str | None = Field(
-        default=None, description="Cleaned page content as markdown (null unless success)."
+        default=None,
+        description="Cleaned page content as markdown (null unless success).",
     )
     metadata: dict[str, str] | None = Field(
         default=None, description="Page metadata such as title and description."

surfsense_backend/app/capabilities/youtube/comments/executor.py

@@ -25,9 +25,19 @@ def build_comments_executor(scrape_fn: CommentsFn | None = None) -> Executor:
             maxComments=payload.max_comments,
             sortCommentsBy=payload.sort_by,
         )
-        emit_progress("starting", "Fetching YouTube comments", total=payload.max_comments, unit="comment")
+        emit_progress(
+            "starting",
+            "Fetching YouTube comments",
+            total=payload.max_comments,
+            unit="comment",
+        )
         items = await scrape_fn(actor_input)
-        emit_progress("done", f"Scraped {len(items)} comment(s)", current=len(items), unit="comment")
+        emit_progress(
+            "done",
+            f"Scraped {len(items)} comment(s)",
+            current=len(items),
+            unit="comment",
+        )
         return CommentsOutput(items=items)
 
     return execute

surfsense_backend/app/capabilities/youtube/scrape/executor.py

@@ -31,9 +31,16 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
             downloadSubtitles=payload.download_subtitles,
             subtitlesLanguage=payload.subtitles_language,
         )
-        emit_progress("starting", "Resolving YouTube targets", total=payload.max_results, unit="video")
+        emit_progress(
+            "starting",
+            "Resolving YouTube targets",
+            total=payload.max_results,
+            unit="video",
+        )
         items = await scrape_fn(actor_input)
-        emit_progress("done", f"Scraped {len(items)} video(s)", current=len(items), unit="video")
+        emit_progress(
+            "done", f"Scraped {len(items)} video(s)", current=len(items), unit="video"
+        )
         return ScrapeOutput(items=items)
 
     return execute

surfsense_backend/app/config/__init__.py

@@ -713,9 +713,7 @@ class Config:
     YOUTUBE_MICROS_PER_VIDEO = int(os.getenv("YOUTUBE_MICROS_PER_VIDEO", "2500"))
     # Kept separate from the video rate so comments can be re-tuned toward the
     # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing.
-    YOUTUBE_MICROS_PER_COMMENT = int(
-        os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500")
-    )
+    YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500"))
 
     # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service
     # so the UI can nudge the user to top up / enable auto-reload. $0.50.

surfsense_backend/app/db.py

@@ -2795,9 +2795,7 @@ class Run(Base, TimestampMixin):
     __tablename__ = "runs"
     __allow_unmapped__ = True
 
-    __table_args__ = (
-        Index("ix_runs_workspace_created", "workspace_id", "created_at"),
-    )
+    __table_args__ = (Index("ix_runs_workspace_created", "workspace_id", "created_at"),)
 
     id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     workspace_id = Column(

surfsense_backend/app/proprietary/platforms/google_search/fetch.py

@@ -31,6 +31,7 @@ if we add another sticky vendor is to extend ``_STICKY_HOSTS``.
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import logging
 import random
 import sys
@@ -158,8 +159,7 @@ async def _vet_fresh_ip(url: str, base: str) -> str | None:
         return proxy if await _precheck(url, proxy) else None
 
     tasks = [
-        asyncio.create_task(vet(_sticky_variant(base)))
-        for _ in range(_VET_CONCURRENCY)
+        asyncio.create_task(vet(_sticky_variant(base))) for _ in range(_VET_CONCURRENCY)
     ]
     winner: str | None = None
     for fut in asyncio.as_completed(tasks):
@@ -306,10 +306,8 @@ async def _drop_session_on_loop(mobile: bool) -> None:
     async with _session_lock:
         session = _sessions.pop(mobile, None)
     if session is not None:
-        try:
+        with contextlib.suppress(Exception):  # already dead; nothing to salvage
             await session.close()
-        except Exception:  # already dead; nothing to salvage
-            pass
 
 
 async def _drop_session(mobile: bool) -> None:

surfsense_backend/app/proprietary/platforms/google_search/parsers.py

@@ -218,7 +218,9 @@ def parse_organic(doc: Adaptor, *, include_icons: bool = False) -> list[OrganicR
     return results
 
 
-def parse_paid_results(doc: Adaptor, *, include_icons: bool = False) -> list[PaidResult]:
+def parse_paid_results(
+    doc: Adaptor, *, include_icons: bool = False
+) -> list[PaidResult]:
     """Text ads (``div[data-text-ad]``), covering the top and bottom ad blocks.
 
     Fields mirror an organic result: the heading is the title, the ad's anchor
@@ -239,7 +241,11 @@ def parse_paid_results(doc: Adaptor, *, include_icons: bool = False) -> list[Pai
         description = None
         for cand in block.css(".Va3FIb"):
             text = _text(cand)
-            if text and text != title and (description is None or len(text) > len(description)):
+            if (
+                text
+                and text != title
+                and (description is None or len(text) > len(description))
+            ):
                 description = text
         slot = block.attrib.get("data-ta-slot-pos")
         ads.append(

surfsense_backend/app/proprietary/platforms/reddit/scraper.py

@@ -291,8 +291,8 @@ async def _user_flow(
         date_limit=input_model.postDateLimit,
     ):
         # A user listing mixes posts (t3) and comments (t1); a post has a title.
-        parsed = parse_post(data) if data.get("title") is not None else parse_comment(
-            data
+        parsed = (
+            parse_post(data) if data.get("title") is not None else parse_comment(data)
         )
         item = _emit(parsed, include_nsfw=input_model.includeNSFW)
         if item is not None:
@@ -401,8 +401,7 @@ async def iter_reddit(
                 continue
             resolved.append(r)
         jobs = [
-            _dispatch(r, input_model)
-            for r in _capped_targets(resolved, input_model)
+            _dispatch(r, input_model) for r in _capped_targets(resolved, input_model)
         ]
         async for item in fan_out(jobs):
             yield item
@@ -449,9 +448,7 @@ async def scrape_reddit(
     results: list[dict[str, Any]] = []
     async for item in iter_reddit(input_model):
         results.append(item)
-        emit_progress(
-            "scraping", current=len(results), total=limit, unit="item"
-        )
+        emit_progress("scraping", current=len(results), total=limit, unit="item")
         if limit is not None and len(results) >= limit:
             break
     return results

surfsense_backend/app/proprietary/platforms/reddit/url_resolver.py

@@ -28,9 +28,7 @@ _REDDIT_HOSTS = frozenset(
 )
 
 # Listing sorts that can appear as a trailing subreddit path segment.
-_SUBREDDIT_SORTS = frozenset(
-    {"hot", "new", "top", "rising", "controversial", "best"}
-)
+_SUBREDDIT_SORTS = frozenset({"hot", "new", "top", "rising", "controversial", "best"})
 _USER_CONTENT = frozenset({"overview", "submitted", "comments"})
 
 

surfsense_backend/app/proprietary/platforms/youtube/subtitles.py

@@ -69,9 +69,7 @@ def _fetch_subtitles_sync(
         api = _build_client()
         try:
             transcript_list = api.list(video_id)
-            transcript = _select_transcript(
-                transcript_list, language, prefer_generated
-            )
+            transcript = _select_transcript(transcript_list, language, prefer_generated)
             fetched = transcript.fetch()
             break
         except RequestBlocked:  # covers IpBlocked; rotate to a fresh exit IP

surfsense_backend/app/proprietary/web_crawler/connector.py

@@ -91,7 +91,9 @@ _CURRENCY_AMOUNT_RE = re.compile(
     re.IGNORECASE,
 )
 
-_STRIP_XPATH = "//script | //style | //noscript | //template | //svg | //iframe | //head"
+_STRIP_XPATH = (
+    "//script | //style | //noscript | //template | //svg | //iframe | //head"
+)
 
 
 def _visible_text(raw_html: str) -> str:
@@ -278,9 +280,7 @@ class WebCrawlerConnector:
                 else:
                     reached_without_content = True
                     errors.append("Scrapling static: empty extraction")
-                    self._log_tier_outcome(
-                        "scrapling-static", url, tier_start, "empty"
-                    )
+                    self._log_tier_outcome("scrapling-static", url, tier_start, "empty")
             except Exception as exc:
                 errors.append(f"Scrapling static: {exc!s}")
                 self._log_tier_outcome(

surfsense_backend/app/proprietary/web_crawler/site_crawler.py

@@ -156,9 +156,7 @@ class _SiteSpider(Spider):
         # ladder + proxy rotation; never let the spider re-fetch on top of that.
         return False
 
-    async def parse(
-        self, response: Response
-    ) -> AsyncGenerator[Request | None, None]:
+    async def parse(self, response: Response) -> AsyncGenerator[Request | None, None]:
         outcome: CrawlOutcome = response._outcome  # type: ignore[attr-defined]
         depth: int = response.meta.get("_depth", 0)
         referrer: str | None = response.meta.get("_referrer")

surfsense_backend/app/proprietary/web_crawler/url_policy.py

@@ -73,9 +73,7 @@ def _anchor_context(anchor: Any) -> str:
     return ""
 
 
-def extract_link_records(
-    page_html: str | None, base_url: str
-) -> list[dict[str, str]]:
+def extract_link_records(page_html: str | None, base_url: str) -> list[dict[str, str]]:
     """Structured ``<a>`` inventory: ``{url, text, context, rel, kind}`` per target.
 
     ``kind`` is one of ``internal`` (same site as ``base_url``), ``external``,

surfsense_backend/app/services/mcp_oauth/registry.py

@@ -218,9 +218,7 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = {
             "create-pages",
             "update-page",
         ],
-        readonly_tools=frozenset(
-            {"notion-search", "notion-fetch", "search", "fetch"}
-        ),
+        readonly_tools=frozenset({"notion-search", "notion-fetch", "search", "fetch"}),
         account_metadata_keys=["workspace_name"],
     ),
     "confluence": MCPServiceConfig(

surfsense_backend/app/services/wallet_credit.py

@@ -22,7 +22,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.services.etl_credit_service import InsufficientCreditsError
 
-__all__ = ["InsufficientCreditsError", "apply_debit", "check_balance", "spendable_micros"]
+__all__ = [
+    "InsufficientCreditsError",
+    "apply_debit",
+    "check_balance",
+    "spendable_micros",
+]
 
 
 async def spendable_micros(session: AsyncSession, user_id: str | UUID) -> int:

surfsense_backend/app/tasks/celery_tasks/document_tasks.py

@@ -1393,4 +1393,3 @@ async def _index_uploaded_folder_files_async(
                 heartbeat_task.cancel()
             if notification_id is not None:
                 _stop_heartbeat(notification_id)
-

surfsense_backend/app/utils/crawl/contacts.py

@@ -87,9 +87,30 @@ _NOISE_EMAIL_DOMAINS = frozenset(
 # or version-pinned dep (``react@18.2.0.js``) matches the email shape.
 _ASSET_TLDS = frozenset(
     {
-        "png", "jpg", "jpeg", "gif", "svg", "webp", "ico", "bmp",
-        "css", "js", "mjs", "cjs", "ts", "map", "json", "xml",
-        "woff", "woff2", "ttf", "eot", "otf", "php", "html", "htm",
+        "png",
+        "jpg",
+        "jpeg",
+        "gif",
+        "svg",
+        "webp",
+        "ico",
+        "bmp",
+        "css",
+        "js",
+        "mjs",
+        "cjs",
+        "ts",
+        "map",
+        "json",
+        "xml",
+        "woff",
+        "woff2",
+        "ttf",
+        "eot",
+        "otf",
+        "php",
+        "html",
+        "htm",
     }
 )
 
@@ -100,19 +121,47 @@ _EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}")
 # common locals like hello/info/contact/support.
 _PLACEHOLDER_EMAIL_LOCALS = frozenset(
     {
-        "youremail", "yourname", "youraddress", "myemail", "email", "name",
-        "user", "username", "someone", "somebody", "johndoe", "janedoe",
-        "firstname", "lastname", "firstnamelastname", "firstlast",
-        "test", "example", "sample", "placeholder",
+        "youremail",
+        "yourname",
+        "youraddress",
+        "myemail",
+        "email",
+        "name",
+        "user",
+        "username",
+        "someone",
+        "somebody",
+        "johndoe",
+        "janedoe",
+        "firstname",
+        "lastname",
+        "firstnamelastname",
+        "firstlast",
+        "test",
+        "example",
+        "sample",
+        "placeholder",
     }
 )
 
 # Placeholder profile handles left in site templates ("github.com/username").
 _PLACEHOLDER_SOCIAL_SEGMENTS = frozenset(
     {
-        "username", "yourusername", "yourhandle", "handle", "user",
-        "profile", "yourprofile", "yourname", "yourpage", "pagename",
-        "youraccount", "account", "example", "placeholder", "yourcompany",
+        "username",
+        "yourusername",
+        "yourhandle",
+        "handle",
+        "user",
+        "profile",
+        "yourprofile",
+        "yourname",
+        "yourpage",
+        "pagename",
+        "youraccount",
+        "account",
+        "example",
+        "placeholder",
+        "yourcompany",
     }
 )