diff --git a/VERSION b/VERSION index f092e2be2..d788d4335 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.30 +0.0.31 diff --git a/surfsense_backend/alembic/env.py b/surfsense_backend/alembic/env.py index ce20133c6..f2130892a 100644 --- a/surfsense_backend/alembic/env.py +++ b/surfsense_backend/alembic/env.py @@ -5,12 +5,12 @@ import sys from logging.config import fileConfig import sqlalchemy as sa -from alembic.script import ScriptDirectory from sqlalchemy import pool from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config from alembic import context +from alembic.script import ScriptDirectory # Ensure the app directory is in the Python path # This allows Alembic to find your models @@ -81,9 +81,7 @@ def _fast_forward_fresh_db(connection: Connection) -> bool: step rather than resurrecting the replay. """ for table in ("documents", "searchspaces", BOOTSTRAP_MARKER_TABLE): - if connection.execute( - sa.text("SELECT to_regclass(:t)"), {"t": table} - ).scalar(): + if connection.execute(sa.text("SELECT to_regclass(:t)"), {"t": table}).scalar(): return False if connection.execute(sa.text("SELECT to_regclass('alembic_version')")).scalar(): current = connection.execute( diff --git a/surfsense_backend/alembic/versions/171_add_runs_and_tool_output_spills.py b/surfsense_backend/alembic/versions/171_add_runs_and_tool_output_spills.py index 34c3628eb..2511ef3f9 100644 --- a/surfsense_backend/alembic/versions/171_add_runs_and_tool_output_spills.py +++ b/surfsense_backend/alembic/versions/171_add_runs_and_tool_output_spills.py @@ -41,9 +41,7 @@ def upgrade() -> None: ); """ ) - op.execute( - "CREATE INDEX IF NOT EXISTS ix_runs_workspace_id ON runs (workspace_id)" - ) + op.execute("CREATE INDEX IF NOT EXISTS ix_runs_workspace_id ON runs (workspace_id)") op.execute("CREATE INDEX IF NOT EXISTS ix_runs_user_id ON runs (user_id)") op.execute("CREATE INDEX IF NOT EXISTS ix_runs_capability ON runs (capability)") op.execute("CREATE INDEX IF NOT EXISTS ix_runs_created_at ON runs (created_at)") diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/agent.py index 3a4b45319..f4cc503c3 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/agent.py @@ -50,9 +50,7 @@ def _augment_allowlist_for_collision_prefixes( continue original = meta.get("mcp_original_tool_name") if original in trusted_names and tool.name not in trusted_names: - alias_rules.append( - Rule(permission=tool.name, pattern="*", action="allow") - ) + alias_rules.append(Rule(permission=tool.name, pattern="*", action="allow")) if not alias_rules: return dependencies diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/get_connected_accounts.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/get_connected_accounts.py index 0f98b5f24..292530229 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/get_connected_accounts.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/get_connected_accounts.py @@ -85,8 +85,7 @@ def create_get_connected_accounts_tool(*, workspace_id: int) -> BaseTool: # ``server_config`` presence == the connector produces agent # tools. Native rows without it are connected for indexing # only and need a reconnect via MCP. - "usable_in_chat": isinstance(cfg, dict) - and "server_config" in cfg, + "usable_in_chat": isinstance(cfg, dict) and "server_config" in cfg, "account": account_meta, } ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py index eae0105ac..6359e9842 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py @@ -83,7 +83,9 @@ def resolve_tool_name_collisions( or original_name, "mcp_collision_prefixed": True, } - resolved.append(tool.model_copy(update={"name": new_name, "metadata": new_meta})) + resolved.append( + tool.model_copy(update={"name": new_name, "metadata": new_meta}) + ) return resolved diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py index 7ef03a68c..2b2c57cc4 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py @@ -128,7 +128,7 @@ def _cell(value: Any) -> str: """Render one CSV cell: scalars as-is, nested structures as compact JSON.""" if value is None: return "" - if isinstance(value, (dict, list)): + if isinstance(value, dict | list): return json.dumps(value, ensure_ascii=False, default=str) return str(value) @@ -230,9 +230,7 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]: count = min(max(1, limit), _MAX_LIMIT) window = lines[start : start + count] if not window: - return ( - f"No lines at offset {start} (total {len(lines)} lines in {ref})." - ) + return f"No lines at offset {start} (total {len(lines)} lines in {ref})." window_body = "\n".join(window) start_char = max(0, char_offset) if start_char >= len(window_body) > 0: @@ -328,10 +326,14 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]: records = _rows_from_body(body, rows) if include_pattern: inc = _build_matcher(include_pattern.strip()) - records = [r for r in records if inc(" ".join(map(_cell, r.values()))) is not None] + records = [ + r for r in records if inc(" ".join(map(_cell, r.values()))) is not None + ] if exclude_pattern: exc = _build_matcher(exclude_pattern.strip()) - records = [r for r in records if exc(" ".join(map(_cell, r.values()))) is None] + records = [ + r for r in records if exc(" ".join(map(_cell, r.values()))) is None + ] if not records: return ( f"Error: no rows to export from {ref} " @@ -353,7 +355,9 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]: preview_lines = csv_text.split("\n")[:4] truncated_note = ( - f" (capped at {_EXPORT_MAX_ROWS} rows)" if row_count >= _EXPORT_MAX_ROWS else "" + f" (capped at {_EXPORT_MAX_ROWS} rows)" + if row_count >= _EXPORT_MAX_ROWS + else "" ) return ( f"Exported {row_count} rows{truncated_note} to {final_path} " diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index f5618fdf0..dde2ce7fa 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -627,9 +627,13 @@ async def _sweep_stale_scraper_runs() -> None: async with async_session_maker() as session: swept = await fail_stale_running_runs(session) if swept: - logger.info("[startup] Marked %d stale running scraper run(s) as error", swept) + logger.info( + "[startup] Marked %d stale running scraper run(s) as error", swept + ) except Exception: - logger.warning("[startup] Stale scraper-run sweep failed (non-fatal)", exc_info=True) + logger.warning( + "[startup] Stale scraper-run sweep failed (non-fatal)", exc_info=True + ) @asynccontextmanager diff --git a/surfsense_backend/app/capabilities/core/access/rest.py b/surfsense_backend/app/capabilities/core/access/rest.py index 59e8bc8ca..328a7c5a1 100644 --- a/surfsense_backend/app/capabilities/core/access/rest.py +++ b/surfsense_backend/app/capabilities/core/access/rest.py @@ -209,7 +209,10 @@ async def _execute_async_run( raise except (SurfSenseError, HTTPException) as exc: await _finalize_async( - run_id, status="error", error=str(exc), started=started, + run_id, + status="error", + error=str(exc), + started=started, progress=reporter.coarse, ) _publish_finished(run_id, "error", error=str(exc)) diff --git a/surfsense_backend/app/capabilities/core/runs.py b/surfsense_backend/app/capabilities/core/runs.py index 5ad9edbc5..161ec0bf7 100644 --- a/surfsense_backend/app/capabilities/core/runs.py +++ b/surfsense_backend/app/capabilities/core/runs.py @@ -291,7 +291,9 @@ async def record_spill( return None -async def _maybe_cleanup(session: AsyncSession, table: str, retention_days: int) -> None: +async def _maybe_cleanup( + session: AsyncSession, table: str, retention_days: int +) -> None: """Delete a bounded batch of expired rows on ~1% of inserts.""" if random.random() >= _CLEANUP_SAMPLE_RATE: return diff --git a/surfsense_backend/app/capabilities/google_maps/reviews/executor.py b/surfsense_backend/app/capabilities/google_maps/reviews/executor.py index e7edd1dd8..c6bec3775 100644 --- a/surfsense_backend/app/capabilities/google_maps/reviews/executor.py +++ b/surfsense_backend/app/capabilities/google_maps/reviews/executor.py @@ -30,14 +30,21 @@ def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor: reviewsStartDate=payload.start_date, language=payload.language, ) - emit_progress("starting", "Fetching Google Maps reviews", total=payload.max_reviews, unit="review") + emit_progress( + "starting", + "Fetching Google Maps reviews", + total=payload.max_reviews, + unit="review", + ) try: items = await scrape_fn(actor_input) except SignInRequiredError as exc: raise ForbiddenError( f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED" ) from exc - emit_progress("done", f"Scraped {len(items)} review(s)", current=len(items), unit="review") + emit_progress( + "done", f"Scraped {len(items)} review(s)", current=len(items), unit="review" + ) return ReviewsOutput(items=items) return execute diff --git a/surfsense_backend/app/capabilities/google_maps/scrape/executor.py b/surfsense_backend/app/capabilities/google_maps/scrape/executor.py index 73519b41c..12b5c25bf 100644 --- a/surfsense_backend/app/capabilities/google_maps/scrape/executor.py +++ b/surfsense_backend/app/capabilities/google_maps/scrape/executor.py @@ -33,14 +33,18 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: maxReviews=payload.max_reviews, maxImages=payload.max_images, ) - emit_progress("starting", "Searching Google Maps", total=payload.max_places, unit="place") + emit_progress( + "starting", "Searching Google Maps", total=payload.max_places, unit="place" + ) try: items = await scrape_fn(actor_input) except SignInRequiredError as exc: raise ForbiddenError( f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED" ) from exc - emit_progress("done", f"Scraped {len(items)} place(s)", current=len(items), unit="place") + emit_progress( + "done", f"Scraped {len(items)} place(s)", current=len(items), unit="place" + ) return ScrapeOutput(items=items) return execute diff --git a/surfsense_backend/app/capabilities/google_search/scrape/executor.py b/surfsense_backend/app/capabilities/google_search/scrape/executor.py index 593ec6240..022f1eac6 100644 --- a/surfsense_backend/app/capabilities/google_search/scrape/executor.py +++ b/surfsense_backend/app/capabilities/google_search/scrape/executor.py @@ -43,7 +43,12 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: unit="page", ) items = await scrape_fn(actor_input, limit=_MAX_SERP_ITEMS) - emit_progress("done", f"Scraped {len(items)} SERP page(s)", current=len(items), unit="page") + emit_progress( + "done", + f"Scraped {len(items)} SERP page(s)", + current=len(items), + unit="page", + ) return ScrapeOutput(items=items) return execute diff --git a/surfsense_backend/app/capabilities/reddit/scrape/executor.py b/surfsense_backend/app/capabilities/reddit/scrape/executor.py index a3fdc8557..da72c8bcf 100644 --- a/surfsense_backend/app/capabilities/reddit/scrape/executor.py +++ b/surfsense_backend/app/capabilities/reddit/scrape/executor.py @@ -36,7 +36,9 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: postDateLimit=payload.post_date_limit, commentDateLimit=payload.comment_date_limit, ) - emit_progress("starting", "Resolving Reddit targets", total=payload.max_items, unit="item") + emit_progress( + "starting", "Resolving Reddit targets", total=payload.max_items, unit="item" + ) try: items = await scrape_fn(actor_input, limit=payload.max_items) except RedditAccessBlockedError as exc: @@ -46,7 +48,9 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: f"Reddit refused anonymous access: {exc}", code="REDDIT_ACCESS_BLOCKED", ) from exc - emit_progress("done", f"Scraped {len(items)} item(s)", current=len(items), unit="item") + emit_progress( + "done", f"Scraped {len(items)} item(s)", current=len(items), unit="item" + ) return ScrapeOutput(items=items) return execute diff --git a/surfsense_backend/app/capabilities/web/crawl/executor.py b/surfsense_backend/app/capabilities/web/crawl/executor.py index 069486b67..75d2f9c48 100644 --- a/surfsense_backend/app/capabilities/web/crawl/executor.py +++ b/surfsense_backend/app/capabilities/web/crawl/executor.py @@ -60,7 +60,9 @@ def build_crawl_executor(engine: WebCrawlerConnector | None = None) -> Executor: unit="page", ) items = [_to_item(page, payload.maxLength) for page in pages] - emit_progress("done", f"Crawled {len(items)} page(s)", current=len(items), unit="page") + emit_progress( + "done", f"Crawled {len(items)} page(s)", current=len(items), unit="page" + ) return CrawlOutput( items=items, contacts=_aggregate_contacts(items), diff --git a/surfsense_backend/app/capabilities/web/crawl/schemas.py b/surfsense_backend/app/capabilities/web/crawl/schemas.py index 6894bd8e9..d6c9c38a3 100644 --- a/surfsense_backend/app/capabilities/web/crawl/schemas.py +++ b/surfsense_backend/app/capabilities/web/crawl/schemas.py @@ -165,7 +165,8 @@ class CrawlItem(BaseModel): default=None, description="Crawl provenance (loaded URL, depth, referrer)." ) markdown: str | None = Field( - default=None, description="Cleaned page content as markdown (null unless success)." + default=None, + description="Cleaned page content as markdown (null unless success).", ) metadata: dict[str, str] | None = Field( default=None, description="Page metadata such as title and description." diff --git a/surfsense_backend/app/capabilities/youtube/comments/executor.py b/surfsense_backend/app/capabilities/youtube/comments/executor.py index c311a7a60..a682f118d 100644 --- a/surfsense_backend/app/capabilities/youtube/comments/executor.py +++ b/surfsense_backend/app/capabilities/youtube/comments/executor.py @@ -25,9 +25,19 @@ def build_comments_executor(scrape_fn: CommentsFn | None = None) -> Executor: maxComments=payload.max_comments, sortCommentsBy=payload.sort_by, ) - emit_progress("starting", "Fetching YouTube comments", total=payload.max_comments, unit="comment") + emit_progress( + "starting", + "Fetching YouTube comments", + total=payload.max_comments, + unit="comment", + ) items = await scrape_fn(actor_input) - emit_progress("done", f"Scraped {len(items)} comment(s)", current=len(items), unit="comment") + emit_progress( + "done", + f"Scraped {len(items)} comment(s)", + current=len(items), + unit="comment", + ) return CommentsOutput(items=items) return execute diff --git a/surfsense_backend/app/capabilities/youtube/scrape/executor.py b/surfsense_backend/app/capabilities/youtube/scrape/executor.py index 0e7539005..1a1ce175d 100644 --- a/surfsense_backend/app/capabilities/youtube/scrape/executor.py +++ b/surfsense_backend/app/capabilities/youtube/scrape/executor.py @@ -31,9 +31,16 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: downloadSubtitles=payload.download_subtitles, subtitlesLanguage=payload.subtitles_language, ) - emit_progress("starting", "Resolving YouTube targets", total=payload.max_results, unit="video") + emit_progress( + "starting", + "Resolving YouTube targets", + total=payload.max_results, + unit="video", + ) items = await scrape_fn(actor_input) - emit_progress("done", f"Scraped {len(items)} video(s)", current=len(items), unit="video") + emit_progress( + "done", f"Scraped {len(items)} video(s)", current=len(items), unit="video" + ) return ScrapeOutput(items=items) return execute diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 68ea93de6..5d8e96881 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -713,9 +713,7 @@ class Config: YOUTUBE_MICROS_PER_VIDEO = int(os.getenv("YOUTUBE_MICROS_PER_VIDEO", "2500")) # Kept separate from the video rate so comments can be re-tuned toward the # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing. - YOUTUBE_MICROS_PER_COMMENT = int( - os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500") - ) + YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500")) # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # so the UI can nudge the user to top up / enable auto-reload. $0.50. diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index b44ec2e13..380a8b2bf 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -2795,9 +2795,7 @@ class Run(Base, TimestampMixin): __tablename__ = "runs" __allow_unmapped__ = True - __table_args__ = ( - Index("ix_runs_workspace_created", "workspace_id", "created_at"), - ) + __table_args__ = (Index("ix_runs_workspace_created", "workspace_id", "created_at"),) id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) workspace_id = Column( diff --git a/surfsense_backend/app/proprietary/platforms/google_search/fetch.py b/surfsense_backend/app/proprietary/platforms/google_search/fetch.py index c969d6bc4..68cc90d26 100644 --- a/surfsense_backend/app/proprietary/platforms/google_search/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/google_search/fetch.py @@ -31,6 +31,7 @@ if we add another sticky vendor is to extend ``_STICKY_HOSTS``. from __future__ import annotations import asyncio +import contextlib import logging import random import sys @@ -158,8 +159,7 @@ async def _vet_fresh_ip(url: str, base: str) -> str | None: return proxy if await _precheck(url, proxy) else None tasks = [ - asyncio.create_task(vet(_sticky_variant(base))) - for _ in range(_VET_CONCURRENCY) + asyncio.create_task(vet(_sticky_variant(base))) for _ in range(_VET_CONCURRENCY) ] winner: str | None = None for fut in asyncio.as_completed(tasks): @@ -306,10 +306,8 @@ async def _drop_session_on_loop(mobile: bool) -> None: async with _session_lock: session = _sessions.pop(mobile, None) if session is not None: - try: + with contextlib.suppress(Exception): # already dead; nothing to salvage await session.close() - except Exception: # already dead; nothing to salvage - pass async def _drop_session(mobile: bool) -> None: diff --git a/surfsense_backend/app/proprietary/platforms/google_search/parsers.py b/surfsense_backend/app/proprietary/platforms/google_search/parsers.py index e74fb4549..04e278079 100644 --- a/surfsense_backend/app/proprietary/platforms/google_search/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/google_search/parsers.py @@ -218,7 +218,9 @@ def parse_organic(doc: Adaptor, *, include_icons: bool = False) -> list[OrganicR return results -def parse_paid_results(doc: Adaptor, *, include_icons: bool = False) -> list[PaidResult]: +def parse_paid_results( + doc: Adaptor, *, include_icons: bool = False +) -> list[PaidResult]: """Text ads (``div[data-text-ad]``), covering the top and bottom ad blocks. Fields mirror an organic result: the heading is the title, the ad's anchor @@ -239,7 +241,11 @@ def parse_paid_results(doc: Adaptor, *, include_icons: bool = False) -> list[Pai description = None for cand in block.css(".Va3FIb"): text = _text(cand) - if text and text != title and (description is None or len(text) > len(description)): + if ( + text + and text != title + and (description is None or len(text) > len(description)) + ): description = text slot = block.attrib.get("data-ta-slot-pos") ads.append( diff --git a/surfsense_backend/app/proprietary/platforms/reddit/scraper.py b/surfsense_backend/app/proprietary/platforms/reddit/scraper.py index 0af12d5e5..a14e1c13f 100644 --- a/surfsense_backend/app/proprietary/platforms/reddit/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/reddit/scraper.py @@ -291,8 +291,8 @@ async def _user_flow( date_limit=input_model.postDateLimit, ): # A user listing mixes posts (t3) and comments (t1); a post has a title. - parsed = parse_post(data) if data.get("title") is not None else parse_comment( - data + parsed = ( + parse_post(data) if data.get("title") is not None else parse_comment(data) ) item = _emit(parsed, include_nsfw=input_model.includeNSFW) if item is not None: @@ -401,8 +401,7 @@ async def iter_reddit( continue resolved.append(r) jobs = [ - _dispatch(r, input_model) - for r in _capped_targets(resolved, input_model) + _dispatch(r, input_model) for r in _capped_targets(resolved, input_model) ] async for item in fan_out(jobs): yield item @@ -449,9 +448,7 @@ async def scrape_reddit( results: list[dict[str, Any]] = [] async for item in iter_reddit(input_model): results.append(item) - emit_progress( - "scraping", current=len(results), total=limit, unit="item" - ) + emit_progress("scraping", current=len(results), total=limit, unit="item") if limit is not None and len(results) >= limit: break return results diff --git a/surfsense_backend/app/proprietary/platforms/reddit/url_resolver.py b/surfsense_backend/app/proprietary/platforms/reddit/url_resolver.py index 0af80b524..8490d9de6 100644 --- a/surfsense_backend/app/proprietary/platforms/reddit/url_resolver.py +++ b/surfsense_backend/app/proprietary/platforms/reddit/url_resolver.py @@ -28,9 +28,7 @@ _REDDIT_HOSTS = frozenset( ) # Listing sorts that can appear as a trailing subreddit path segment. -_SUBREDDIT_SORTS = frozenset( - {"hot", "new", "top", "rising", "controversial", "best"} -) +_SUBREDDIT_SORTS = frozenset({"hot", "new", "top", "rising", "controversial", "best"}) _USER_CONTENT = frozenset({"overview", "submitted", "comments"}) diff --git a/surfsense_backend/app/proprietary/platforms/youtube/subtitles.py b/surfsense_backend/app/proprietary/platforms/youtube/subtitles.py index 31cb672f0..c249d7d6c 100644 --- a/surfsense_backend/app/proprietary/platforms/youtube/subtitles.py +++ b/surfsense_backend/app/proprietary/platforms/youtube/subtitles.py @@ -69,9 +69,7 @@ def _fetch_subtitles_sync( api = _build_client() try: transcript_list = api.list(video_id) - transcript = _select_transcript( - transcript_list, language, prefer_generated - ) + transcript = _select_transcript(transcript_list, language, prefer_generated) fetched = transcript.fetch() break except RequestBlocked: # covers IpBlocked; rotate to a fresh exit IP diff --git a/surfsense_backend/app/proprietary/web_crawler/connector.py b/surfsense_backend/app/proprietary/web_crawler/connector.py index c44340334..76118b118 100644 --- a/surfsense_backend/app/proprietary/web_crawler/connector.py +++ b/surfsense_backend/app/proprietary/web_crawler/connector.py @@ -91,7 +91,9 @@ _CURRENCY_AMOUNT_RE = re.compile( re.IGNORECASE, ) -_STRIP_XPATH = "//script | //style | //noscript | //template | //svg | //iframe | //head" +_STRIP_XPATH = ( + "//script | //style | //noscript | //template | //svg | //iframe | //head" +) def _visible_text(raw_html: str) -> str: @@ -278,9 +280,7 @@ class WebCrawlerConnector: else: reached_without_content = True errors.append("Scrapling static: empty extraction") - self._log_tier_outcome( - "scrapling-static", url, tier_start, "empty" - ) + self._log_tier_outcome("scrapling-static", url, tier_start, "empty") except Exception as exc: errors.append(f"Scrapling static: {exc!s}") self._log_tier_outcome( diff --git a/surfsense_backend/app/proprietary/web_crawler/site_crawler.py b/surfsense_backend/app/proprietary/web_crawler/site_crawler.py index 0fef706a9..546e74aee 100644 --- a/surfsense_backend/app/proprietary/web_crawler/site_crawler.py +++ b/surfsense_backend/app/proprietary/web_crawler/site_crawler.py @@ -156,9 +156,7 @@ class _SiteSpider(Spider): # ladder + proxy rotation; never let the spider re-fetch on top of that. return False - async def parse( - self, response: Response - ) -> AsyncGenerator[Request | None, None]: + async def parse(self, response: Response) -> AsyncGenerator[Request | None, None]: outcome: CrawlOutcome = response._outcome # type: ignore[attr-defined] depth: int = response.meta.get("_depth", 0) referrer: str | None = response.meta.get("_referrer") diff --git a/surfsense_backend/app/proprietary/web_crawler/url_policy.py b/surfsense_backend/app/proprietary/web_crawler/url_policy.py index 46f74c8c3..db07d6aa9 100644 --- a/surfsense_backend/app/proprietary/web_crawler/url_policy.py +++ b/surfsense_backend/app/proprietary/web_crawler/url_policy.py @@ -73,9 +73,7 @@ def _anchor_context(anchor: Any) -> str: return "" -def extract_link_records( - page_html: str | None, base_url: str -) -> list[dict[str, str]]: +def extract_link_records(page_html: str | None, base_url: str) -> list[dict[str, str]]: """Structured ```` inventory: ``{url, text, context, rel, kind}`` per target. ``kind`` is one of ``internal`` (same site as ``base_url``), ``external``, diff --git a/surfsense_backend/app/services/mcp_oauth/registry.py b/surfsense_backend/app/services/mcp_oauth/registry.py index f4cfabfd0..c61aedfd6 100644 --- a/surfsense_backend/app/services/mcp_oauth/registry.py +++ b/surfsense_backend/app/services/mcp_oauth/registry.py @@ -218,9 +218,7 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = { "create-pages", "update-page", ], - readonly_tools=frozenset( - {"notion-search", "notion-fetch", "search", "fetch"} - ), + readonly_tools=frozenset({"notion-search", "notion-fetch", "search", "fetch"}), account_metadata_keys=["workspace_name"], ), "confluence": MCPServiceConfig( diff --git a/surfsense_backend/app/services/wallet_credit.py b/surfsense_backend/app/services/wallet_credit.py index 218dd777f..cfac2a335 100644 --- a/surfsense_backend/app/services/wallet_credit.py +++ b/surfsense_backend/app/services/wallet_credit.py @@ -22,7 +22,12 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.services.etl_credit_service import InsufficientCreditsError -__all__ = ["InsufficientCreditsError", "apply_debit", "check_balance", "spendable_micros"] +__all__ = [ + "InsufficientCreditsError", + "apply_debit", + "check_balance", + "spendable_micros", +] async def spendable_micros(session: AsyncSession, user_id: str | UUID) -> int: diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index 3e26050ef..e2ca5345e 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -1393,4 +1393,3 @@ async def _index_uploaded_folder_files_async( heartbeat_task.cancel() if notification_id is not None: _stop_heartbeat(notification_id) - diff --git a/surfsense_backend/app/utils/crawl/contacts.py b/surfsense_backend/app/utils/crawl/contacts.py index b487b10f1..872db33d9 100644 --- a/surfsense_backend/app/utils/crawl/contacts.py +++ b/surfsense_backend/app/utils/crawl/contacts.py @@ -87,9 +87,30 @@ _NOISE_EMAIL_DOMAINS = frozenset( # or version-pinned dep (``react@18.2.0.js``) matches the email shape. _ASSET_TLDS = frozenset( { - "png", "jpg", "jpeg", "gif", "svg", "webp", "ico", "bmp", - "css", "js", "mjs", "cjs", "ts", "map", "json", "xml", - "woff", "woff2", "ttf", "eot", "otf", "php", "html", "htm", + "png", + "jpg", + "jpeg", + "gif", + "svg", + "webp", + "ico", + "bmp", + "css", + "js", + "mjs", + "cjs", + "ts", + "map", + "json", + "xml", + "woff", + "woff2", + "ttf", + "eot", + "otf", + "php", + "html", + "htm", } ) @@ -100,19 +121,47 @@ _EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}") # common locals like hello/info/contact/support. _PLACEHOLDER_EMAIL_LOCALS = frozenset( { - "youremail", "yourname", "youraddress", "myemail", "email", "name", - "user", "username", "someone", "somebody", "johndoe", "janedoe", - "firstname", "lastname", "firstnamelastname", "firstlast", - "test", "example", "sample", "placeholder", + "youremail", + "yourname", + "youraddress", + "myemail", + "email", + "name", + "user", + "username", + "someone", + "somebody", + "johndoe", + "janedoe", + "firstname", + "lastname", + "firstnamelastname", + "firstlast", + "test", + "example", + "sample", + "placeholder", } ) # Placeholder profile handles left in site templates ("github.com/username"). _PLACEHOLDER_SOCIAL_SEGMENTS = frozenset( { - "username", "yourusername", "yourhandle", "handle", "user", - "profile", "yourprofile", "yourname", "yourpage", "pagename", - "youraccount", "account", "example", "placeholder", "yourcompany", + "username", + "yourusername", + "yourhandle", + "handle", + "user", + "profile", + "yourprofile", + "yourname", + "yourpage", + "pagename", + "youraccount", + "account", + "example", + "placeholder", + "yourcompany", } ) diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 002c527c0..2faa2be17 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "surf-new-backend" -version = "0.0.30" +version = "0.0.31" description = "SurfSense Backend" requires-python = ">=3.12" dependencies = [ diff --git a/surfsense_backend/scripts/check_migration_flow.py b/surfsense_backend/scripts/check_migration_flow.py index 155e5ac84..18587f836 100644 --- a/surfsense_backend/scripts/check_migration_flow.py +++ b/surfsense_backend/scripts/check_migration_flow.py @@ -94,6 +94,7 @@ async def set_version(version: str | None) -> None: async def assert_at_head() -> None: import asyncpg + from alembic.script import ScriptDirectory head = ScriptDirectory(str(BACKEND_DIR / "alembic")).get_current_head() diff --git a/surfsense_backend/scripts/check_zero_publication_bootstrap.py b/surfsense_backend/scripts/check_zero_publication_bootstrap.py index e8956992a..3738901c1 100644 --- a/surfsense_backend/scripts/check_zero_publication_bootstrap.py +++ b/surfsense_backend/scripts/check_zero_publication_bootstrap.py @@ -40,7 +40,9 @@ async def main() -> None: await admin.execute(f'DROP DATABASE IF EXISTS "{SCRATCH_DB}" WITH (FORCE)') await admin.close() - print("OK: ensure_publication creates and verifies on a create_all DB, idempotently.") + print( + "OK: ensure_publication creates and verifies on a create_all DB, idempotently." + ) asyncio.run(main()) diff --git a/surfsense_backend/scripts/e2e_google_search.py b/surfsense_backend/scripts/e2e_google_search.py index acf062819..9c3c02fcf 100644 --- a/surfsense_backend/scripts/e2e_google_search.py +++ b/surfsense_backend/scripts/e2e_google_search.py @@ -28,7 +28,9 @@ sys.path.insert(0, str(_ROOT)) load_dotenv(_ROOT / ".env") logging.basicConfig(level=logging.WARNING) -logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(logging.INFO) +logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel( + logging.INFO +) from app.proprietary.platforms.google_search import ( # noqa: E402 GoogleSearchScrapeInput, @@ -41,15 +43,19 @@ async def run_ai_mode(label: str, *, queries: str) -> None: print(f"\n=== {label} ===") t0 = time.perf_counter() inp = GoogleSearchScrapeInput( - queries=queries, countryCode="us", languageCode="en", + queries=queries, + countryCode="us", + languageCode="en", aiModeSearch={"enableAiMode": True}, ) items = await scrape_serps(inp, limit=2) ai_items = [i for i in items if i["aiModeResult"]] assert ai_items, f"{label}: no aiModeResult item emitted" res = ai_items[0]["aiModeResult"] - print(f" text={len(res['text'])} chars, sources={len(res['sources'])} " - f"({time.perf_counter()-t0:.0f}s)") + print( + f" text={len(res['text'])} chars, sources={len(res['sources'])} " + f"({time.perf_counter() - t0:.0f}s)" + ) print(f" {res['text'][:130]!r}") for s in res["sources"][:3]: print(f" src: {(s['title'] or '')[:60]!r}") @@ -59,9 +65,16 @@ async def run_ai_mode(label: str, *, queries: str) -> None: async def run( - label: str, *, expect_ads=False, expect_products=False, expect_paa_answers=False, - expect_sitelinks=False, expect_aio=False, expect_device=None, - expect_icons=False, **kwargs + label: str, + *, + expect_ads=False, + expect_products=False, + expect_paa_answers=False, + expect_sitelinks=False, + expect_aio=False, + expect_device=None, + expect_icons=False, + **kwargs, ) -> None: print(f"\n=== {label} ===") t0 = time.perf_counter() @@ -72,18 +85,24 @@ async def run( paa_answered = [p for p in it["peopleAlsoAsk"] if p["answer"]] sitelinked = [o for o in it["organicResults"] if o["siteLinks"]] print(f" term={it['searchQuery']['term']!r} resultsTotal={it['resultsTotal']}") - print(f" organic={len(it['organicResults'])} paidResults={len(it['paidResults'])} " - f"paidProducts={len(it['paidProducts'])} related={len(it['relatedQueries'])} " - f"suggested={len(it['suggestedResults'])} " - f"paa={len(it['peopleAlsoAsk'])} (answered={len(paa_answered)}) " - f"({time.perf_counter()-t0:.0f}s)") + print( + f" organic={len(it['organicResults'])} paidResults={len(it['paidResults'])} " + f"paidProducts={len(it['paidProducts'])} related={len(it['relatedQueries'])} " + f"suggested={len(it['suggestedResults'])} " + f"paa={len(it['peopleAlsoAsk'])} (answered={len(paa_answered)}) " + f"({time.perf_counter() - t0:.0f}s)" + ) for o in sitelinked[:2]: - print(f" [sitelinks on #{o['position']}] " - + ", ".join(s["title"] for s in o["siteLinks"][:5])) + print( + f" [sitelinks on #{o['position']}] " + + ", ".join(s["title"] for s in o["siteLinks"][:5]) + ) aio = it["aiOverview"] if aio: - print(f" [aiOverview] content={len(aio['content'])} chars, " - f"sources={len(aio['sources'])}") + print( + f" [aiOverview] content={len(aio['content'])} chars, " + f"sources={len(aio['sources'])}" + ) print(f" {aio['content'][:110]!r}") for s in aio["sources"][:3]: print(f" src: {(s['title'] or '')[:55]!r}") @@ -113,10 +132,15 @@ async def run( f"{label}: device={it['searchQuery']['device']}" ) if expect_icons: - iconed = [o for o in it["organicResults"] - if (o["icon"] or "").startswith("data:image")] - print(f" [icons] {len(iconed)}/{len(it['organicResults'])} organic " - f"carry a base64 favicon") + iconed = [ + o + for o in it["organicResults"] + if (o["icon"] or "").startswith("data:image") + ] + print( + f" [icons] {len(iconed)}/{len(it['organicResults'])} organic " + f"carry a base64 favicon" + ) assert iconed, f"{label}: expected base64 icons on organic results" @@ -124,24 +148,47 @@ _CASES = { "plain": lambda: run("plain query", queries="python asyncio tutorial"), "site": lambda: run("site: filter", queries="machine learning", site="arxiv.org"), "ads": lambda: run("text ads", queries="car insurance quotes", expect_ads=True), - "products": lambda: run("product ads", queries="buy running shoes", expect_products=True), - "focus": lambda: run("focusOnPaidAds (commercial)", queries="car insurance quotes", - focusOnPaidAds=True, expect_ads=True), - "focus-neg": lambda: run("focusOnPaidAds (non-commercial, retries capped)", - queries="python asyncio tutorial", focusOnPaidAds=True), - "paa": lambda: run("people also ask", queries="what is seo", expect_paa_answers=True), - "sitelinks": lambda: run("sitelinks + suggested (brand query)", queries="amazon", - expect_sitelinks=True), + "products": lambda: run( + "product ads", queries="buy running shoes", expect_products=True + ), + "focus": lambda: run( + "focusOnPaidAds (commercial)", + queries="car insurance quotes", + focusOnPaidAds=True, + expect_ads=True, + ), + "focus-neg": lambda: run( + "focusOnPaidAds (non-commercial, retries capped)", + queries="python asyncio tutorial", + focusOnPaidAds=True, + ), + "paa": lambda: run( + "people also ask", queries="what is seo", expect_paa_answers=True + ), + "sitelinks": lambda: run( + "sitelinks + suggested (brand query)", queries="amazon", expect_sitelinks=True + ), "aio": lambda: run("AI Overview", queries="benefits of green tea", expect_aio=True), - "mobile": lambda: run("mobile layout (mobileResults)", queries="best seo tools", - mobileResults=True, expect_device="MOBILE"), - "unfiltered": lambda: run("includeUnfilteredResults (filter=0)", - queries="python asyncio tutorial", - includeUnfilteredResults=True), - "icons": lambda: run("includeIcons (base64 favicons)", queries="github", - includeIcons=True, expect_icons=True), - "aimode": lambda: run_ai_mode("Google AI Mode (udm=50)", - queries="what is quantum computing"), + "mobile": lambda: run( + "mobile layout (mobileResults)", + queries="best seo tools", + mobileResults=True, + expect_device="MOBILE", + ), + "unfiltered": lambda: run( + "includeUnfilteredResults (filter=0)", + queries="python asyncio tutorial", + includeUnfilteredResults=True, + ), + "icons": lambda: run( + "includeIcons (base64 favicons)", + queries="github", + includeIcons=True, + expect_icons=True, + ), + "aimode": lambda: run_ai_mode( + "Google AI Mode (udm=50)", queries="what is quantum computing" + ), } diff --git a/surfsense_backend/scripts/e2e_reddit_scraper.py b/surfsense_backend/scripts/e2e_reddit_scraper.py index 941f009aa..7bedc0812 100644 --- a/surfsense_backend/scripts/e2e_reddit_scraper.py +++ b/surfsense_backend/scripts/e2e_reddit_scraper.py @@ -182,9 +182,7 @@ async def step5_dump_fixtures() -> bool: wrote.append("sample_post.json") # A single comment thing, for the comment-mapping fixture. comment_kids = children(post[1]) if len(post) > 1 else [] - first_comment = next( - (c for c in comment_kids if c.get("kind") == "t1"), None - ) + first_comment = next((c for c in comment_kids if c.get("kind") == "t1"), None) if first_comment: (_FIXTURE_DIR / "sample_comment.json").write_text( json.dumps(first_comment), encoding="utf-8" diff --git a/surfsense_backend/tests/e2e/fakes/chat_llm.py b/surfsense_backend/tests/e2e/fakes/chat_llm.py index 4e6be0995..6c8d2429e 100644 --- a/surfsense_backend/tests/e2e/fakes/chat_llm.py +++ b/surfsense_backend/tests/e2e/fakes/chat_llm.py @@ -562,8 +562,8 @@ class FakeChatLLM(BaseChatModel): # Marker unique to a connector subagent's prompt: the main agent must # delegate via ``task``; only the subagent has connector tools registered. - in_connector_subagent = ( - "connected-apps specialist" in _messages_to_text(messages) + in_connector_subagent = "connected-apps specialist" in _messages_to_text( + messages ) # Main agent: delegate live-tool connector work to its subagent (which diff --git a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_web_search_delegation.py b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_web_search_delegation.py index 04b09af7d..70d624089 100644 --- a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_web_search_delegation.py +++ b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_web_search_delegation.py @@ -29,9 +29,7 @@ def _last_ai_text(messages: list) -> str | None: @pytest.mark.asyncio -async def test_web_query_delegates_to_google_search( - db_session, db_user, db_workspace -): +async def test_web_query_delegates_to_google_search(db_session, db_user, db_workspace): """A web-search query routes through ``task(google_search)`` and resumes. Scripted sequence (the fake model is shared and consumed in order across the @@ -76,9 +74,7 @@ async def test_web_query_delegates_to_google_search( ) task_tool_messages = [ - m - for m in result["messages"] - if isinstance(m, ToolMessage) and m.name == "task" + m for m in result["messages"] if isinstance(m, ToolMessage) and m.name == "task" ] assert task_tool_messages, "web query did not delegate through the task tool" assert _last_ai_text(result["messages"]) == ( diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_discovery_migration.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_discovery_migration.py index 5205e363a..428b2f24f 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_discovery_migration.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_discovery_migration.py @@ -62,13 +62,15 @@ def _tool(name: str, metadata: dict) -> StructuredTool: def test_all_mcp_connectors_route_to_discovery(): for connector_type in _MCP_ROUTED: assert ( - CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS[connector_type] - == MCP_DISCOVERY_NAME + CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS[connector_type] == MCP_DISCOVERY_NAME ), connector_type def test_file_connectors_keep_native_routes(): - assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["GOOGLE_DRIVE_CONNECTOR"] == "google_drive" + assert ( + CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["GOOGLE_DRIVE_CONNECTOR"] + == "google_drive" + ) assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["DROPBOX_CONNECTOR"] == "dropbox" assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["ONEDRIVE_CONNECTOR"] == "onedrive" @@ -87,7 +89,9 @@ def test_discovery_gating_is_any_of(): def test_discovery_gating_tokens_match_routed_connectors(): - assert SUBAGENT_TO_REQUIRED_CONNECTOR_MAP[MCP_DISCOVERY_NAME] == frozenset(_MCP_ROUTED) + assert SUBAGENT_TO_REQUIRED_CONNECTOR_MAP[MCP_DISCOVERY_NAME] == frozenset( + _MCP_ROUTED + ) def test_legacy_aliases_resolve_to_a_live_subagent(): @@ -104,7 +108,12 @@ def test_collision_only_prefixes_shared_names(): _tool("search", {"mcp_connector_id": 2, "mcp_transport": "http"}), _tool("list_bases", {"mcp_connector_id": 2, "mcp_transport": "http"}), ] - resolved = {t.name: t for t in resolve_tool_name_collisions(tools, {1: "NOTION_CONNECTOR", 2: "AIRTABLE_CONNECTOR"})} + resolved = { + t.name: t + for t in resolve_tool_name_collisions( + tools, {1: "NOTION_CONNECTOR", 2: "AIRTABLE_CONNECTOR"} + ) + } # The unique tool keeps its bare name (trusted_tools / history stay valid). assert "list_bases" in resolved diff --git a/surfsense_backend/tests/unit/capabilities/access/test_rest_router.py b/surfsense_backend/tests/unit/capabilities/access/test_rest_router.py index b0e64032f..a406f29bd 100644 --- a/surfsense_backend/tests/unit/capabilities/access/test_rest_router.py +++ b/surfsense_backend/tests/unit/capabilities/access/test_rest_router.py @@ -423,8 +423,12 @@ async def test_run_events_replays_buffer_then_finishes(monkeypatch): row = _fake_run_row(status="running") raw = str(row.id) - run_event_bus.publish(raw, {"type": "run.progress", "phase": "scraping", "current": 1}) - run_event_bus.publish(raw, {"type": "run.finished", "status": "success", "item_count": 2}) + run_event_bus.publish( + raw, {"type": "run.progress", "phase": "scraping", "current": 1} + ) + run_event_bus.publish( + raw, {"type": "run.finished", "status": "success", "item_count": 2} + ) app = _build_app_with_rows(monkeypatch, [row]) try: diff --git a/surfsense_backend/tests/unit/capabilities/reddit/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/reddit/scrape/test_executor.py index ca302f82f..257b974a0 100644 --- a/surfsense_backend/tests/unit/capabilities/reddit/scrape/test_executor.py +++ b/surfsense_backend/tests/unit/capabilities/reddit/scrape/test_executor.py @@ -43,7 +43,9 @@ async def test_maps_urls_to_start_urls_and_wraps_items(): assert out.items[0].dataType == "post" (actor_input, _limit) = scraper.calls[0] - assert [u.url for u in actor_input.startUrls] == ["https://www.reddit.com/r/python/"] + assert [u.url for u in actor_input.startUrls] == [ + "https://www.reddit.com/r/python/" + ] assert actor_input.searches == [] diff --git a/surfsense_backend/tests/unit/capabilities/test_billing.py b/surfsense_backend/tests/unit/capabilities/test_billing.py index 28021af47..36d9a2978 100644 --- a/surfsense_backend/tests/unit/capabilities/test_billing.py +++ b/surfsense_backend/tests/unit/capabilities/test_billing.py @@ -233,7 +233,9 @@ async def test_gate_reserves_worst_case_captcha_when_solving_enabled(monkeypatch with pytest.raises(InsufficientCreditsError): await gate_capability( - CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session) + CrawlInput(startUrls=["https://a.com"]), + BillingUnit.WEB_CRAWL, + _ctx(session), ) diff --git a/surfsense_backend/tests/unit/capabilities/test_run_truncation.py b/surfsense_backend/tests/unit/capabilities/test_run_truncation.py index 43f1519be..5723c2ff2 100644 --- a/surfsense_backend/tests/unit/capabilities/test_run_truncation.py +++ b/surfsense_backend/tests/unit/capabilities/test_run_truncation.py @@ -64,7 +64,9 @@ def test_preview_is_char_budgeted_and_references_run(): per_item = "y" * 500 items = [f'{{"i": {i}, "v": "{per_item}"}}' for i in range(500)] body = "\n".join(items) - serialized = SerializedOutput(text=body, item_count=len(items), char_count=len(body)) + serialized = SerializedOutput( + text=body, item_count=len(items), char_count=len(body) + ) preview = _build_preview(serialized, run_id="abc") @@ -143,8 +145,11 @@ async def test_read_run_paginates(monkeypatch): read_run, _ = _tools() out = await read_run.ainvoke( - {"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", - "offset": 2, "limit": 3} + { + "ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", + "offset": 2, + "limit": 3, + } ) assert "item_2" in out and "item_3" in out and "item_4" in out assert "item_0" not in out and "item_5" not in out @@ -184,8 +189,7 @@ async def test_search_run_excerpts_huge_matched_line(monkeypatch): _, search_run = _tools() out = await search_run.ainvoke( - {"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", - "pattern": "NEEDLE"} + {"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", "pattern": "NEEDLE"} ) assert "NEEDLE" in out assert "match at char 100000" in out @@ -215,8 +219,10 @@ async def test_search_run_matches(monkeypatch): _patch_session(monkeypatch, _BODY, []) _, search_run = _tools() out = await search_run.ainvoke( - {"ref": "spill_" + "0" * 8 + "-0000-0000-0000-000000000000", - "pattern": "item_7"} + { + "ref": "spill_" + "0" * 8 + "-0000-0000-0000-000000000000", + "pattern": "item_7", + } ) assert "item_7" in out assert "item_1" not in out.split("item_7")[0] @@ -232,14 +238,30 @@ _CRAWL_BODY = "\n".join( "url": "https://x.com/team/", "status": "success", "links": [ - {"url": "https://x.com/author/jane/", "text": "Jane Doe", - "context": "Jane Doe General Partner", "kind": "internal"}, - {"url": "https://x.com/author/bob/", "text": "Bob Roe", - "context": "Bob Roe Operations", "kind": "internal"}, + { + "url": "https://x.com/author/jane/", + "text": "Jane Doe", + "context": "Jane Doe General Partner", + "kind": "internal", + }, + { + "url": "https://x.com/author/bob/", + "text": "Bob Roe", + "context": "Bob Roe Operations", + "kind": "internal", + }, # Duplicate of Jane (nav + card) — must dedupe. - {"url": "https://x.com/author/jane/", "text": "Jane Doe", - "context": "Jane Doe General Partner", "kind": "internal"}, - {"url": "https://x.com/about/", "text": "About", "kind": "internal"}, + { + "url": "https://x.com/author/jane/", + "text": "Jane Doe", + "context": "Jane Doe General Partner", + "kind": "internal", + }, + { + "url": "https://x.com/about/", + "text": "About", + "kind": "internal", + }, ], } ), diff --git a/surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py b/surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py index d26846f93..6294cf4f1 100644 --- a/surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py +++ b/surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py @@ -101,7 +101,11 @@ async def test_aggregated_contacts_carry_provenance_and_site_wide_flag() -> None class _ContactsEngine: async def crawl_url(self, url: str) -> CrawlOutcome: socials = [footer] + ([person] if url.endswith("/about") else []) - links = ["https://e.com/about", "https://e.com/blog"] if url == "https://e.com/" else [] + links = ( + ["https://e.com/about", "https://e.com/blog"] + if url == "https://e.com/" + else [] + ) return CrawlOutcome( status=_SUCCESS, result={ diff --git a/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py b/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py index 9e7503851..f7b9a2b39 100644 --- a/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py +++ b/surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py @@ -43,18 +43,20 @@ def test_estimated_units_for_single_url_is_seed_count() -> None: def test_estimated_units_for_spider_is_max_pages() -> None: - model = CrawlInput( - startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25 - ) + model = CrawlInput(startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25) assert model.estimated_units == 25 def test_billable_units_counts_only_successes() -> None: out = CrawlOutput( items=[ - CrawlItem(url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)), + CrawlItem( + url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0) + ), CrawlItem(url="b", status="empty", crawl=CrawlMeta(loadedUrl="b", depth=1)), - CrawlItem(url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)), + CrawlItem( + url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1) + ), ] ) assert out.billable_units == 1 diff --git a/surfsense_backend/tests/unit/platforms/google_search/test_skeleton.py b/surfsense_backend/tests/unit/platforms/google_search/test_skeleton.py index 2025089c5..41a960a0e 100644 --- a/surfsense_backend/tests/unit/platforms/google_search/test_skeleton.py +++ b/surfsense_backend/tests/unit/platforms/google_search/test_skeleton.py @@ -274,8 +274,13 @@ def test_parse_serp_extracts_all_blocks(): assert prod.prices == ["$24.99", "$30"] # Related searches exclude the numeric pagination anchor (a.fl). - assert [r.title for r in item.relatedQueries] == ["easy apple pie", "apple pie recipe"] - assert item.relatedQueries[0].url == "https://www.google.com/search?q=easy+apple+pie" + assert [r.title for r in item.relatedQueries] == [ + "easy apple pie", + "apple pie recipe", + ] + assert ( + item.relatedQueries[0].url == "https://www.google.com/search?q=easy+apple+pie" + ) # suggestedResults are the related queries re-shaped with type/position. assert [(s.position, s.title, s.type) for s in item.suggestedResults] == [ diff --git a/surfsense_backend/tests/unit/platforms/reddit/test_parsers.py b/surfsense_backend/tests/unit/platforms/reddit/test_parsers.py index b61479305..77f1312a4 100644 --- a/surfsense_backend/tests/unit/platforms/reddit/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/reddit/test_parsers.py @@ -95,8 +95,15 @@ def test_flatten_comments_counts_replies_and_stops_at_more(): "kind": "Listing", "data": { "children": [ - {"kind": "t1", "data": {"name": "t1_2", "id": "2", - "body": "reply", "replies": ""}}, + { + "kind": "t1", + "data": { + "name": "t1_2", + "id": "2", + "body": "reply", + "replies": "", + }, + }, {"kind": "more", "data": {}}, # stub -> ignored ] }, @@ -112,8 +119,10 @@ def test_flatten_comments_counts_replies_and_stops_at_more(): def test_flatten_comments_honors_max(): tree = [ - {"kind": "t1", "data": {"name": f"t1_{i}", "id": str(i), "body": "x", - "replies": ""}} + { + "kind": "t1", + "data": {"name": f"t1_{i}", "id": str(i), "body": "x", "replies": ""}, + } for i in range(5) ] assert len(flatten_comments(tree, max_comments=2)) == 2 @@ -128,9 +137,17 @@ def test_children_and_after(): def test_parse_community_maps_members(): - thing = {"kind": "t5", "data": {"name": "t5_s", "id": "s", - "display_name": "py", "display_name_prefixed": "r/py", - "subscribers": 1234, "url": "/r/py/"}} + thing = { + "kind": "t5", + "data": { + "name": "t5_s", + "id": "s", + "display_name": "py", + "display_name_prefixed": "r/py", + "subscribers": 1234, + "url": "/r/py/", + }, + } item = parse_community(thing) assert item["dataType"] == "community" assert item["numberOfMembers"] == 1234 diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/test_site_crawler.py b/surfsense_backend/tests/unit/proprietary/web_crawler/test_site_crawler.py index cbb12d43b..1f8f64d54 100644 --- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_site_crawler.py +++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_site_crawler.py @@ -125,9 +125,7 @@ async def test_dedupes_on_canonical_url() -> None: } ) - await crawl_site( - engine, ["https://e.com/"], max_crawl_depth=3, max_crawl_pages=10 - ) + await crawl_site(engine, ["https://e.com/"], max_crawl_depth=3, max_crawl_pages=10) assert engine.calls.count("https://e.com/a") == 1 assert engine.calls.count("https://e.com/") == 1 diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py b/surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py index 9ff7fec26..2c209ce16 100644 --- a/surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py +++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py @@ -116,7 +116,12 @@ def test_extract_link_records_dedupes_keeping_first_nonempty_text() -> None: html = 'Pricing' records = extract_link_records(html, "https://example.com/") assert records == [ - {"url": "https://example.com/p", "text": "Pricing", "rel": "", "kind": "internal"} + { + "url": "https://example.com/p", + "text": "Pricing", + "rel": "", + "kind": "internal", + } ] diff --git a/surfsense_backend/tests/unit/services/test_web_crawl_credit_service.py b/surfsense_backend/tests/unit/services/test_web_crawl_credit_service.py index 7c7cd0e40..ef806cfeb 100644 --- a/surfsense_backend/tests/unit/services/test_web_crawl_credit_service.py +++ b/surfsense_backend/tests/unit/services/test_web_crawl_credit_service.py @@ -241,5 +241,3 @@ class TestCheckBalance: session, _user = _make_session(balance_micros=0) await WebCrawlCreditService(session).check_balance(_USER_ID, 0) session.execute.assert_not_called() - - diff --git a/surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py b/surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py index e2d96e0e1..af1288046 100644 --- a/surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py +++ b/surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py @@ -47,9 +47,7 @@ def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None: def test_no_country_suffix_yields_empty_location( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr( - Config, "PROXY_URL", "http://tok:secret@gw.dataimpulse.com:823" - ) + monkeypatch.setattr(Config, "PROXY_URL", "http://tok:secret@gw.dataimpulse.com:823") assert DataImpulseProvider().get_location() == "" diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index c46b8d4fc..a5b492fc8 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -33,6 +33,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -64,6 +67,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -95,6 +101,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -126,6 +135,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -157,6 +169,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -188,6 +203,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -219,6 +237,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -250,6 +271,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -281,6 +305,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -312,6 +339,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -353,6 +383,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -384,6 +418,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -415,6 +452,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -446,6 +486,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -477,6 +520,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -508,6 +554,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -539,6 +588,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -570,6 +622,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -601,6 +656,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -632,6 +690,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -663,6 +724,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -704,6 +768,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -735,6 +803,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -766,6 +837,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -797,6 +871,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -828,6 +905,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -859,6 +939,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -890,6 +973,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -921,6 +1007,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -952,6 +1041,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -983,6 +1075,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1014,6 +1109,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1055,6 +1153,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1086,6 +1188,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1147,6 +1252,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1178,6 +1289,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1209,6 +1323,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1270,6 +1387,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1331,6 +1454,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1362,6 +1491,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", ] conflicts = [[ @@ -10221,7 +10353,7 @@ wheels = [ [[package]] name = "surf-new-backend" -version = "0.0.30" +version = "0.0.31" source = { editable = "." } dependencies = [ { name = "alembic" }, diff --git a/surfsense_browser_extension/package.json b/surfsense_browser_extension/package.json index 728909d06..8dda0d82f 100644 --- a/surfsense_browser_extension/package.json +++ b/surfsense_browser_extension/package.json @@ -1,7 +1,7 @@ { "name": "surfsense_browser_extension", "displayName": "Surfsense Browser Extension", - "version": "0.0.30", + "version": "0.0.31", "description": "Extension to collect Browsing History for SurfSense.", "author": "https://github.com/MODSetter", "engines": { diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index f5915a875..f12b9722c 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -1,7 +1,7 @@ { "name": "surfsense-desktop", "productName": "SurfSense", - "version": "0.0.30", + "version": "0.0.31", "description": "SurfSense Desktop App", "main": "dist/main.js", "scripts": { diff --git a/surfsense_web/package.json b/surfsense_web/package.json index 0d4113937..efa04b4f6 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -1,6 +1,6 @@ { "name": "surfsense_web", - "version": "0.0.30", + "version": "0.0.31", "private": true, "packageManager": "pnpm@10.26.0", "description": "SurfSense Frontend",