apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-08 22:22:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: bumped version to 0.0.31

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-06 21:43:15 -07:00
parent 8df8565e0a
commit 1c9ab207ef
56 changed files with 520 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

2
VERSION
View file

@ -1 +1 @@
0.0.30
0.0.31

6
surfsense_backend/alembic/env.py
View file

@ -5,12 +5,12 @@ import sys
from logging.config import fileConfig
import sqlalchemy as sa
from alembic.script import ScriptDirectory
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
from alembic.script import ScriptDirectory
# Ensure the app directory is in the Python path
# This allows Alembic to find your models
@ -81,9 +81,7 @@ def _fast_forward_fresh_db(connection: Connection) -> bool:
step rather than resurrecting the replay.
"""
for table in ("documents", "searchspaces", BOOTSTRAP_MARKER_TABLE):
if connection.execute(
sa.text("SELECT to_regclass(:t)"), {"t": table}
).scalar():
if connection.execute(sa.text("SELECT to_regclass(:t)"), {"t": table}).scalar():
return False
if connection.execute(sa.text("SELECT to_regclass('alembic_version')")).scalar():
current = connection.execute(

4
surfsense_backend/alembic/versions/171_add_runs_and_tool_output_spills.py
View file

@ -41,9 +41,7 @@ def upgrade() -> None:
);
"""
)
op.execute(
"CREATE INDEX IF NOT EXISTS ix_runs_workspace_id ON runs (workspace_id)"
)
op.execute("CREATE INDEX IF NOT EXISTS ix_runs_workspace_id ON runs (workspace_id)")
op.execute("CREATE INDEX IF NOT EXISTS ix_runs_user_id ON runs (user_id)")
op.execute("CREATE INDEX IF NOT EXISTS ix_runs_capability ON runs (capability)")
op.execute("CREATE INDEX IF NOT EXISTS ix_runs_created_at ON runs (created_at)")

4
surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/agent.py
View file

@ -50,9 +50,7 @@ def _augment_allowlist_for_collision_prefixes(
continue
original = meta.get("mcp_original_tool_name")
if original in trusted_names and tool.name not in trusted_names:
alias_rules.append(
Rule(permission=tool.name, pattern="*", action="allow")
)
alias_rules.append(Rule(permission=tool.name, pattern="*", action="allow"))
if not alias_rules:
return dependencies

3
surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/mcp_discovery/tools/get_connected_accounts.py
View file

@ -85,8 +85,7 @@ def create_get_connected_accounts_tool(*, workspace_id: int) -> BaseTool:
# ``server_config`` presence == the connector produces agent
# tools. Native rows without it are connected for indexing
# only and need a reconnect via MCP.
"usable_in_chat": isinstance(cfg, dict)
and "server_config" in cfg,
"usable_in_chat": isinstance(cfg, dict) and "server_config" in cfg,
"account": account_meta,
}
)

4
surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py
View file

@ -83,7 +83,9 @@ def resolve_tool_name_collisions(
or original_name,
"mcp_collision_prefixed": True,
}
resolved.append(tool.model_copy(update={"name": new_name, "metadata": new_meta}))
resolved.append(
tool.model_copy(update={"name": new_name, "metadata": new_meta})
)
return resolved

18
surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py
View file

@ -128,7 +128,7 @@ def _cell(value: Any) -> str:
"""Render one CSV cell: scalars as-is, nested structures as compact JSON."""
if value is None:
return ""
if isinstance(value, (dict, list)):
if isinstance(value, dict | list):
return json.dumps(value, ensure_ascii=False, default=str)
return str(value)
@ -230,9 +230,7 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]:
count = min(max(1, limit), _MAX_LIMIT)
window = lines[start : start + count]
if not window:
return (
f"No lines at offset {start} (total {len(lines)} lines in {ref})."
)
return f"No lines at offset {start} (total {len(lines)} lines in {ref})."
window_body = "\n".join(window)
start_char = max(0, char_offset)
if start_char >= len(window_body) > 0:
@ -328,10 +326,14 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]:
records = _rows_from_body(body, rows)
if include_pattern:
inc = _build_matcher(include_pattern.strip())
records = [r for r in records if inc(" ".join(map(_cell, r.values()))) is not None]
records = [
r for r in records if inc(" ".join(map(_cell, r.values()))) is not None
]
if exclude_pattern:
exc = _build_matcher(exclude_pattern.strip())
records = [r for r in records if exc(" ".join(map(_cell, r.values()))) is None]
records = [
r for r in records if exc(" ".join(map(_cell, r.values()))) is None
]
if not records:
return (
f"Error: no rows to export from {ref} "
@ -353,7 +355,9 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]:
preview_lines = csv_text.split("\n")[:4]
truncated_note = (
f" (capped at {_EXPORT_MAX_ROWS} rows)" if row_count >= _EXPORT_MAX_ROWS else ""
f" (capped at {_EXPORT_MAX_ROWS} rows)"
if row_count >= _EXPORT_MAX_ROWS
else ""
)
return (
f"Exported {row_count} rows{truncated_note} to {final_path} "

8
surfsense_backend/app/app.py
View file

@ -627,9 +627,13 @@ async def _sweep_stale_scraper_runs() -> None:
async with async_session_maker() as session:
swept = await fail_stale_running_runs(session)
if swept:
logger.info("[startup] Marked %d stale running scraper run(s) as error", swept)
logger.info(
"[startup] Marked %d stale running scraper run(s) as error", swept
)
except Exception:
logger.warning("[startup] Stale scraper-run sweep failed (non-fatal)", exc_info=True)
logger.warning(
"[startup] Stale scraper-run sweep failed (non-fatal)", exc_info=True
)
@asynccontextmanager

5
surfsense_backend/app/capabilities/core/access/rest.py
View file

@ -209,7 +209,10 @@ async def _execute_async_run(
raise
except (SurfSenseError, HTTPException) as exc:
await _finalize_async(
run_id, status="error", error=str(exc), started=started,
run_id,
status="error",
error=str(exc),
started=started,
progress=reporter.coarse,
)
_publish_finished(run_id, "error", error=str(exc))

4
surfsense_backend/app/capabilities/core/runs.py
View file

@ -291,7 +291,9 @@ async def record_spill(
return None
async def _maybe_cleanup(session: AsyncSession, table: str, retention_days: int) -> None:
async def _maybe_cleanup(
session: AsyncSession, table: str, retention_days: int
) -> None:
"""Delete a bounded batch of expired rows on ~1% of inserts."""
if random.random() >= _CLEANUP_SAMPLE_RATE:
return

11
surfsense_backend/app/capabilities/google_maps/reviews/executor.py
View file

@ -30,14 +30,21 @@ def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor:
reviewsStartDate=payload.start_date,
language=payload.language,
)
emit_progress("starting", "Fetching Google Maps reviews", total=payload.max_reviews, unit="review")
emit_progress(
"starting",
"Fetching Google Maps reviews",
total=payload.max_reviews,
unit="review",
)
try:
items = await scrape_fn(actor_input)
except SignInRequiredError as exc:
raise ForbiddenError(
f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED"
) from exc
emit_progress("done", f"Scraped {len(items)} review(s)", current=len(items), unit="review")
emit_progress(
"done", f"Scraped {len(items)} review(s)", current=len(items), unit="review"
)
return ReviewsOutput(items=items)
return execute

8
surfsense_backend/app/capabilities/google_maps/scrape/executor.py
View file

@ -33,14 +33,18 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
maxReviews=payload.max_reviews,
maxImages=payload.max_images,
)
emit_progress("starting", "Searching Google Maps", total=payload.max_places, unit="place")
emit_progress(
"starting", "Searching Google Maps", total=payload.max_places, unit="place"
)
try:
items = await scrape_fn(actor_input)
except SignInRequiredError as exc:
raise ForbiddenError(
f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED"
) from exc
emit_progress("done", f"Scraped {len(items)} place(s)", current=len(items), unit="place")
emit_progress(
"done", f"Scraped {len(items)} place(s)", current=len(items), unit="place"
)
return ScrapeOutput(items=items)
return execute

7
surfsense_backend/app/capabilities/google_search/scrape/executor.py
View file

@ -43,7 +43,12 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
unit="page",
)
items = await scrape_fn(actor_input, limit=_MAX_SERP_ITEMS)
emit_progress("done", f"Scraped {len(items)} SERP page(s)", current=len(items), unit="page")
emit_progress(
"done",
f"Scraped {len(items)} SERP page(s)",
current=len(items),
unit="page",
)
return ScrapeOutput(items=items)
return execute

8
surfsense_backend/app/capabilities/reddit/scrape/executor.py
View file

@ -36,7 +36,9 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
postDateLimit=payload.post_date_limit,
commentDateLimit=payload.comment_date_limit,
)
emit_progress("starting", "Resolving Reddit targets", total=payload.max_items, unit="item")
emit_progress(
"starting", "Resolving Reddit targets", total=payload.max_items, unit="item"
)
try:
items = await scrape_fn(actor_input, limit=payload.max_items)
except RedditAccessBlockedError as exc:
@ -46,7 +48,9 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
f"Reddit refused anonymous access: {exc}",
code="REDDIT_ACCESS_BLOCKED",
) from exc
emit_progress("done", f"Scraped {len(items)} item(s)", current=len(items), unit="item")
emit_progress(
"done", f"Scraped {len(items)} item(s)", current=len(items), unit="item"
)
return ScrapeOutput(items=items)
return execute

4
surfsense_backend/app/capabilities/web/crawl/executor.py
View file

@ -60,7 +60,9 @@ def build_crawl_executor(engine: WebCrawlerConnector | None = None) -> Executor:
unit="page",
)
items = [_to_item(page, payload.maxLength) for page in pages]
emit_progress("done", f"Crawled {len(items)} page(s)", current=len(items), unit="page")
emit_progress(
"done", f"Crawled {len(items)} page(s)", current=len(items), unit="page"
)
return CrawlOutput(
items=items,
contacts=_aggregate_contacts(items),

3
surfsense_backend/app/capabilities/web/crawl/schemas.py
View file

@ -165,7 +165,8 @@ class CrawlItem(BaseModel):
default=None, description="Crawl provenance (loaded URL, depth, referrer)."
)
markdown: str | None = Field(
default=None, description="Cleaned page content as markdown (null unless success)."
default=None,
description="Cleaned page content as markdown (null unless success).",
)
metadata: dict[str, str] | None = Field(
default=None, description="Page metadata such as title and description."

14
surfsense_backend/app/capabilities/youtube/comments/executor.py
View file

@ -25,9 +25,19 @@ def build_comments_executor(scrape_fn: CommentsFn | None = None) -> Executor:
maxComments=payload.max_comments,
sortCommentsBy=payload.sort_by,
)
emit_progress("starting", "Fetching YouTube comments", total=payload.max_comments, unit="comment")
emit_progress(
"starting",
"Fetching YouTube comments",
total=payload.max_comments,
unit="comment",
)
items = await scrape_fn(actor_input)
emit_progress("done", f"Scraped {len(items)} comment(s)", current=len(items), unit="comment")
emit_progress(
"done",
f"Scraped {len(items)} comment(s)",
current=len(items),
unit="comment",
)
return CommentsOutput(items=items)
return execute

11
surfsense_backend/app/capabilities/youtube/scrape/executor.py
View file

@ -31,9 +31,16 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
downloadSubtitles=payload.download_subtitles,
subtitlesLanguage=payload.subtitles_language,
)
emit_progress("starting", "Resolving YouTube targets", total=payload.max_results, unit="video")
emit_progress(
"starting",
"Resolving YouTube targets",
total=payload.max_results,
unit="video",
)
items = await scrape_fn(actor_input)
emit_progress("done", f"Scraped {len(items)} video(s)", current=len(items), unit="video")
emit_progress(
"done", f"Scraped {len(items)} video(s)", current=len(items), unit="video"
)
return ScrapeOutput(items=items)
return execute

4
surfsense_backend/app/config/__init__.py
View file

@ -713,9 +713,7 @@ class Config:
YOUTUBE_MICROS_PER_VIDEO = int(os.getenv("YOUTUBE_MICROS_PER_VIDEO", "2500"))
# Kept separate from the video rate so comments can be re-tuned toward the
# cheaper per-comment market ($0.40-2.00/1k) without touching video pricing.
YOUTUBE_MICROS_PER_COMMENT = int(
os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500")
)
YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500"))
# Low-balance WARNING threshold (micro-USD). Surfaced by the quota service
# so the UI can nudge the user to top up / enable auto-reload. $0.50.

4
surfsense_backend/app/db.py
View file

@ -2795,9 +2795,7 @@ class Run(Base, TimestampMixin):
__tablename__ = "runs"
__allow_unmapped__ = True
__table_args__ = (
Index("ix_runs_workspace_created", "workspace_id", "created_at"),
)
__table_args__ = (Index("ix_runs_workspace_created", "workspace_id", "created_at"),)
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
workspace_id = Column(

8
surfsense_backend/app/proprietary/platforms/google_search/fetch.py
View file

@ -31,6 +31,7 @@ if we add another sticky vendor is to extend ``_STICKY_HOSTS``.
from __future__ import annotations
import asyncio
import contextlib
import logging
import random
import sys
@ -158,8 +159,7 @@ async def _vet_fresh_ip(url: str, base: str) -> str | None:
return proxy if await _precheck(url, proxy) else None
tasks = [
asyncio.create_task(vet(_sticky_variant(base)))
for _ in range(_VET_CONCURRENCY)
asyncio.create_task(vet(_sticky_variant(base))) for _ in range(_VET_CONCURRENCY)
]
winner: str | None = None
for fut in asyncio.as_completed(tasks):
@ -306,10 +306,8 @@ async def _drop_session_on_loop(mobile: bool) -> None:
async with _session_lock:
session = _sessions.pop(mobile, None)
if session is not None:
try:
with contextlib.suppress(Exception): # already dead; nothing to salvage
await session.close()
except Exception: # already dead; nothing to salvage
pass
async def _drop_session(mobile: bool) -> None:

10
surfsense_backend/app/proprietary/platforms/google_search/parsers.py
View file

@ -218,7 +218,9 @@ def parse_organic(doc: Adaptor, *, include_icons: bool = False) -> list[OrganicR
return results
def parse_paid_results(doc: Adaptor, *, include_icons: bool = False) -> list[PaidResult]:
def parse_paid_results(
doc: Adaptor, *, include_icons: bool = False
) -> list[PaidResult]:
"""Text ads (``div[data-text-ad]``), covering the top and bottom ad blocks.
Fields mirror an organic result: the heading is the title, the ad's anchor
@ -239,7 +241,11 @@ def parse_paid_results(doc: Adaptor, *, include_icons: bool = False) -> list[Pai
description = None
for cand in block.css(".Va3FIb"):
text = _text(cand)
if text and text != title and (description is None or len(text) > len(description)):
if (
text
and text != title
and (description is None or len(text) > len(description))
):
description = text
slot = block.attrib.get("data-ta-slot-pos")
ads.append(

11
surfsense_backend/app/proprietary/platforms/reddit/scraper.py
View file

@ -291,8 +291,8 @@ async def _user_flow(
date_limit=input_model.postDateLimit,
):
# A user listing mixes posts (t3) and comments (t1); a post has a title.
parsed = parse_post(data) if data.get("title") is not None else parse_comment(
data
parsed = (
parse_post(data) if data.get("title") is not None else parse_comment(data)
)
item = _emit(parsed, include_nsfw=input_model.includeNSFW)
if item is not None:
@ -401,8 +401,7 @@ async def iter_reddit(
continue
resolved.append(r)
jobs = [
_dispatch(r, input_model)
for r in _capped_targets(resolved, input_model)
_dispatch(r, input_model) for r in _capped_targets(resolved, input_model)
]
async for item in fan_out(jobs):
yield item
@ -449,9 +448,7 @@ async def scrape_reddit(
results: list[dict[str, Any]] = []
async for item in iter_reddit(input_model):
results.append(item)
emit_progress(
"scraping", current=len(results), total=limit, unit="item"
)
emit_progress("scraping", current=len(results), total=limit, unit="item")
if limit is not None and len(results) >= limit:
break
return results

4
surfsense_backend/app/proprietary/platforms/reddit/url_resolver.py
View file

@ -28,9 +28,7 @@ _REDDIT_HOSTS = frozenset(
)
# Listing sorts that can appear as a trailing subreddit path segment.
_SUBREDDIT_SORTS = frozenset(
{"hot", "new", "top", "rising", "controversial", "best"}
)
_SUBREDDIT_SORTS = frozenset({"hot", "new", "top", "rising", "controversial", "best"})
_USER_CONTENT = frozenset({"overview", "submitted", "comments"})

4
surfsense_backend/app/proprietary/platforms/youtube/subtitles.py
View file

@ -69,9 +69,7 @@ def _fetch_subtitles_sync(
api = _build_client()
try:
transcript_list = api.list(video_id)
transcript = _select_transcript(
transcript_list, language, prefer_generated
)
transcript = _select_transcript(transcript_list, language, prefer_generated)
fetched = transcript.fetch()
break
except RequestBlocked: # covers IpBlocked; rotate to a fresh exit IP

8
surfsense_backend/app/proprietary/web_crawler/connector.py
View file

@ -91,7 +91,9 @@ _CURRENCY_AMOUNT_RE = re.compile(
re.IGNORECASE,
)
_STRIP_XPATH = "//script | //style | //noscript | //template | //svg | //iframe | //head"
_STRIP_XPATH = (
"//script | //style | //noscript | //template | //svg | //iframe | //head"
)
def _visible_text(raw_html: str) -> str:
@ -278,9 +280,7 @@ class WebCrawlerConnector:
else:
reached_without_content = True
errors.append("Scrapling static: empty extraction")
self._log_tier_outcome(
"scrapling-static", url, tier_start, "empty"
)
self._log_tier_outcome("scrapling-static", url, tier_start, "empty")
except Exception as exc:
errors.append(f"Scrapling static: {exc!s}")
self._log_tier_outcome(

4
surfsense_backend/app/proprietary/web_crawler/site_crawler.py
View file

@ -156,9 +156,7 @@ class _SiteSpider(Spider):
# ladder + proxy rotation; never let the spider re-fetch on top of that.
return False
async def parse(
self, response: Response
) -> AsyncGenerator[Request | None, None]:
async def parse(self, response: Response) -> AsyncGenerator[Request | None, None]:
outcome: CrawlOutcome = response._outcome # type: ignore[attr-defined]
depth: int = response.meta.get("_depth", 0)
referrer: str | None = response.meta.get("_referrer")

4
surfsense_backend/app/proprietary/web_crawler/url_policy.py
View file

@ -73,9 +73,7 @@ def _anchor_context(anchor: Any) -> str:
return ""
def extract_link_records(
page_html: str | None, base_url: str
) -> list[dict[str, str]]:
def extract_link_records(page_html: str | None, base_url: str) -> list[dict[str, str]]:
"""Structured ``<a>`` inventory: ``{url, text, context, rel, kind}`` per target.
``kind`` is one of ``internal`` (same site as ``base_url``), ``external``,

4
surfsense_backend/app/services/mcp_oauth/registry.py
View file

@ -218,9 +218,7 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = {
"create-pages",
"update-page",
],
readonly_tools=frozenset(
{"notion-search", "notion-fetch", "search", "fetch"}
),
readonly_tools=frozenset({"notion-search", "notion-fetch", "search", "fetch"}),
account_metadata_keys=["workspace_name"],
),
"confluence": MCPServiceConfig(

7
surfsense_backend/app/services/wallet_credit.py
View file

@ -22,7 +22,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.services.etl_credit_service import InsufficientCreditsError
__all__ = ["InsufficientCreditsError", "apply_debit", "check_balance", "spendable_micros"]
__all__ = [
"InsufficientCreditsError",
"apply_debit",
"check_balance",
"spendable_micros",
]
async def spendable_micros(session: AsyncSession, user_id: str | UUID) -> int:

1
surfsense_backend/app/tasks/celery_tasks/document_tasks.py
View file

@ -1393,4 +1393,3 @@ async def _index_uploaded_folder_files_async(
heartbeat_task.cancel()
if notification_id is not None:
_stop_heartbeat(notification_id)

69
surfsense_backend/app/utils/crawl/contacts.py
View file

@ -87,9 +87,30 @@ _NOISE_EMAIL_DOMAINS = frozenset(
# or version-pinned dep (``react@18.2.0.js``) matches the email shape.
_ASSET_TLDS = frozenset(
{
"png", "jpg", "jpeg", "gif", "svg", "webp", "ico", "bmp",
"css", "js", "mjs", "cjs", "ts", "map", "json", "xml",
"woff", "woff2", "ttf", "eot", "otf", "php", "html", "htm",
"png",
"jpg",
"jpeg",
"gif",
"svg",
"webp",
"ico",
"bmp",
"css",
"js",
"mjs",
"cjs",
"ts",
"map",
"json",
"xml",
"woff",
"woff2",
"ttf",
"eot",
"otf",
"php",
"html",
"htm",
}
)
@ -100,19 +121,47 @@ _EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}")
# common locals like hello/info/contact/support.
_PLACEHOLDER_EMAIL_LOCALS = frozenset(
{
"youremail", "yourname", "youraddress", "myemail", "email", "name",
"user", "username", "someone", "somebody", "johndoe", "janedoe",
"firstname", "lastname", "firstnamelastname", "firstlast",
"test", "example", "sample", "placeholder",
"youremail",
"yourname",
"youraddress",
"myemail",
"email",
"name",
"user",
"username",
"someone",
"somebody",
"johndoe",
"janedoe",
"firstname",
"lastname",
"firstnamelastname",
"firstlast",
"test",
"example",
"sample",
"placeholder",
}
)
# Placeholder profile handles left in site templates ("github.com/username").
_PLACEHOLDER_SOCIAL_SEGMENTS = frozenset(
{
"username", "yourusername", "yourhandle", "handle", "user",
"profile", "yourprofile", "yourname", "yourpage", "pagename",
"youraccount", "account", "example", "placeholder", "yourcompany",
"username",
"yourusername",
"yourhandle",
"handle",
"user",
"profile",
"yourprofile",
"yourname",
"yourpage",
"pagename",
"youraccount",
"account",
"example",
"placeholder",
"yourcompany",
}
)

2
surfsense_backend/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "surf-new-backend"
version = "0.0.30"
version = "0.0.31"
description = "SurfSense Backend"
requires-python = ">=3.12"
dependencies = [

1
surfsense_backend/scripts/check_migration_flow.py
View file

@ -94,6 +94,7 @@ async def set_version(version: str | None) -> None:
async def assert_at_head() -> None:
import asyncpg
from alembic.script import ScriptDirectory
head = ScriptDirectory(str(BACKEND_DIR / "alembic")).get_current_head()

4
surfsense_backend/scripts/check_zero_publication_bootstrap.py
View file

@ -40,7 +40,9 @@ async def main() -> None:
await admin.execute(f'DROP DATABASE IF EXISTS "{SCRATCH_DB}" WITH (FORCE)')
await admin.close()
print("OK: ensure_publication creates and verifies on a create_all DB, idempotently.")
print(
"OK: ensure_publication creates and verifies on a create_all DB, idempotently."
)
asyncio.run(main())

121
surfsense_backend/scripts/e2e_google_search.py
View file

@ -28,7 +28,9 @@ sys.path.insert(0, str(_ROOT))
load_dotenv(_ROOT / ".env")
logging.basicConfig(level=logging.WARNING)
logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(logging.INFO)
logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(
logging.INFO
)
from app.proprietary.platforms.google_search import ( # noqa: E402
GoogleSearchScrapeInput,
@ -41,15 +43,19 @@ async def run_ai_mode(label: str, *, queries: str) -> None:
print(f"\n=== {label} ===")
t0 = time.perf_counter()
inp = GoogleSearchScrapeInput(
queries=queries, countryCode="us", languageCode="en",
queries=queries,
countryCode="us",
languageCode="en",
aiModeSearch={"enableAiMode": True},
)
items = await scrape_serps(inp, limit=2)
ai_items = [i for i in items if i["aiModeResult"]]
assert ai_items, f"{label}: no aiModeResult item emitted"
res = ai_items[0]["aiModeResult"]
print(f" text={len(res['text'])} chars, sources={len(res['sources'])} "
f"({time.perf_counter()-t0:.0f}s)")
print(
f" text={len(res['text'])} chars, sources={len(res['sources'])} "
f"({time.perf_counter() - t0:.0f}s)"
)
print(f" {res['text'][:130]!r}")
for s in res["sources"][:3]:
print(f" src: {(s['title'] or '')[:60]!r}")
@ -59,9 +65,16 @@ async def run_ai_mode(label: str, *, queries: str) -> None:
async def run(
label: str, *, expect_ads=False, expect_products=False, expect_paa_answers=False,
expect_sitelinks=False, expect_aio=False, expect_device=None,
expect_icons=False, **kwargs
label: str,
*,
expect_ads=False,
expect_products=False,
expect_paa_answers=False,
expect_sitelinks=False,
expect_aio=False,
expect_device=None,
expect_icons=False,
**kwargs,
) -> None:
print(f"\n=== {label} ===")
t0 = time.perf_counter()
@ -72,18 +85,24 @@ async def run(
paa_answered = [p for p in it["peopleAlsoAsk"] if p["answer"]]
sitelinked = [o for o in it["organicResults"] if o["siteLinks"]]
print(f" term={it['searchQuery']['term']!r} resultsTotal={it['resultsTotal']}")
print(f" organic={len(it['organicResults'])} paidResults={len(it['paidResults'])} "
f"paidProducts={len(it['paidProducts'])} related={len(it['relatedQueries'])} "
f"suggested={len(it['suggestedResults'])} "
f"paa={len(it['peopleAlsoAsk'])} (answered={len(paa_answered)}) "
f"({time.perf_counter()-t0:.0f}s)")
print(
f" organic={len(it['organicResults'])} paidResults={len(it['paidResults'])} "
f"paidProducts={len(it['paidProducts'])} related={len(it['relatedQueries'])} "
f"suggested={len(it['suggestedResults'])} "
f"paa={len(it['peopleAlsoAsk'])} (answered={len(paa_answered)}) "
f"({time.perf_counter() - t0:.0f}s)"
)
for o in sitelinked[:2]:
print(f" [sitelinks on #{o['position']}] "
+ ", ".join(s["title"] for s in o["siteLinks"][:5]))
print(
f" [sitelinks on #{o['position']}] "
+ ", ".join(s["title"] for s in o["siteLinks"][:5])
)
aio = it["aiOverview"]
if aio:
print(f" [aiOverview] content={len(aio['content'])} chars, "
f"sources={len(aio['sources'])}")
print(
f" [aiOverview] content={len(aio['content'])} chars, "
f"sources={len(aio['sources'])}"
)
print(f" {aio['content'][:110]!r}")
for s in aio["sources"][:3]:
print(f" src: {(s['title'] or '')[:55]!r}")
@ -113,10 +132,15 @@ async def run(
f"{label}: device={it['searchQuery']['device']}"
)
if expect_icons:
iconed = [o for o in it["organicResults"]
if (o["icon"] or "").startswith("data:image")]
print(f" [icons] {len(iconed)}/{len(it['organicResults'])} organic "
f"carry a base64 favicon")
iconed = [
o
for o in it["organicResults"]
if (o["icon"] or "").startswith("data:image")
]
print(
f" [icons] {len(iconed)}/{len(it['organicResults'])} organic "
f"carry a base64 favicon"
)
assert iconed, f"{label}: expected base64 icons on organic results"
@ -124,24 +148,47 @@ _CASES = {
"plain": lambda: run("plain query", queries="python asyncio tutorial"),
"site": lambda: run("site: filter", queries="machine learning", site="arxiv.org"),
"ads": lambda: run("text ads", queries="car insurance quotes", expect_ads=True),
"products": lambda: run("product ads", queries="buy running shoes", expect_products=True),
"focus": lambda: run("focusOnPaidAds (commercial)", queries="car insurance quotes",
focusOnPaidAds=True, expect_ads=True),
"focus-neg": lambda: run("focusOnPaidAds (non-commercial, retries capped)",
queries="python asyncio tutorial", focusOnPaidAds=True),
"paa": lambda: run("people also ask", queries="what is seo", expect_paa_answers=True),
"sitelinks": lambda: run("sitelinks + suggested (brand query)", queries="amazon",
expect_sitelinks=True),
"products": lambda: run(
"product ads", queries="buy running shoes", expect_products=True
),
"focus": lambda: run(
"focusOnPaidAds (commercial)",
queries="car insurance quotes",
focusOnPaidAds=True,
expect_ads=True,
),
"focus-neg": lambda: run(
"focusOnPaidAds (non-commercial, retries capped)",
queries="python asyncio tutorial",
focusOnPaidAds=True,
),
"paa": lambda: run(
"people also ask", queries="what is seo", expect_paa_answers=True
),
"sitelinks": lambda: run(
"sitelinks + suggested (brand query)", queries="amazon", expect_sitelinks=True
),
"aio": lambda: run("AI Overview", queries="benefits of green tea", expect_aio=True),
"mobile": lambda: run("mobile layout (mobileResults)", queries="best seo tools",
mobileResults=True, expect_device="MOBILE"),
"unfiltered": lambda: run("includeUnfilteredResults (filter=0)",
queries="python asyncio tutorial",
includeUnfilteredResults=True),
"icons": lambda: run("includeIcons (base64 favicons)", queries="github",
includeIcons=True, expect_icons=True),
"aimode": lambda: run_ai_mode("Google AI Mode (udm=50)",
queries="what is quantum computing"),
"mobile": lambda: run(
"mobile layout (mobileResults)",
queries="best seo tools",
mobileResults=True,
expect_device="MOBILE",
),
"unfiltered": lambda: run(
"includeUnfilteredResults (filter=0)",
queries="python asyncio tutorial",
includeUnfilteredResults=True,
),
"icons": lambda: run(
"includeIcons (base64 favicons)",
queries="github",
includeIcons=True,
expect_icons=True,
),
"aimode": lambda: run_ai_mode(
"Google AI Mode (udm=50)", queries="what is quantum computing"
),
}

4
surfsense_backend/scripts/e2e_reddit_scraper.py
View file

@ -182,9 +182,7 @@ async def step5_dump_fixtures() -> bool:
wrote.append("sample_post.json")
# A single comment thing, for the comment-mapping fixture.
comment_kids = children(post[1]) if len(post) > 1 else []
first_comment = next(
(c for c in comment_kids if c.get("kind") == "t1"), None
)
first_comment = next((c for c in comment_kids if c.get("kind") == "t1"), None)
if first_comment:
(_FIXTURE_DIR / "sample_comment.json").write_text(
json.dumps(first_comment), encoding="utf-8"

4
surfsense_backend/tests/e2e/fakes/chat_llm.py
View file

@ -562,8 +562,8 @@ class FakeChatLLM(BaseChatModel):
# Marker unique to a connector subagent's prompt: the main agent must
# delegate via ``task``; only the subagent has connector tools registered.
in_connector_subagent = (
"connected-apps specialist" in _messages_to_text(messages)
in_connector_subagent = "connected-apps specialist" in _messages_to_text(
messages
)
# Main agent: delegate live-tool connector work to its subagent (which

8
surfsense_backend/tests/integration/agents/multi_agent_chat/test_web_search_delegation.py
View file

@ -29,9 +29,7 @@ def _last_ai_text(messages: list) -> str | None:
@pytest.mark.asyncio
async def test_web_query_delegates_to_google_search(
db_session, db_user, db_workspace
):
async def test_web_query_delegates_to_google_search(db_session, db_user, db_workspace):
"""A web-search query routes through ``task(google_search)`` and resumes.
Scripted sequence (the fake model is shared and consumed in order across the
@ -76,9 +74,7 @@ async def test_web_query_delegates_to_google_search(
)
task_tool_messages = [
m
for m in result["messages"]
if isinstance(m, ToolMessage) and m.name == "task"
m for m in result["messages"] if isinstance(m, ToolMessage) and m.name == "task"
]
assert task_tool_messages, "web query did not delegate through the task tool"
assert _last_ai_text(result["messages"]) == (

19
surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_discovery_migration.py
View file

@ -62,13 +62,15 @@ def _tool(name: str, metadata: dict) -> StructuredTool:
def test_all_mcp_connectors_route_to_discovery():
for connector_type in _MCP_ROUTED:
assert (
CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS[connector_type]
== MCP_DISCOVERY_NAME
CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS[connector_type] == MCP_DISCOVERY_NAME
), connector_type
def test_file_connectors_keep_native_routes():
assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["GOOGLE_DRIVE_CONNECTOR"] == "google_drive"
assert (
CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["GOOGLE_DRIVE_CONNECTOR"]
== "google_drive"
)
assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["DROPBOX_CONNECTOR"] == "dropbox"
assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["ONEDRIVE_CONNECTOR"] == "onedrive"
@ -87,7 +89,9 @@ def test_discovery_gating_is_any_of():
def test_discovery_gating_tokens_match_routed_connectors():
assert SUBAGENT_TO_REQUIRED_CONNECTOR_MAP[MCP_DISCOVERY_NAME] == frozenset(_MCP_ROUTED)
assert SUBAGENT_TO_REQUIRED_CONNECTOR_MAP[MCP_DISCOVERY_NAME] == frozenset(
_MCP_ROUTED
)
def test_legacy_aliases_resolve_to_a_live_subagent():
@ -104,7 +108,12 @@ def test_collision_only_prefixes_shared_names():
_tool("search", {"mcp_connector_id": 2, "mcp_transport": "http"}),
_tool("list_bases", {"mcp_connector_id": 2, "mcp_transport": "http"}),
]
resolved = {t.name: t for t in resolve_tool_name_collisions(tools, {1: "NOTION_CONNECTOR", 2: "AIRTABLE_CONNECTOR"})}
resolved = {
t.name: t
for t in resolve_tool_name_collisions(
tools, {1: "NOTION_CONNECTOR", 2: "AIRTABLE_CONNECTOR"}
)
}
# The unique tool keeps its bare name (trusted_tools / history stay valid).
assert "list_bases" in resolved

8
surfsense_backend/tests/unit/capabilities/access/test_rest_router.py
View file

@ -423,8 +423,12 @@ async def test_run_events_replays_buffer_then_finishes(monkeypatch):
row = _fake_run_row(status="running")
raw = str(row.id)
run_event_bus.publish(raw, {"type": "run.progress", "phase": "scraping", "current": 1})
run_event_bus.publish(raw, {"type": "run.finished", "status": "success", "item_count": 2})
run_event_bus.publish(
raw, {"type": "run.progress", "phase": "scraping", "current": 1}
)
run_event_bus.publish(
raw, {"type": "run.finished", "status": "success", "item_count": 2}
)
app = _build_app_with_rows(monkeypatch, [row])
try:

4
surfsense_backend/tests/unit/capabilities/reddit/scrape/test_executor.py
View file

@ -43,7 +43,9 @@ async def test_maps_urls_to_start_urls_and_wraps_items():
assert out.items[0].dataType == "post"
(actor_input, _limit) = scraper.calls[0]
assert [u.url for u in actor_input.startUrls] == ["https://www.reddit.com/r/python/"]
assert [u.url for u in actor_input.startUrls] == [
"https://www.reddit.com/r/python/"
]
assert actor_input.searches == []

4
surfsense_backend/tests/unit/capabilities/test_billing.py
View file

@ -233,7 +233,9 @@ async def test_gate_reserves_worst_case_captcha_when_solving_enabled(monkeypatch
with pytest.raises(InsufficientCreditsError):
await gate_capability(
CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
CrawlInput(startUrls=["https://a.com"]),
BillingUnit.WEB_CRAWL,
_ctx(session),
)

50
surfsense_backend/tests/unit/capabilities/test_run_truncation.py
View file

@ -64,7 +64,9 @@ def test_preview_is_char_budgeted_and_references_run():
per_item = "y" * 500
items = [f'{{"i": {i}, "v": "{per_item}"}}' for i in range(500)]
body = "\n".join(items)
serialized = SerializedOutput(text=body, item_count=len(items), char_count=len(body))
serialized = SerializedOutput(
text=body, item_count=len(items), char_count=len(body)
)
preview = _build_preview(serialized, run_id="abc")
@ -143,8 +145,11 @@ async def test_read_run_paginates(monkeypatch):
read_run, _ = _tools()
out = await read_run.ainvoke(
{"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
"offset": 2, "limit": 3}
{
"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
"offset": 2,
"limit": 3,
}
)
assert "item_2" in out and "item_3" in out and "item_4" in out
assert "item_0" not in out and "item_5" not in out
@ -184,8 +189,7 @@ async def test_search_run_excerpts_huge_matched_line(monkeypatch):
_, search_run = _tools()
out = await search_run.ainvoke(
{"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
"pattern": "NEEDLE"}
{"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", "pattern": "NEEDLE"}
)
assert "NEEDLE" in out
assert "match at char 100000" in out
@ -215,8 +219,10 @@ async def test_search_run_matches(monkeypatch):
_patch_session(monkeypatch, _BODY, [])
_, search_run = _tools()
out = await search_run.ainvoke(
{"ref": "spill_" + "0" * 8 + "-0000-0000-0000-000000000000",
"pattern": "item_7"}
{
"ref": "spill_" + "0" * 8 + "-0000-0000-0000-000000000000",
"pattern": "item_7",
}
)
assert "item_7" in out
assert "item_1" not in out.split("item_7")[0]
@ -232,14 +238,30 @@ _CRAWL_BODY = "\n".join(
"url": "https://x.com/team/",
"status": "success",
"links": [
{"url": "https://x.com/author/jane/", "text": "Jane Doe",
"context": "Jane Doe General Partner", "kind": "internal"},
{"url": "https://x.com/author/bob/", "text": "Bob Roe",
"context": "Bob Roe Operations", "kind": "internal"},
{
"url": "https://x.com/author/jane/",
"text": "Jane Doe",
"context": "Jane Doe General Partner",
"kind": "internal",
},
{
"url": "https://x.com/author/bob/",
"text": "Bob Roe",
"context": "Bob Roe Operations",
"kind": "internal",
},
# Duplicate of Jane (nav + card) — must dedupe.
{"url": "https://x.com/author/jane/", "text": "Jane Doe",
"context": "Jane Doe General Partner", "kind": "internal"},
{"url": "https://x.com/about/", "text": "About", "kind": "internal"},
{
"url": "https://x.com/author/jane/",
"text": "Jane Doe",
"context": "Jane Doe General Partner",
"kind": "internal",
},
{
"url": "https://x.com/about/",
"text": "About",
"kind": "internal",
},
],
}
),

6
surfsense_backend/tests/unit/capabilities/web/crawl/test_executor.py
View file

@ -101,7 +101,11 @@ async def test_aggregated_contacts_carry_provenance_and_site_wide_flag() -> None
class _ContactsEngine:
async def crawl_url(self, url: str) -> CrawlOutcome:
socials = [footer] + ([person] if url.endswith("/about") else [])
links = ["https://e.com/about", "https://e.com/blog"] if url == "https://e.com/" else []
links = (
["https://e.com/about", "https://e.com/blog"]
if url == "https://e.com/"
else []
)
return CrawlOutcome(
status=_SUCCESS,
result={

12
surfsense_backend/tests/unit/capabilities/web/crawl/test_schemas.py
View file

@ -43,18 +43,20 @@ def test_estimated_units_for_single_url_is_seed_count() -> None:
def test_estimated_units_for_spider_is_max_pages() -> None:
model = CrawlInput(
startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25
)
model = CrawlInput(startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25)
assert model.estimated_units == 25
def test_billable_units_counts_only_successes() -> None:
out = CrawlOutput(
items=[
CrawlItem(url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)),
CrawlItem(
url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)
),
CrawlItem(url="b", status="empty", crawl=CrawlMeta(loadedUrl="b", depth=1)),
CrawlItem(url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)),
CrawlItem(
url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)
),
]
)
assert out.billable_units == 1

9
surfsense_backend/tests/unit/platforms/google_search/test_skeleton.py
View file

@ -274,8 +274,13 @@ def test_parse_serp_extracts_all_blocks():
assert prod.prices == ["$24.99", "$30"]
# Related searches exclude the numeric pagination anchor (a.fl).
assert [r.title for r in item.relatedQueries] == ["easy apple pie", "apple pie recipe"]
assert item.relatedQueries[0].url == "https://www.google.com/search?q=easy+apple+pie"
assert [r.title for r in item.relatedQueries] == [
"easy apple pie",
"apple pie recipe",
]
assert (
item.relatedQueries[0].url == "https://www.google.com/search?q=easy+apple+pie"
)
# suggestedResults are the related queries re-shaped with type/position.
assert [(s.position, s.title, s.type) for s in item.suggestedResults] == [

31
surfsense_backend/tests/unit/platforms/reddit/test_parsers.py
View file

@ -95,8 +95,15 @@ def test_flatten_comments_counts_replies_and_stops_at_more():
"kind": "Listing",
"data": {
"children": [
{"kind": "t1", "data": {"name": "t1_2", "id": "2",
"body": "reply", "replies": ""}},
{
"kind": "t1",
"data": {
"name": "t1_2",
"id": "2",
"body": "reply",
"replies": "",
},
},
{"kind": "more", "data": {}}, # stub -> ignored
]
},
@ -112,8 +119,10 @@ def test_flatten_comments_counts_replies_and_stops_at_more():
def test_flatten_comments_honors_max():
tree = [
{"kind": "t1", "data": {"name": f"t1_{i}", "id": str(i), "body": "x",
"replies": ""}}
{
"kind": "t1",
"data": {"name": f"t1_{i}", "id": str(i), "body": "x", "replies": ""},
}
for i in range(5)
]
assert len(flatten_comments(tree, max_comments=2)) == 2
@ -128,9 +137,17 @@ def test_children_and_after():
def test_parse_community_maps_members():
thing = {"kind": "t5", "data": {"name": "t5_s", "id": "s",
"display_name": "py", "display_name_prefixed": "r/py",
"subscribers": 1234, "url": "/r/py/"}}
thing = {
"kind": "t5",
"data": {
"name": "t5_s",
"id": "s",
"display_name": "py",
"display_name_prefixed": "r/py",
"subscribers": 1234,
"url": "/r/py/",
},
}
item = parse_community(thing)
assert item["dataType"] == "community"
assert item["numberOfMembers"] == 1234

4
surfsense_backend/tests/unit/proprietary/web_crawler/test_site_crawler.py
View file

@ -125,9 +125,7 @@ async def test_dedupes_on_canonical_url() -> None:
}
)
await crawl_site(
engine, ["https://e.com/"], max_crawl_depth=3, max_crawl_pages=10
)
await crawl_site(engine, ["https://e.com/"], max_crawl_depth=3, max_crawl_pages=10)
assert engine.calls.count("https://e.com/a") == 1
assert engine.calls.count("https://e.com/") == 1

7
surfsense_backend/tests/unit/proprietary/web_crawler/test_url_policy.py
View file

@ -116,7 +116,12 @@ def test_extract_link_records_dedupes_keeping_first_nonempty_text() -> None:
html = '<a href="/p"><img src="logo.png"/></a><a href="/p">Pricing</a>'
records = extract_link_records(html, "https://example.com/")
assert records == [
{"url": "https://example.com/p", "text": "Pricing", "rel": "", "kind": "internal"}
{
"url": "https://example.com/p",
"text": "Pricing",
"rel": "",
"kind": "internal",
}
]

2
surfsense_backend/tests/unit/services/test_web_crawl_credit_service.py
View file

@ -241,5 +241,3 @@ class TestCheckBalance:
session, _user = _make_session(balance_micros=0)
await WebCrawlCreditService(session).check_balance(_USER_ID, 0)
session.execute.assert_not_called()

4
surfsense_backend/tests/unit/utils/proxy/test_dataimpulse_provider.py
View file

@ -47,9 +47,7 @@ def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None:
def test_no_country_suffix_yields_empty_location(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
Config, "PROXY_URL", "http://tok:secret@gw.dataimpulse.com:823"
)
monkeypatch.setattr(Config, "PROXY_URL", "http://tok:secret@gw.dataimpulse.com:823")
assert DataImpulseProvider().get_location() == ""

134
surfsense_backend/uv.lock generated
View file

@ -33,6 +33,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -64,6 +67,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -95,6 +101,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -126,6 +135,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -157,6 +169,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -188,6 +203,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -219,6 +237,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -250,6 +271,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -281,6 +305,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -312,6 +339,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -353,6 +383,10 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -384,6 +418,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -415,6 +452,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -446,6 +486,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -477,6 +520,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -508,6 +554,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -539,6 +588,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -570,6 +622,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -601,6 +656,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -632,6 +690,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -663,6 +724,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -704,6 +768,10 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -735,6 +803,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -766,6 +837,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -797,6 +871,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -828,6 +905,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -859,6 +939,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -890,6 +973,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -921,6 +1007,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -952,6 +1041,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -983,6 +1075,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -1014,6 +1109,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -1055,6 +1153,10 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -1086,6 +1188,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -1147,6 +1252,12 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -1178,6 +1289,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -1209,6 +1323,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -1270,6 +1387,12 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -1331,6 +1454,12 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'",
"python_version < '0'",
@ -1362,6 +1491,9 @@ resolution-markers = [
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
]
conflicts = [[
@ -10221,7 +10353,7 @@ wheels = [
[[package]]
name = "surf-new-backend"
version = "0.0.30"
version = "0.0.31"
source = { editable = "." }
dependencies = [
{ name = "alembic" },

2
surfsense_browser_extension/package.json
View file

@ -1,7 +1,7 @@
{
"name": "surfsense_browser_extension",
"displayName": "Surfsense Browser Extension",
"version": "0.0.30",
"version": "0.0.31",
"description": "Extension to collect Browsing History for SurfSense.",
"author": "https://github.com/MODSetter",
"engines": {

2
surfsense_desktop/package.json
View file

@ -1,7 +1,7 @@
{
"name": "surfsense-desktop",
"productName": "SurfSense",
"version": "0.0.30",
"version": "0.0.31",
"description": "SurfSense Desktop App",
"main": "dist/main.js",
"scripts": {

2
surfsense_web/package.json
View file

@ -1,6 +1,6 @@
{
"name": "surfsense_web",
"version": "0.0.30",
"version": "0.0.31",
"private": true,
"packageManager": "pnpm@10.26.0",
"description": "SurfSense Frontend",