chore: bumped version to 0.0.31

This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-06 21:43:15 -07:00
parent 8df8565e0a
commit 1c9ab207ef
56 changed files with 520 additions and 190 deletions

View file

@ -1 +1 @@
0.0.30 0.0.31

View file

@ -5,12 +5,12 @@ import sys
from logging.config import fileConfig from logging.config import fileConfig
import sqlalchemy as sa import sqlalchemy as sa
from alembic.script import ScriptDirectory
from sqlalchemy import pool from sqlalchemy import pool
from sqlalchemy.engine import Connection from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context from alembic import context
from alembic.script import ScriptDirectory
# Ensure the app directory is in the Python path # Ensure the app directory is in the Python path
# This allows Alembic to find your models # This allows Alembic to find your models
@ -81,9 +81,7 @@ def _fast_forward_fresh_db(connection: Connection) -> bool:
step rather than resurrecting the replay. step rather than resurrecting the replay.
""" """
for table in ("documents", "searchspaces", BOOTSTRAP_MARKER_TABLE): for table in ("documents", "searchspaces", BOOTSTRAP_MARKER_TABLE):
if connection.execute( if connection.execute(sa.text("SELECT to_regclass(:t)"), {"t": table}).scalar():
sa.text("SELECT to_regclass(:t)"), {"t": table}
).scalar():
return False return False
if connection.execute(sa.text("SELECT to_regclass('alembic_version')")).scalar(): if connection.execute(sa.text("SELECT to_regclass('alembic_version')")).scalar():
current = connection.execute( current = connection.execute(

View file

@ -41,9 +41,7 @@ def upgrade() -> None:
); );
""" """
) )
op.execute( op.execute("CREATE INDEX IF NOT EXISTS ix_runs_workspace_id ON runs (workspace_id)")
"CREATE INDEX IF NOT EXISTS ix_runs_workspace_id ON runs (workspace_id)"
)
op.execute("CREATE INDEX IF NOT EXISTS ix_runs_user_id ON runs (user_id)") op.execute("CREATE INDEX IF NOT EXISTS ix_runs_user_id ON runs (user_id)")
op.execute("CREATE INDEX IF NOT EXISTS ix_runs_capability ON runs (capability)") op.execute("CREATE INDEX IF NOT EXISTS ix_runs_capability ON runs (capability)")
op.execute("CREATE INDEX IF NOT EXISTS ix_runs_created_at ON runs (created_at)") op.execute("CREATE INDEX IF NOT EXISTS ix_runs_created_at ON runs (created_at)")

View file

@ -50,9 +50,7 @@ def _augment_allowlist_for_collision_prefixes(
continue continue
original = meta.get("mcp_original_tool_name") original = meta.get("mcp_original_tool_name")
if original in trusted_names and tool.name not in trusted_names: if original in trusted_names and tool.name not in trusted_names:
alias_rules.append( alias_rules.append(Rule(permission=tool.name, pattern="*", action="allow"))
Rule(permission=tool.name, pattern="*", action="allow")
)
if not alias_rules: if not alias_rules:
return dependencies return dependencies

View file

@ -85,8 +85,7 @@ def create_get_connected_accounts_tool(*, workspace_id: int) -> BaseTool:
# ``server_config`` presence == the connector produces agent # ``server_config`` presence == the connector produces agent
# tools. Native rows without it are connected for indexing # tools. Native rows without it are connected for indexing
# only and need a reconnect via MCP. # only and need a reconnect via MCP.
"usable_in_chat": isinstance(cfg, dict) "usable_in_chat": isinstance(cfg, dict) and "server_config" in cfg,
and "server_config" in cfg,
"account": account_meta, "account": account_meta,
} }
) )

View file

@ -83,7 +83,9 @@ def resolve_tool_name_collisions(
or original_name, or original_name,
"mcp_collision_prefixed": True, "mcp_collision_prefixed": True,
} }
resolved.append(tool.model_copy(update={"name": new_name, "metadata": new_meta})) resolved.append(
tool.model_copy(update={"name": new_name, "metadata": new_meta})
)
return resolved return resolved

View file

@ -128,7 +128,7 @@ def _cell(value: Any) -> str:
"""Render one CSV cell: scalars as-is, nested structures as compact JSON.""" """Render one CSV cell: scalars as-is, nested structures as compact JSON."""
if value is None: if value is None:
return "" return ""
if isinstance(value, (dict, list)): if isinstance(value, dict | list):
return json.dumps(value, ensure_ascii=False, default=str) return json.dumps(value, ensure_ascii=False, default=str)
return str(value) return str(value)
@ -230,9 +230,7 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]:
count = min(max(1, limit), _MAX_LIMIT) count = min(max(1, limit), _MAX_LIMIT)
window = lines[start : start + count] window = lines[start : start + count]
if not window: if not window:
return ( return f"No lines at offset {start} (total {len(lines)} lines in {ref})."
f"No lines at offset {start} (total {len(lines)} lines in {ref})."
)
window_body = "\n".join(window) window_body = "\n".join(window)
start_char = max(0, char_offset) start_char = max(0, char_offset)
if start_char >= len(window_body) > 0: if start_char >= len(window_body) > 0:
@ -328,10 +326,14 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]:
records = _rows_from_body(body, rows) records = _rows_from_body(body, rows)
if include_pattern: if include_pattern:
inc = _build_matcher(include_pattern.strip()) inc = _build_matcher(include_pattern.strip())
records = [r for r in records if inc(" ".join(map(_cell, r.values()))) is not None] records = [
r for r in records if inc(" ".join(map(_cell, r.values()))) is not None
]
if exclude_pattern: if exclude_pattern:
exc = _build_matcher(exclude_pattern.strip()) exc = _build_matcher(exclude_pattern.strip())
records = [r for r in records if exc(" ".join(map(_cell, r.values()))) is None] records = [
r for r in records if exc(" ".join(map(_cell, r.values()))) is None
]
if not records: if not records:
return ( return (
f"Error: no rows to export from {ref} " f"Error: no rows to export from {ref} "
@ -353,7 +355,9 @@ def build_run_reader_tools(*, workspace_id: int) -> list[BaseTool]:
preview_lines = csv_text.split("\n")[:4] preview_lines = csv_text.split("\n")[:4]
truncated_note = ( truncated_note = (
f" (capped at {_EXPORT_MAX_ROWS} rows)" if row_count >= _EXPORT_MAX_ROWS else "" f" (capped at {_EXPORT_MAX_ROWS} rows)"
if row_count >= _EXPORT_MAX_ROWS
else ""
) )
return ( return (
f"Exported {row_count} rows{truncated_note} to {final_path} " f"Exported {row_count} rows{truncated_note} to {final_path} "

View file

@ -627,9 +627,13 @@ async def _sweep_stale_scraper_runs() -> None:
async with async_session_maker() as session: async with async_session_maker() as session:
swept = await fail_stale_running_runs(session) swept = await fail_stale_running_runs(session)
if swept: if swept:
logger.info("[startup] Marked %d stale running scraper run(s) as error", swept) logger.info(
"[startup] Marked %d stale running scraper run(s) as error", swept
)
except Exception: except Exception:
logger.warning("[startup] Stale scraper-run sweep failed (non-fatal)", exc_info=True) logger.warning(
"[startup] Stale scraper-run sweep failed (non-fatal)", exc_info=True
)
@asynccontextmanager @asynccontextmanager

View file

@ -209,7 +209,10 @@ async def _execute_async_run(
raise raise
except (SurfSenseError, HTTPException) as exc: except (SurfSenseError, HTTPException) as exc:
await _finalize_async( await _finalize_async(
run_id, status="error", error=str(exc), started=started, run_id,
status="error",
error=str(exc),
started=started,
progress=reporter.coarse, progress=reporter.coarse,
) )
_publish_finished(run_id, "error", error=str(exc)) _publish_finished(run_id, "error", error=str(exc))

View file

@ -291,7 +291,9 @@ async def record_spill(
return None return None
async def _maybe_cleanup(session: AsyncSession, table: str, retention_days: int) -> None: async def _maybe_cleanup(
session: AsyncSession, table: str, retention_days: int
) -> None:
"""Delete a bounded batch of expired rows on ~1% of inserts.""" """Delete a bounded batch of expired rows on ~1% of inserts."""
if random.random() >= _CLEANUP_SAMPLE_RATE: if random.random() >= _CLEANUP_SAMPLE_RATE:
return return

View file

@ -30,14 +30,21 @@ def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor:
reviewsStartDate=payload.start_date, reviewsStartDate=payload.start_date,
language=payload.language, language=payload.language,
) )
emit_progress("starting", "Fetching Google Maps reviews", total=payload.max_reviews, unit="review") emit_progress(
"starting",
"Fetching Google Maps reviews",
total=payload.max_reviews,
unit="review",
)
try: try:
items = await scrape_fn(actor_input) items = await scrape_fn(actor_input)
except SignInRequiredError as exc: except SignInRequiredError as exc:
raise ForbiddenError( raise ForbiddenError(
f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED" f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED"
) from exc ) from exc
emit_progress("done", f"Scraped {len(items)} review(s)", current=len(items), unit="review") emit_progress(
"done", f"Scraped {len(items)} review(s)", current=len(items), unit="review"
)
return ReviewsOutput(items=items) return ReviewsOutput(items=items)
return execute return execute

View file

@ -33,14 +33,18 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
maxReviews=payload.max_reviews, maxReviews=payload.max_reviews,
maxImages=payload.max_images, maxImages=payload.max_images,
) )
emit_progress("starting", "Searching Google Maps", total=payload.max_places, unit="place") emit_progress(
"starting", "Searching Google Maps", total=payload.max_places, unit="place"
)
try: try:
items = await scrape_fn(actor_input) items = await scrape_fn(actor_input)
except SignInRequiredError as exc: except SignInRequiredError as exc:
raise ForbiddenError( raise ForbiddenError(
f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED" f"Google sign in required: {exc}", code="GOOGLE_SIGNIN_REQUIRED"
) from exc ) from exc
emit_progress("done", f"Scraped {len(items)} place(s)", current=len(items), unit="place") emit_progress(
"done", f"Scraped {len(items)} place(s)", current=len(items), unit="place"
)
return ScrapeOutput(items=items) return ScrapeOutput(items=items)
return execute return execute

View file

@ -43,7 +43,12 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
unit="page", unit="page",
) )
items = await scrape_fn(actor_input, limit=_MAX_SERP_ITEMS) items = await scrape_fn(actor_input, limit=_MAX_SERP_ITEMS)
emit_progress("done", f"Scraped {len(items)} SERP page(s)", current=len(items), unit="page") emit_progress(
"done",
f"Scraped {len(items)} SERP page(s)",
current=len(items),
unit="page",
)
return ScrapeOutput(items=items) return ScrapeOutput(items=items)
return execute return execute

View file

@ -36,7 +36,9 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
postDateLimit=payload.post_date_limit, postDateLimit=payload.post_date_limit,
commentDateLimit=payload.comment_date_limit, commentDateLimit=payload.comment_date_limit,
) )
emit_progress("starting", "Resolving Reddit targets", total=payload.max_items, unit="item") emit_progress(
"starting", "Resolving Reddit targets", total=payload.max_items, unit="item"
)
try: try:
items = await scrape_fn(actor_input, limit=payload.max_items) items = await scrape_fn(actor_input, limit=payload.max_items)
except RedditAccessBlockedError as exc: except RedditAccessBlockedError as exc:
@ -46,7 +48,9 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
f"Reddit refused anonymous access: {exc}", f"Reddit refused anonymous access: {exc}",
code="REDDIT_ACCESS_BLOCKED", code="REDDIT_ACCESS_BLOCKED",
) from exc ) from exc
emit_progress("done", f"Scraped {len(items)} item(s)", current=len(items), unit="item") emit_progress(
"done", f"Scraped {len(items)} item(s)", current=len(items), unit="item"
)
return ScrapeOutput(items=items) return ScrapeOutput(items=items)
return execute return execute

View file

@ -60,7 +60,9 @@ def build_crawl_executor(engine: WebCrawlerConnector | None = None) -> Executor:
unit="page", unit="page",
) )
items = [_to_item(page, payload.maxLength) for page in pages] items = [_to_item(page, payload.maxLength) for page in pages]
emit_progress("done", f"Crawled {len(items)} page(s)", current=len(items), unit="page") emit_progress(
"done", f"Crawled {len(items)} page(s)", current=len(items), unit="page"
)
return CrawlOutput( return CrawlOutput(
items=items, items=items,
contacts=_aggregate_contacts(items), contacts=_aggregate_contacts(items),

View file

@ -165,7 +165,8 @@ class CrawlItem(BaseModel):
default=None, description="Crawl provenance (loaded URL, depth, referrer)." default=None, description="Crawl provenance (loaded URL, depth, referrer)."
) )
markdown: str | None = Field( markdown: str | None = Field(
default=None, description="Cleaned page content as markdown (null unless success)." default=None,
description="Cleaned page content as markdown (null unless success).",
) )
metadata: dict[str, str] | None = Field( metadata: dict[str, str] | None = Field(
default=None, description="Page metadata such as title and description." default=None, description="Page metadata such as title and description."

View file

@ -25,9 +25,19 @@ def build_comments_executor(scrape_fn: CommentsFn | None = None) -> Executor:
maxComments=payload.max_comments, maxComments=payload.max_comments,
sortCommentsBy=payload.sort_by, sortCommentsBy=payload.sort_by,
) )
emit_progress("starting", "Fetching YouTube comments", total=payload.max_comments, unit="comment") emit_progress(
"starting",
"Fetching YouTube comments",
total=payload.max_comments,
unit="comment",
)
items = await scrape_fn(actor_input) items = await scrape_fn(actor_input)
emit_progress("done", f"Scraped {len(items)} comment(s)", current=len(items), unit="comment") emit_progress(
"done",
f"Scraped {len(items)} comment(s)",
current=len(items),
unit="comment",
)
return CommentsOutput(items=items) return CommentsOutput(items=items)
return execute return execute

View file

@ -31,9 +31,16 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor:
downloadSubtitles=payload.download_subtitles, downloadSubtitles=payload.download_subtitles,
subtitlesLanguage=payload.subtitles_language, subtitlesLanguage=payload.subtitles_language,
) )
emit_progress("starting", "Resolving YouTube targets", total=payload.max_results, unit="video") emit_progress(
"starting",
"Resolving YouTube targets",
total=payload.max_results,
unit="video",
)
items = await scrape_fn(actor_input) items = await scrape_fn(actor_input)
emit_progress("done", f"Scraped {len(items)} video(s)", current=len(items), unit="video") emit_progress(
"done", f"Scraped {len(items)} video(s)", current=len(items), unit="video"
)
return ScrapeOutput(items=items) return ScrapeOutput(items=items)
return execute return execute

View file

@ -713,9 +713,7 @@ class Config:
YOUTUBE_MICROS_PER_VIDEO = int(os.getenv("YOUTUBE_MICROS_PER_VIDEO", "2500")) YOUTUBE_MICROS_PER_VIDEO = int(os.getenv("YOUTUBE_MICROS_PER_VIDEO", "2500"))
# Kept separate from the video rate so comments can be re-tuned toward the # Kept separate from the video rate so comments can be re-tuned toward the
# cheaper per-comment market ($0.40-2.00/1k) without touching video pricing. # cheaper per-comment market ($0.40-2.00/1k) without touching video pricing.
YOUTUBE_MICROS_PER_COMMENT = int( YOUTUBE_MICROS_PER_COMMENT = int(os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500"))
os.getenv("YOUTUBE_MICROS_PER_COMMENT", "1500")
)
# Low-balance WARNING threshold (micro-USD). Surfaced by the quota service # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service
# so the UI can nudge the user to top up / enable auto-reload. $0.50. # so the UI can nudge the user to top up / enable auto-reload. $0.50.

View file

@ -2795,9 +2795,7 @@ class Run(Base, TimestampMixin):
__tablename__ = "runs" __tablename__ = "runs"
__allow_unmapped__ = True __allow_unmapped__ = True
__table_args__ = ( __table_args__ = (Index("ix_runs_workspace_created", "workspace_id", "created_at"),)
Index("ix_runs_workspace_created", "workspace_id", "created_at"),
)
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
workspace_id = Column( workspace_id = Column(

View file

@ -31,6 +31,7 @@ if we add another sticky vendor is to extend ``_STICKY_HOSTS``.
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import contextlib
import logging import logging
import random import random
import sys import sys
@ -158,8 +159,7 @@ async def _vet_fresh_ip(url: str, base: str) -> str | None:
return proxy if await _precheck(url, proxy) else None return proxy if await _precheck(url, proxy) else None
tasks = [ tasks = [
asyncio.create_task(vet(_sticky_variant(base))) asyncio.create_task(vet(_sticky_variant(base))) for _ in range(_VET_CONCURRENCY)
for _ in range(_VET_CONCURRENCY)
] ]
winner: str | None = None winner: str | None = None
for fut in asyncio.as_completed(tasks): for fut in asyncio.as_completed(tasks):
@ -306,10 +306,8 @@ async def _drop_session_on_loop(mobile: bool) -> None:
async with _session_lock: async with _session_lock:
session = _sessions.pop(mobile, None) session = _sessions.pop(mobile, None)
if session is not None: if session is not None:
try: with contextlib.suppress(Exception): # already dead; nothing to salvage
await session.close() await session.close()
except Exception: # already dead; nothing to salvage
pass
async def _drop_session(mobile: bool) -> None: async def _drop_session(mobile: bool) -> None:

View file

@ -218,7 +218,9 @@ def parse_organic(doc: Adaptor, *, include_icons: bool = False) -> list[OrganicR
return results return results
def parse_paid_results(doc: Adaptor, *, include_icons: bool = False) -> list[PaidResult]: def parse_paid_results(
doc: Adaptor, *, include_icons: bool = False
) -> list[PaidResult]:
"""Text ads (``div[data-text-ad]``), covering the top and bottom ad blocks. """Text ads (``div[data-text-ad]``), covering the top and bottom ad blocks.
Fields mirror an organic result: the heading is the title, the ad's anchor Fields mirror an organic result: the heading is the title, the ad's anchor
@ -239,7 +241,11 @@ def parse_paid_results(doc: Adaptor, *, include_icons: bool = False) -> list[Pai
description = None description = None
for cand in block.css(".Va3FIb"): for cand in block.css(".Va3FIb"):
text = _text(cand) text = _text(cand)
if text and text != title and (description is None or len(text) > len(description)): if (
text
and text != title
and (description is None or len(text) > len(description))
):
description = text description = text
slot = block.attrib.get("data-ta-slot-pos") slot = block.attrib.get("data-ta-slot-pos")
ads.append( ads.append(

View file

@ -291,8 +291,8 @@ async def _user_flow(
date_limit=input_model.postDateLimit, date_limit=input_model.postDateLimit,
): ):
# A user listing mixes posts (t3) and comments (t1); a post has a title. # A user listing mixes posts (t3) and comments (t1); a post has a title.
parsed = parse_post(data) if data.get("title") is not None else parse_comment( parsed = (
data parse_post(data) if data.get("title") is not None else parse_comment(data)
) )
item = _emit(parsed, include_nsfw=input_model.includeNSFW) item = _emit(parsed, include_nsfw=input_model.includeNSFW)
if item is not None: if item is not None:
@ -401,8 +401,7 @@ async def iter_reddit(
continue continue
resolved.append(r) resolved.append(r)
jobs = [ jobs = [
_dispatch(r, input_model) _dispatch(r, input_model) for r in _capped_targets(resolved, input_model)
for r in _capped_targets(resolved, input_model)
] ]
async for item in fan_out(jobs): async for item in fan_out(jobs):
yield item yield item
@ -449,9 +448,7 @@ async def scrape_reddit(
results: list[dict[str, Any]] = [] results: list[dict[str, Any]] = []
async for item in iter_reddit(input_model): async for item in iter_reddit(input_model):
results.append(item) results.append(item)
emit_progress( emit_progress("scraping", current=len(results), total=limit, unit="item")
"scraping", current=len(results), total=limit, unit="item"
)
if limit is not None and len(results) >= limit: if limit is not None and len(results) >= limit:
break break
return results return results

View file

@ -28,9 +28,7 @@ _REDDIT_HOSTS = frozenset(
) )
# Listing sorts that can appear as a trailing subreddit path segment. # Listing sorts that can appear as a trailing subreddit path segment.
_SUBREDDIT_SORTS = frozenset( _SUBREDDIT_SORTS = frozenset({"hot", "new", "top", "rising", "controversial", "best"})
{"hot", "new", "top", "rising", "controversial", "best"}
)
_USER_CONTENT = frozenset({"overview", "submitted", "comments"}) _USER_CONTENT = frozenset({"overview", "submitted", "comments"})

View file

@ -69,9 +69,7 @@ def _fetch_subtitles_sync(
api = _build_client() api = _build_client()
try: try:
transcript_list = api.list(video_id) transcript_list = api.list(video_id)
transcript = _select_transcript( transcript = _select_transcript(transcript_list, language, prefer_generated)
transcript_list, language, prefer_generated
)
fetched = transcript.fetch() fetched = transcript.fetch()
break break
except RequestBlocked: # covers IpBlocked; rotate to a fresh exit IP except RequestBlocked: # covers IpBlocked; rotate to a fresh exit IP

View file

@ -91,7 +91,9 @@ _CURRENCY_AMOUNT_RE = re.compile(
re.IGNORECASE, re.IGNORECASE,
) )
_STRIP_XPATH = "//script | //style | //noscript | //template | //svg | //iframe | //head" _STRIP_XPATH = (
"//script | //style | //noscript | //template | //svg | //iframe | //head"
)
def _visible_text(raw_html: str) -> str: def _visible_text(raw_html: str) -> str:
@ -278,9 +280,7 @@ class WebCrawlerConnector:
else: else:
reached_without_content = True reached_without_content = True
errors.append("Scrapling static: empty extraction") errors.append("Scrapling static: empty extraction")
self._log_tier_outcome( self._log_tier_outcome("scrapling-static", url, tier_start, "empty")
"scrapling-static", url, tier_start, "empty"
)
except Exception as exc: except Exception as exc:
errors.append(f"Scrapling static: {exc!s}") errors.append(f"Scrapling static: {exc!s}")
self._log_tier_outcome( self._log_tier_outcome(

View file

@ -156,9 +156,7 @@ class _SiteSpider(Spider):
# ladder + proxy rotation; never let the spider re-fetch on top of that. # ladder + proxy rotation; never let the spider re-fetch on top of that.
return False return False
async def parse( async def parse(self, response: Response) -> AsyncGenerator[Request | None, None]:
self, response: Response
) -> AsyncGenerator[Request | None, None]:
outcome: CrawlOutcome = response._outcome # type: ignore[attr-defined] outcome: CrawlOutcome = response._outcome # type: ignore[attr-defined]
depth: int = response.meta.get("_depth", 0) depth: int = response.meta.get("_depth", 0)
referrer: str | None = response.meta.get("_referrer") referrer: str | None = response.meta.get("_referrer")

View file

@ -73,9 +73,7 @@ def _anchor_context(anchor: Any) -> str:
return "" return ""
def extract_link_records( def extract_link_records(page_html: str | None, base_url: str) -> list[dict[str, str]]:
page_html: str | None, base_url: str
) -> list[dict[str, str]]:
"""Structured ``<a>`` inventory: ``{url, text, context, rel, kind}`` per target. """Structured ``<a>`` inventory: ``{url, text, context, rel, kind}`` per target.
``kind`` is one of ``internal`` (same site as ``base_url``), ``external``, ``kind`` is one of ``internal`` (same site as ``base_url``), ``external``,

View file

@ -218,9 +218,7 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = {
"create-pages", "create-pages",
"update-page", "update-page",
], ],
readonly_tools=frozenset( readonly_tools=frozenset({"notion-search", "notion-fetch", "search", "fetch"}),
{"notion-search", "notion-fetch", "search", "fetch"}
),
account_metadata_keys=["workspace_name"], account_metadata_keys=["workspace_name"],
), ),
"confluence": MCPServiceConfig( "confluence": MCPServiceConfig(

View file

@ -22,7 +22,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.services.etl_credit_service import InsufficientCreditsError from app.services.etl_credit_service import InsufficientCreditsError
__all__ = ["InsufficientCreditsError", "apply_debit", "check_balance", "spendable_micros"] __all__ = [
"InsufficientCreditsError",
"apply_debit",
"check_balance",
"spendable_micros",
]
async def spendable_micros(session: AsyncSession, user_id: str | UUID) -> int: async def spendable_micros(session: AsyncSession, user_id: str | UUID) -> int:

View file

@ -1393,4 +1393,3 @@ async def _index_uploaded_folder_files_async(
heartbeat_task.cancel() heartbeat_task.cancel()
if notification_id is not None: if notification_id is not None:
_stop_heartbeat(notification_id) _stop_heartbeat(notification_id)

View file

@ -87,9 +87,30 @@ _NOISE_EMAIL_DOMAINS = frozenset(
# or version-pinned dep (``react@18.2.0.js``) matches the email shape. # or version-pinned dep (``react@18.2.0.js``) matches the email shape.
_ASSET_TLDS = frozenset( _ASSET_TLDS = frozenset(
{ {
"png", "jpg", "jpeg", "gif", "svg", "webp", "ico", "bmp", "png",
"css", "js", "mjs", "cjs", "ts", "map", "json", "xml", "jpg",
"woff", "woff2", "ttf", "eot", "otf", "php", "html", "htm", "jpeg",
"gif",
"svg",
"webp",
"ico",
"bmp",
"css",
"js",
"mjs",
"cjs",
"ts",
"map",
"json",
"xml",
"woff",
"woff2",
"ttf",
"eot",
"otf",
"php",
"html",
"htm",
} }
) )
@ -100,19 +121,47 @@ _EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}")
# common locals like hello/info/contact/support. # common locals like hello/info/contact/support.
_PLACEHOLDER_EMAIL_LOCALS = frozenset( _PLACEHOLDER_EMAIL_LOCALS = frozenset(
{ {
"youremail", "yourname", "youraddress", "myemail", "email", "name", "youremail",
"user", "username", "someone", "somebody", "johndoe", "janedoe", "yourname",
"firstname", "lastname", "firstnamelastname", "firstlast", "youraddress",
"test", "example", "sample", "placeholder", "myemail",
"email",
"name",
"user",
"username",
"someone",
"somebody",
"johndoe",
"janedoe",
"firstname",
"lastname",
"firstnamelastname",
"firstlast",
"test",
"example",
"sample",
"placeholder",
} }
) )
# Placeholder profile handles left in site templates ("github.com/username"). # Placeholder profile handles left in site templates ("github.com/username").
_PLACEHOLDER_SOCIAL_SEGMENTS = frozenset( _PLACEHOLDER_SOCIAL_SEGMENTS = frozenset(
{ {
"username", "yourusername", "yourhandle", "handle", "user", "username",
"profile", "yourprofile", "yourname", "yourpage", "pagename", "yourusername",
"youraccount", "account", "example", "placeholder", "yourcompany", "yourhandle",
"handle",
"user",
"profile",
"yourprofile",
"yourname",
"yourpage",
"pagename",
"youraccount",
"account",
"example",
"placeholder",
"yourcompany",
} }
) )

View file

@ -1,6 +1,6 @@
[project] [project]
name = "surf-new-backend" name = "surf-new-backend"
version = "0.0.30" version = "0.0.31"
description = "SurfSense Backend" description = "SurfSense Backend"
requires-python = ">=3.12" requires-python = ">=3.12"
dependencies = [ dependencies = [

View file

@ -94,6 +94,7 @@ async def set_version(version: str | None) -> None:
async def assert_at_head() -> None: async def assert_at_head() -> None:
import asyncpg import asyncpg
from alembic.script import ScriptDirectory from alembic.script import ScriptDirectory
head = ScriptDirectory(str(BACKEND_DIR / "alembic")).get_current_head() head = ScriptDirectory(str(BACKEND_DIR / "alembic")).get_current_head()

View file

@ -40,7 +40,9 @@ async def main() -> None:
await admin.execute(f'DROP DATABASE IF EXISTS "{SCRATCH_DB}" WITH (FORCE)') await admin.execute(f'DROP DATABASE IF EXISTS "{SCRATCH_DB}" WITH (FORCE)')
await admin.close() await admin.close()
print("OK: ensure_publication creates and verifies on a create_all DB, idempotently.") print(
"OK: ensure_publication creates and verifies on a create_all DB, idempotently."
)
asyncio.run(main()) asyncio.run(main())

View file

@ -28,7 +28,9 @@ sys.path.insert(0, str(_ROOT))
load_dotenv(_ROOT / ".env") load_dotenv(_ROOT / ".env")
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(logging.INFO) logging.getLogger("app.proprietary.platforms.google_search.scraper").setLevel(
logging.INFO
)
from app.proprietary.platforms.google_search import ( # noqa: E402 from app.proprietary.platforms.google_search import ( # noqa: E402
GoogleSearchScrapeInput, GoogleSearchScrapeInput,
@ -41,15 +43,19 @@ async def run_ai_mode(label: str, *, queries: str) -> None:
print(f"\n=== {label} ===") print(f"\n=== {label} ===")
t0 = time.perf_counter() t0 = time.perf_counter()
inp = GoogleSearchScrapeInput( inp = GoogleSearchScrapeInput(
queries=queries, countryCode="us", languageCode="en", queries=queries,
countryCode="us",
languageCode="en",
aiModeSearch={"enableAiMode": True}, aiModeSearch={"enableAiMode": True},
) )
items = await scrape_serps(inp, limit=2) items = await scrape_serps(inp, limit=2)
ai_items = [i for i in items if i["aiModeResult"]] ai_items = [i for i in items if i["aiModeResult"]]
assert ai_items, f"{label}: no aiModeResult item emitted" assert ai_items, f"{label}: no aiModeResult item emitted"
res = ai_items[0]["aiModeResult"] res = ai_items[0]["aiModeResult"]
print(f" text={len(res['text'])} chars, sources={len(res['sources'])} " print(
f"({time.perf_counter()-t0:.0f}s)") f" text={len(res['text'])} chars, sources={len(res['sources'])} "
f"({time.perf_counter() - t0:.0f}s)"
)
print(f" {res['text'][:130]!r}") print(f" {res['text'][:130]!r}")
for s in res["sources"][:3]: for s in res["sources"][:3]:
print(f" src: {(s['title'] or '')[:60]!r}") print(f" src: {(s['title'] or '')[:60]!r}")
@ -59,9 +65,16 @@ async def run_ai_mode(label: str, *, queries: str) -> None:
async def run( async def run(
label: str, *, expect_ads=False, expect_products=False, expect_paa_answers=False, label: str,
expect_sitelinks=False, expect_aio=False, expect_device=None, *,
expect_icons=False, **kwargs expect_ads=False,
expect_products=False,
expect_paa_answers=False,
expect_sitelinks=False,
expect_aio=False,
expect_device=None,
expect_icons=False,
**kwargs,
) -> None: ) -> None:
print(f"\n=== {label} ===") print(f"\n=== {label} ===")
t0 = time.perf_counter() t0 = time.perf_counter()
@ -72,18 +85,24 @@ async def run(
paa_answered = [p for p in it["peopleAlsoAsk"] if p["answer"]] paa_answered = [p for p in it["peopleAlsoAsk"] if p["answer"]]
sitelinked = [o for o in it["organicResults"] if o["siteLinks"]] sitelinked = [o for o in it["organicResults"] if o["siteLinks"]]
print(f" term={it['searchQuery']['term']!r} resultsTotal={it['resultsTotal']}") print(f" term={it['searchQuery']['term']!r} resultsTotal={it['resultsTotal']}")
print(f" organic={len(it['organicResults'])} paidResults={len(it['paidResults'])} " print(
f"paidProducts={len(it['paidProducts'])} related={len(it['relatedQueries'])} " f" organic={len(it['organicResults'])} paidResults={len(it['paidResults'])} "
f"suggested={len(it['suggestedResults'])} " f"paidProducts={len(it['paidProducts'])} related={len(it['relatedQueries'])} "
f"paa={len(it['peopleAlsoAsk'])} (answered={len(paa_answered)}) " f"suggested={len(it['suggestedResults'])} "
f"({time.perf_counter()-t0:.0f}s)") f"paa={len(it['peopleAlsoAsk'])} (answered={len(paa_answered)}) "
f"({time.perf_counter() - t0:.0f}s)"
)
for o in sitelinked[:2]: for o in sitelinked[:2]:
print(f" [sitelinks on #{o['position']}] " print(
+ ", ".join(s["title"] for s in o["siteLinks"][:5])) f" [sitelinks on #{o['position']}] "
+ ", ".join(s["title"] for s in o["siteLinks"][:5])
)
aio = it["aiOverview"] aio = it["aiOverview"]
if aio: if aio:
print(f" [aiOverview] content={len(aio['content'])} chars, " print(
f"sources={len(aio['sources'])}") f" [aiOverview] content={len(aio['content'])} chars, "
f"sources={len(aio['sources'])}"
)
print(f" {aio['content'][:110]!r}") print(f" {aio['content'][:110]!r}")
for s in aio["sources"][:3]: for s in aio["sources"][:3]:
print(f" src: {(s['title'] or '')[:55]!r}") print(f" src: {(s['title'] or '')[:55]!r}")
@ -113,10 +132,15 @@ async def run(
f"{label}: device={it['searchQuery']['device']}" f"{label}: device={it['searchQuery']['device']}"
) )
if expect_icons: if expect_icons:
iconed = [o for o in it["organicResults"] iconed = [
if (o["icon"] or "").startswith("data:image")] o
print(f" [icons] {len(iconed)}/{len(it['organicResults'])} organic " for o in it["organicResults"]
f"carry a base64 favicon") if (o["icon"] or "").startswith("data:image")
]
print(
f" [icons] {len(iconed)}/{len(it['organicResults'])} organic "
f"carry a base64 favicon"
)
assert iconed, f"{label}: expected base64 icons on organic results" assert iconed, f"{label}: expected base64 icons on organic results"
@ -124,24 +148,47 @@ _CASES = {
"plain": lambda: run("plain query", queries="python asyncio tutorial"), "plain": lambda: run("plain query", queries="python asyncio tutorial"),
"site": lambda: run("site: filter", queries="machine learning", site="arxiv.org"), "site": lambda: run("site: filter", queries="machine learning", site="arxiv.org"),
"ads": lambda: run("text ads", queries="car insurance quotes", expect_ads=True), "ads": lambda: run("text ads", queries="car insurance quotes", expect_ads=True),
"products": lambda: run("product ads", queries="buy running shoes", expect_products=True), "products": lambda: run(
"focus": lambda: run("focusOnPaidAds (commercial)", queries="car insurance quotes", "product ads", queries="buy running shoes", expect_products=True
focusOnPaidAds=True, expect_ads=True), ),
"focus-neg": lambda: run("focusOnPaidAds (non-commercial, retries capped)", "focus": lambda: run(
queries="python asyncio tutorial", focusOnPaidAds=True), "focusOnPaidAds (commercial)",
"paa": lambda: run("people also ask", queries="what is seo", expect_paa_answers=True), queries="car insurance quotes",
"sitelinks": lambda: run("sitelinks + suggested (brand query)", queries="amazon", focusOnPaidAds=True,
expect_sitelinks=True), expect_ads=True,
),
"focus-neg": lambda: run(
"focusOnPaidAds (non-commercial, retries capped)",
queries="python asyncio tutorial",
focusOnPaidAds=True,
),
"paa": lambda: run(
"people also ask", queries="what is seo", expect_paa_answers=True
),
"sitelinks": lambda: run(
"sitelinks + suggested (brand query)", queries="amazon", expect_sitelinks=True
),
"aio": lambda: run("AI Overview", queries="benefits of green tea", expect_aio=True), "aio": lambda: run("AI Overview", queries="benefits of green tea", expect_aio=True),
"mobile": lambda: run("mobile layout (mobileResults)", queries="best seo tools", "mobile": lambda: run(
mobileResults=True, expect_device="MOBILE"), "mobile layout (mobileResults)",
"unfiltered": lambda: run("includeUnfilteredResults (filter=0)", queries="best seo tools",
queries="python asyncio tutorial", mobileResults=True,
includeUnfilteredResults=True), expect_device="MOBILE",
"icons": lambda: run("includeIcons (base64 favicons)", queries="github", ),
includeIcons=True, expect_icons=True), "unfiltered": lambda: run(
"aimode": lambda: run_ai_mode("Google AI Mode (udm=50)", "includeUnfilteredResults (filter=0)",
queries="what is quantum computing"), queries="python asyncio tutorial",
includeUnfilteredResults=True,
),
"icons": lambda: run(
"includeIcons (base64 favicons)",
queries="github",
includeIcons=True,
expect_icons=True,
),
"aimode": lambda: run_ai_mode(
"Google AI Mode (udm=50)", queries="what is quantum computing"
),
} }

View file

@ -182,9 +182,7 @@ async def step5_dump_fixtures() -> bool:
wrote.append("sample_post.json") wrote.append("sample_post.json")
# A single comment thing, for the comment-mapping fixture. # A single comment thing, for the comment-mapping fixture.
comment_kids = children(post[1]) if len(post) > 1 else [] comment_kids = children(post[1]) if len(post) > 1 else []
first_comment = next( first_comment = next((c for c in comment_kids if c.get("kind") == "t1"), None)
(c for c in comment_kids if c.get("kind") == "t1"), None
)
if first_comment: if first_comment:
(_FIXTURE_DIR / "sample_comment.json").write_text( (_FIXTURE_DIR / "sample_comment.json").write_text(
json.dumps(first_comment), encoding="utf-8" json.dumps(first_comment), encoding="utf-8"

View file

@ -562,8 +562,8 @@ class FakeChatLLM(BaseChatModel):
# Marker unique to a connector subagent's prompt: the main agent must # Marker unique to a connector subagent's prompt: the main agent must
# delegate via ``task``; only the subagent has connector tools registered. # delegate via ``task``; only the subagent has connector tools registered.
in_connector_subagent = ( in_connector_subagent = "connected-apps specialist" in _messages_to_text(
"connected-apps specialist" in _messages_to_text(messages) messages
) )
# Main agent: delegate live-tool connector work to its subagent (which # Main agent: delegate live-tool connector work to its subagent (which

View file

@ -29,9 +29,7 @@ def _last_ai_text(messages: list) -> str | None:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_web_query_delegates_to_google_search( async def test_web_query_delegates_to_google_search(db_session, db_user, db_workspace):
db_session, db_user, db_workspace
):
"""A web-search query routes through ``task(google_search)`` and resumes. """A web-search query routes through ``task(google_search)`` and resumes.
Scripted sequence (the fake model is shared and consumed in order across the Scripted sequence (the fake model is shared and consumed in order across the
@ -76,9 +74,7 @@ async def test_web_query_delegates_to_google_search(
) )
task_tool_messages = [ task_tool_messages = [
m m for m in result["messages"] if isinstance(m, ToolMessage) and m.name == "task"
for m in result["messages"]
if isinstance(m, ToolMessage) and m.name == "task"
] ]
assert task_tool_messages, "web query did not delegate through the task tool" assert task_tool_messages, "web query did not delegate through the task tool"
assert _last_ai_text(result["messages"]) == ( assert _last_ai_text(result["messages"]) == (

View file

@ -62,13 +62,15 @@ def _tool(name: str, metadata: dict) -> StructuredTool:
def test_all_mcp_connectors_route_to_discovery(): def test_all_mcp_connectors_route_to_discovery():
for connector_type in _MCP_ROUTED: for connector_type in _MCP_ROUTED:
assert ( assert (
CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS[connector_type] CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS[connector_type] == MCP_DISCOVERY_NAME
== MCP_DISCOVERY_NAME
), connector_type ), connector_type
def test_file_connectors_keep_native_routes(): def test_file_connectors_keep_native_routes():
assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["GOOGLE_DRIVE_CONNECTOR"] == "google_drive" assert (
CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["GOOGLE_DRIVE_CONNECTOR"]
== "google_drive"
)
assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["DROPBOX_CONNECTOR"] == "dropbox" assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["DROPBOX_CONNECTOR"] == "dropbox"
assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["ONEDRIVE_CONNECTOR"] == "onedrive" assert CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS["ONEDRIVE_CONNECTOR"] == "onedrive"
@ -87,7 +89,9 @@ def test_discovery_gating_is_any_of():
def test_discovery_gating_tokens_match_routed_connectors(): def test_discovery_gating_tokens_match_routed_connectors():
assert SUBAGENT_TO_REQUIRED_CONNECTOR_MAP[MCP_DISCOVERY_NAME] == frozenset(_MCP_ROUTED) assert SUBAGENT_TO_REQUIRED_CONNECTOR_MAP[MCP_DISCOVERY_NAME] == frozenset(
_MCP_ROUTED
)
def test_legacy_aliases_resolve_to_a_live_subagent(): def test_legacy_aliases_resolve_to_a_live_subagent():
@ -104,7 +108,12 @@ def test_collision_only_prefixes_shared_names():
_tool("search", {"mcp_connector_id": 2, "mcp_transport": "http"}), _tool("search", {"mcp_connector_id": 2, "mcp_transport": "http"}),
_tool("list_bases", {"mcp_connector_id": 2, "mcp_transport": "http"}), _tool("list_bases", {"mcp_connector_id": 2, "mcp_transport": "http"}),
] ]
resolved = {t.name: t for t in resolve_tool_name_collisions(tools, {1: "NOTION_CONNECTOR", 2: "AIRTABLE_CONNECTOR"})} resolved = {
t.name: t
for t in resolve_tool_name_collisions(
tools, {1: "NOTION_CONNECTOR", 2: "AIRTABLE_CONNECTOR"}
)
}
# The unique tool keeps its bare name (trusted_tools / history stay valid). # The unique tool keeps its bare name (trusted_tools / history stay valid).
assert "list_bases" in resolved assert "list_bases" in resolved

View file

@ -423,8 +423,12 @@ async def test_run_events_replays_buffer_then_finishes(monkeypatch):
row = _fake_run_row(status="running") row = _fake_run_row(status="running")
raw = str(row.id) raw = str(row.id)
run_event_bus.publish(raw, {"type": "run.progress", "phase": "scraping", "current": 1}) run_event_bus.publish(
run_event_bus.publish(raw, {"type": "run.finished", "status": "success", "item_count": 2}) raw, {"type": "run.progress", "phase": "scraping", "current": 1}
)
run_event_bus.publish(
raw, {"type": "run.finished", "status": "success", "item_count": 2}
)
app = _build_app_with_rows(monkeypatch, [row]) app = _build_app_with_rows(monkeypatch, [row])
try: try:

View file

@ -43,7 +43,9 @@ async def test_maps_urls_to_start_urls_and_wraps_items():
assert out.items[0].dataType == "post" assert out.items[0].dataType == "post"
(actor_input, _limit) = scraper.calls[0] (actor_input, _limit) = scraper.calls[0]
assert [u.url for u in actor_input.startUrls] == ["https://www.reddit.com/r/python/"] assert [u.url for u in actor_input.startUrls] == [
"https://www.reddit.com/r/python/"
]
assert actor_input.searches == [] assert actor_input.searches == []

View file

@ -233,7 +233,9 @@ async def test_gate_reserves_worst_case_captcha_when_solving_enabled(monkeypatch
with pytest.raises(InsufficientCreditsError): with pytest.raises(InsufficientCreditsError):
await gate_capability( await gate_capability(
CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session) CrawlInput(startUrls=["https://a.com"]),
BillingUnit.WEB_CRAWL,
_ctx(session),
) )

View file

@ -64,7 +64,9 @@ def test_preview_is_char_budgeted_and_references_run():
per_item = "y" * 500 per_item = "y" * 500
items = [f'{{"i": {i}, "v": "{per_item}"}}' for i in range(500)] items = [f'{{"i": {i}, "v": "{per_item}"}}' for i in range(500)]
body = "\n".join(items) body = "\n".join(items)
serialized = SerializedOutput(text=body, item_count=len(items), char_count=len(body)) serialized = SerializedOutput(
text=body, item_count=len(items), char_count=len(body)
)
preview = _build_preview(serialized, run_id="abc") preview = _build_preview(serialized, run_id="abc")
@ -143,8 +145,11 @@ async def test_read_run_paginates(monkeypatch):
read_run, _ = _tools() read_run, _ = _tools()
out = await read_run.ainvoke( out = await read_run.ainvoke(
{"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", {
"offset": 2, "limit": 3} "ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000",
"offset": 2,
"limit": 3,
}
) )
assert "item_2" in out and "item_3" in out and "item_4" in out assert "item_2" in out and "item_3" in out and "item_4" in out
assert "item_0" not in out and "item_5" not in out assert "item_0" not in out and "item_5" not in out
@ -184,8 +189,7 @@ async def test_search_run_excerpts_huge_matched_line(monkeypatch):
_, search_run = _tools() _, search_run = _tools()
out = await search_run.ainvoke( out = await search_run.ainvoke(
{"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", {"ref": "run_" + "0" * 8 + "-0000-0000-0000-000000000000", "pattern": "NEEDLE"}
"pattern": "NEEDLE"}
) )
assert "NEEDLE" in out assert "NEEDLE" in out
assert "match at char 100000" in out assert "match at char 100000" in out
@ -215,8 +219,10 @@ async def test_search_run_matches(monkeypatch):
_patch_session(monkeypatch, _BODY, []) _patch_session(monkeypatch, _BODY, [])
_, search_run = _tools() _, search_run = _tools()
out = await search_run.ainvoke( out = await search_run.ainvoke(
{"ref": "spill_" + "0" * 8 + "-0000-0000-0000-000000000000", {
"pattern": "item_7"} "ref": "spill_" + "0" * 8 + "-0000-0000-0000-000000000000",
"pattern": "item_7",
}
) )
assert "item_7" in out assert "item_7" in out
assert "item_1" not in out.split("item_7")[0] assert "item_1" not in out.split("item_7")[0]
@ -232,14 +238,30 @@ _CRAWL_BODY = "\n".join(
"url": "https://x.com/team/", "url": "https://x.com/team/",
"status": "success", "status": "success",
"links": [ "links": [
{"url": "https://x.com/author/jane/", "text": "Jane Doe", {
"context": "Jane Doe General Partner", "kind": "internal"}, "url": "https://x.com/author/jane/",
{"url": "https://x.com/author/bob/", "text": "Bob Roe", "text": "Jane Doe",
"context": "Bob Roe Operations", "kind": "internal"}, "context": "Jane Doe General Partner",
"kind": "internal",
},
{
"url": "https://x.com/author/bob/",
"text": "Bob Roe",
"context": "Bob Roe Operations",
"kind": "internal",
},
# Duplicate of Jane (nav + card) — must dedupe. # Duplicate of Jane (nav + card) — must dedupe.
{"url": "https://x.com/author/jane/", "text": "Jane Doe", {
"context": "Jane Doe General Partner", "kind": "internal"}, "url": "https://x.com/author/jane/",
{"url": "https://x.com/about/", "text": "About", "kind": "internal"}, "text": "Jane Doe",
"context": "Jane Doe General Partner",
"kind": "internal",
},
{
"url": "https://x.com/about/",
"text": "About",
"kind": "internal",
},
], ],
} }
), ),

View file

@ -101,7 +101,11 @@ async def test_aggregated_contacts_carry_provenance_and_site_wide_flag() -> None
class _ContactsEngine: class _ContactsEngine:
async def crawl_url(self, url: str) -> CrawlOutcome: async def crawl_url(self, url: str) -> CrawlOutcome:
socials = [footer] + ([person] if url.endswith("/about") else []) socials = [footer] + ([person] if url.endswith("/about") else [])
links = ["https://e.com/about", "https://e.com/blog"] if url == "https://e.com/" else [] links = (
["https://e.com/about", "https://e.com/blog"]
if url == "https://e.com/"
else []
)
return CrawlOutcome( return CrawlOutcome(
status=_SUCCESS, status=_SUCCESS,
result={ result={

View file

@ -43,18 +43,20 @@ def test_estimated_units_for_single_url_is_seed_count() -> None:
def test_estimated_units_for_spider_is_max_pages() -> None: def test_estimated_units_for_spider_is_max_pages() -> None:
model = CrawlInput( model = CrawlInput(startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25)
startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25
)
assert model.estimated_units == 25 assert model.estimated_units == 25
def test_billable_units_counts_only_successes() -> None: def test_billable_units_counts_only_successes() -> None:
out = CrawlOutput( out = CrawlOutput(
items=[ items=[
CrawlItem(url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)), CrawlItem(
url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)
),
CrawlItem(url="b", status="empty", crawl=CrawlMeta(loadedUrl="b", depth=1)), CrawlItem(url="b", status="empty", crawl=CrawlMeta(loadedUrl="b", depth=1)),
CrawlItem(url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)), CrawlItem(
url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)
),
] ]
) )
assert out.billable_units == 1 assert out.billable_units == 1

View file

@ -274,8 +274,13 @@ def test_parse_serp_extracts_all_blocks():
assert prod.prices == ["$24.99", "$30"] assert prod.prices == ["$24.99", "$30"]
# Related searches exclude the numeric pagination anchor (a.fl). # Related searches exclude the numeric pagination anchor (a.fl).
assert [r.title for r in item.relatedQueries] == ["easy apple pie", "apple pie recipe"] assert [r.title for r in item.relatedQueries] == [
assert item.relatedQueries[0].url == "https://www.google.com/search?q=easy+apple+pie" "easy apple pie",
"apple pie recipe",
]
assert (
item.relatedQueries[0].url == "https://www.google.com/search?q=easy+apple+pie"
)
# suggestedResults are the related queries re-shaped with type/position. # suggestedResults are the related queries re-shaped with type/position.
assert [(s.position, s.title, s.type) for s in item.suggestedResults] == [ assert [(s.position, s.title, s.type) for s in item.suggestedResults] == [

View file

@ -95,8 +95,15 @@ def test_flatten_comments_counts_replies_and_stops_at_more():
"kind": "Listing", "kind": "Listing",
"data": { "data": {
"children": [ "children": [
{"kind": "t1", "data": {"name": "t1_2", "id": "2", {
"body": "reply", "replies": ""}}, "kind": "t1",
"data": {
"name": "t1_2",
"id": "2",
"body": "reply",
"replies": "",
},
},
{"kind": "more", "data": {}}, # stub -> ignored {"kind": "more", "data": {}}, # stub -> ignored
] ]
}, },
@ -112,8 +119,10 @@ def test_flatten_comments_counts_replies_and_stops_at_more():
def test_flatten_comments_honors_max(): def test_flatten_comments_honors_max():
tree = [ tree = [
{"kind": "t1", "data": {"name": f"t1_{i}", "id": str(i), "body": "x", {
"replies": ""}} "kind": "t1",
"data": {"name": f"t1_{i}", "id": str(i), "body": "x", "replies": ""},
}
for i in range(5) for i in range(5)
] ]
assert len(flatten_comments(tree, max_comments=2)) == 2 assert len(flatten_comments(tree, max_comments=2)) == 2
@ -128,9 +137,17 @@ def test_children_and_after():
def test_parse_community_maps_members(): def test_parse_community_maps_members():
thing = {"kind": "t5", "data": {"name": "t5_s", "id": "s", thing = {
"display_name": "py", "display_name_prefixed": "r/py", "kind": "t5",
"subscribers": 1234, "url": "/r/py/"}} "data": {
"name": "t5_s",
"id": "s",
"display_name": "py",
"display_name_prefixed": "r/py",
"subscribers": 1234,
"url": "/r/py/",
},
}
item = parse_community(thing) item = parse_community(thing)
assert item["dataType"] == "community" assert item["dataType"] == "community"
assert item["numberOfMembers"] == 1234 assert item["numberOfMembers"] == 1234

View file

@ -125,9 +125,7 @@ async def test_dedupes_on_canonical_url() -> None:
} }
) )
await crawl_site( await crawl_site(engine, ["https://e.com/"], max_crawl_depth=3, max_crawl_pages=10)
engine, ["https://e.com/"], max_crawl_depth=3, max_crawl_pages=10
)
assert engine.calls.count("https://e.com/a") == 1 assert engine.calls.count("https://e.com/a") == 1
assert engine.calls.count("https://e.com/") == 1 assert engine.calls.count("https://e.com/") == 1

View file

@ -116,7 +116,12 @@ def test_extract_link_records_dedupes_keeping_first_nonempty_text() -> None:
html = '<a href="/p"><img src="logo.png"/></a><a href="/p">Pricing</a>' html = '<a href="/p"><img src="logo.png"/></a><a href="/p">Pricing</a>'
records = extract_link_records(html, "https://example.com/") records = extract_link_records(html, "https://example.com/")
assert records == [ assert records == [
{"url": "https://example.com/p", "text": "Pricing", "rel": "", "kind": "internal"} {
"url": "https://example.com/p",
"text": "Pricing",
"rel": "",
"kind": "internal",
}
] ]

View file

@ -241,5 +241,3 @@ class TestCheckBalance:
session, _user = _make_session(balance_micros=0) session, _user = _make_session(balance_micros=0)
await WebCrawlCreditService(session).check_balance(_USER_ID, 0) await WebCrawlCreditService(session).check_balance(_USER_ID, 0)
session.execute.assert_not_called() session.execute.assert_not_called()

View file

@ -47,9 +47,7 @@ def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None:
def test_no_country_suffix_yields_empty_location( def test_no_country_suffix_yields_empty_location(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
) -> None: ) -> None:
monkeypatch.setattr( monkeypatch.setattr(Config, "PROXY_URL", "http://tok:secret@gw.dataimpulse.com:823")
Config, "PROXY_URL", "http://tok:secret@gw.dataimpulse.com:823"
)
assert DataImpulseProvider().get_location() == "" assert DataImpulseProvider().get_location() == ""

View file

@ -33,6 +33,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -64,6 +67,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -95,6 +101,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -126,6 +135,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -157,6 +169,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -188,6 +203,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -219,6 +237,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -250,6 +271,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -281,6 +305,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -312,6 +339,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -353,6 +383,10 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -384,6 +418,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -415,6 +452,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -446,6 +486,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -477,6 +520,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -508,6 +554,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -539,6 +588,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -570,6 +622,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -601,6 +656,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -632,6 +690,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -663,6 +724,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -704,6 +768,10 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -735,6 +803,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -766,6 +837,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -797,6 +871,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -828,6 +905,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -859,6 +939,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -890,6 +973,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -921,6 +1007,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -952,6 +1041,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -983,6 +1075,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -1014,6 +1109,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -1055,6 +1153,10 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -1086,6 +1188,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -1147,6 +1252,12 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -1178,6 +1289,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -1209,6 +1323,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -1270,6 +1387,12 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -1331,6 +1454,12 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
@ -1362,6 +1491,9 @@ resolution-markers = [
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'", "python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_version < '0'",
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
] ]
conflicts = [[ conflicts = [[
@ -10221,7 +10353,7 @@ wheels = [
[[package]] [[package]]
name = "surf-new-backend" name = "surf-new-backend"
version = "0.0.30" version = "0.0.31"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [
{ name = "alembic" }, { name = "alembic" },

View file

@ -1,7 +1,7 @@
{ {
"name": "surfsense_browser_extension", "name": "surfsense_browser_extension",
"displayName": "Surfsense Browser Extension", "displayName": "Surfsense Browser Extension",
"version": "0.0.30", "version": "0.0.31",
"description": "Extension to collect Browsing History for SurfSense.", "description": "Extension to collect Browsing History for SurfSense.",
"author": "https://github.com/MODSetter", "author": "https://github.com/MODSetter",
"engines": { "engines": {

View file

@ -1,7 +1,7 @@
{ {
"name": "surfsense-desktop", "name": "surfsense-desktop",
"productName": "SurfSense", "productName": "SurfSense",
"version": "0.0.30", "version": "0.0.31",
"description": "SurfSense Desktop App", "description": "SurfSense Desktop App",
"main": "dist/main.js", "main": "dist/main.js",
"scripts": { "scripts": {

View file

@ -1,6 +1,6 @@
{ {
"name": "surfsense_web", "name": "surfsense_web",
"version": "0.0.30", "version": "0.0.31",
"private": true, "private": true,
"packageManager": "pnpm@10.26.0", "packageManager": "pnpm@10.26.0",
"description": "SurfSense Frontend", "description": "SurfSense Frontend",