apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-06-10 20:35:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: linting

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-06-09 00:42:26 -07:00
parent 0a012dbc79
commit ce952d2ad1
127 changed files with 821 additions and 517 deletions
Download patch file Download diff file Expand all files Collapse all files

4
surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py
View file

@ -165,9 +165,7 @@ def downgrade() -> None:
tx = conn.begin_nested() if conn.in_transaction() else conn.begin()
with tx:
conn.execute(
sa.text(
f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-148-downgrade'"
)
sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-148-downgrade'")
)
conn.execute(sa.text(ddl))
conn.execute(

81
surfsense_backend/alembic/versions/149_add_gateway_tables.py
View file

@ -65,6 +65,7 @@ AUTOMATION_RUN_COLS = [
"created_at",
]
def _has_zero_version(conn, table: str) -> bool:
return (
conn.execute(
@ -190,7 +191,8 @@ def upgrade() -> None:
"external_chat_peer_kind", ("direct", "group", "channel", "unknown")
)
external_chat_event_kind_enum = _create_enum(
"external_chat_event_kind", ("message", "edited_message", "callback_query", "other")
"external_chat_event_kind",
("message", "edited_message", "callback_query", "other"),
)
external_chat_event_status_enum = _create_enum(
"external_chat_event_status",
@ -205,7 +207,12 @@ def upgrade() -> None:
sa.Column("mode", external_chat_account_mode_enum, nullable=False),
sa.Column("owner_user_id", postgresql.UUID(as_uuid=True), nullable=True),
sa.Column("owner_search_space_id", sa.Integer(), nullable=True),
sa.Column("is_system_account", sa.Boolean(), nullable=False, server_default="false"),
sa.Column(
"is_system_account",
sa.Boolean(),
nullable=False,
server_default="false",
),
sa.Column("encrypted_credentials", sa.Text(), nullable=True),
sa.Column("bot_username", sa.String(255), nullable=True),
sa.Column("webhook_secret", sa.String(64), nullable=True),
@ -221,7 +228,9 @@ def upgrade() -> None:
nullable=False,
server_default="unknown",
),
sa.Column("last_health_check_at", sa.TIMESTAMP(timezone=True), nullable=True),
sa.Column(
"last_health_check_at", sa.TIMESTAMP(timezone=True), nullable=True
),
sa.Column("suspended_at", sa.TIMESTAMP(timezone=True), nullable=True),
sa.Column("suspended_reason", sa.Text(), nullable=True),
sa.Column(
@ -285,7 +294,9 @@ def upgrade() -> None:
server_default="pending",
),
sa.Column("pairing_code", sa.Text(), nullable=True),
sa.Column("pairing_code_expires_at", sa.TIMESTAMP(timezone=True), nullable=True),
sa.Column(
"pairing_code_expires_at", sa.TIMESTAMP(timezone=True), nullable=True
),
sa.Column("external_peer_id", sa.Text(), nullable=True),
sa.Column(
"external_peer_kind",
@ -327,7 +338,9 @@ def upgrade() -> None:
["account_id"], ["external_chat_accounts.id"], ondelete="CASCADE"
),
sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["search_space_id"], ["searchspaces.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(
["search_space_id"], ["searchspaces.id"], ondelete="CASCADE"
),
sa.ForeignKeyConstraint(
["new_chat_thread_id"], ["new_chat_threads.id"], ondelete="SET NULL"
),
@ -386,7 +399,9 @@ def upgrade() -> None:
nullable=False,
server_default="received",
),
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column(
"attempt_count", sa.Integer(), nullable=False, server_default="0"
),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column(
"received_at",
@ -405,7 +420,9 @@ def upgrade() -> None:
["account_id"], ["external_chat_accounts.id"], ondelete="CASCADE"
),
sa.ForeignKeyConstraint(
["external_chat_binding_id"], ["external_chat_bindings.id"], ondelete="SET NULL"
["external_chat_binding_id"],
["external_chat_bindings.id"],
ondelete="SET NULL",
),
sa.UniqueConstraint(
"account_id",
@ -445,7 +462,9 @@ def upgrade() -> None:
sa.Column("external_chat_binding_id", sa.BigInteger(), nullable=True),
)
if not _constraint_exists(
conn, "new_chat_threads", "fk_new_chat_threads_external_chat_external_chat_binding_id"
conn,
"new_chat_threads",
"fk_new_chat_threads_external_chat_external_chat_binding_id",
):
op.create_foreign_key(
"fk_new_chat_threads_external_chat_external_chat_binding_id",
@ -455,7 +474,9 @@ def upgrade() -> None:
["id"],
ondelete="SET NULL",
)
op.create_index("ix_new_chat_threads_source", "new_chat_threads", ["source"], if_not_exists=True)
op.create_index(
"ix_new_chat_threads_source", "new_chat_threads", ["source"], if_not_exists=True
)
op.create_index(
"ix_new_chat_threads_external_chat_binding_id",
"new_chat_threads",
@ -472,7 +493,11 @@ def upgrade() -> None:
if not _column_exists(conn, "new_chat_messages", "platform_metadata"):
op.add_column(
"new_chat_messages",
sa.Column("platform_metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column(
"platform_metadata",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
),
)
op.create_index(
"ix_new_chat_messages_source",
@ -553,11 +578,15 @@ def downgrade() -> None:
tx = conn.begin_nested() if conn.in_transaction() else conn.begin()
with tx:
conn.execute(
sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-144-downgrade'")
sa.text(
f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-144-downgrade'"
)
)
conn.execute(sa.text(ddl))
conn.execute(
sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-144-downgrade'")
sa.text(
f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-144-downgrade'"
)
)
if _column_exists(conn, "new_chat_messages", "source"):
@ -567,10 +596,14 @@ def downgrade() -> None:
_drop_column_if_exists("new_chat_messages", "platform_metadata")
_drop_column_if_exists("new_chat_messages", "source")
_drop_index_if_exists("ix_new_chat_threads_external_chat_binding_id", "new_chat_threads")
_drop_index_if_exists(
"ix_new_chat_threads_external_chat_binding_id", "new_chat_threads"
)
_drop_index_if_exists("ix_new_chat_threads_source", "new_chat_threads")
if _constraint_exists(
conn, "new_chat_threads", "fk_new_chat_threads_external_chat_external_chat_binding_id"
conn,
"new_chat_threads",
"fk_new_chat_threads_external_chat_external_chat_binding_id",
):
op.drop_constraint(
"fk_new_chat_threads_external_chat_external_chat_binding_id",
@ -583,8 +616,12 @@ def downgrade() -> None:
_drop_index_if_exists(
"ix_external_chat_inbound_binding_received_at", "external_chat_inbound_events"
)
_drop_index_if_exists("ix_external_chat_inbound_request_id", "external_chat_inbound_events")
_drop_index_if_exists("ix_external_chat_inbound_status_received_at", "external_chat_inbound_events")
_drop_index_if_exists(
"ix_external_chat_inbound_request_id", "external_chat_inbound_events"
)
_drop_index_if_exists(
"ix_external_chat_inbound_status_received_at", "external_chat_inbound_events"
)
if _table_exists(conn, "external_chat_inbound_events"):
op.drop_table("external_chat_inbound_events")
@ -606,9 +643,15 @@ def downgrade() -> None:
if _table_exists(conn, "external_chat_bindings"):
op.drop_table("external_chat_bindings")
_drop_index_if_exists("uq_external_chat_accounts_system_platform", "external_chat_accounts")
_drop_index_if_exists("uq_external_chat_accounts_owner_platform", "external_chat_accounts")
_drop_index_if_exists("uq_external_chat_accounts_webhook_secret", "external_chat_accounts")
_drop_index_if_exists(
"uq_external_chat_accounts_system_platform", "external_chat_accounts"
)
_drop_index_if_exists(
"uq_external_chat_accounts_owner_platform", "external_chat_accounts"
)
_drop_index_if_exists(
"uq_external_chat_accounts_webhook_secret", "external_chat_accounts"
)
if _table_exists(conn, "external_chat_accounts"):
op.drop_table("external_chat_accounts")

3
surfsense_backend/alembic/versions/152_add_document_files.py
View file

@ -63,8 +63,7 @@ def upgrade() -> None:
"ON document_files(search_space_id);"
)
op.execute(
"CREATE INDEX IF NOT EXISTS ix_document_files_kind "
"ON document_files(kind);"
"CREATE INDEX IF NOT EXISTS ix_document_files_kind ON document_files(kind);"
)
op.execute(
"CREATE INDEX IF NOT EXISTS ix_document_files_created_by_id "

22
surfsense_backend/alembic/versions/153_restore_automation_runs_to_zero_publication.py
View file

@ -68,8 +68,12 @@ def _has_zero_version(conn, table: str) -> bool:
def _set_table_ddl(*, with_automation_runs: bool, conn) -> str:
doc_cols = DOCUMENT_COLS + (['"_0_version"'] if _has_zero_version(conn, "documents") else [])
user_cols = USER_COLS + (['"_0_version"'] if _has_zero_version(conn, "user") else [])
doc_cols = DOCUMENT_COLS + (
['"_0_version"'] if _has_zero_version(conn, "documents") else []
)
user_cols = USER_COLS + (
['"_0_version"'] if _has_zero_version(conn, "user") else []
)
tables = [
"notifications",
f"documents ({', '.join(doc_cols)})",
@ -96,9 +100,17 @@ def _resync(*, with_automation_runs: bool, tag: str) -> None:
tx = conn.begin_nested() if conn.in_transaction() else conn.begin()
with tx:
conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'"))
conn.execute(sa.text(_set_table_ddl(with_automation_runs=with_automation_runs, conn=conn)))
conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'"))
conn.execute(
sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'")
)
conn.execute(
sa.text(
_set_table_ddl(with_automation_runs=with_automation_runs, conn=conn)
)
)
conn.execute(
sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'")
)
def upgrade() -> None:

23
surfsense_backend/alembic/versions/154_remove_document_summary_llm.py
View file

@ -67,8 +67,12 @@ def _has_zero_version(conn, table: str) -> bool:
def _set_table_ddl(conn) -> str:
doc_cols = DOCUMENT_COLS + (['"_0_version"'] if _has_zero_version(conn, "documents") else [])
user_cols = USER_COLS + (['"_0_version"'] if _has_zero_version(conn, "user") else [])
doc_cols = DOCUMENT_COLS + (
['"_0_version"'] if _has_zero_version(conn, "documents") else []
)
user_cols = USER_COLS + (
['"_0_version"'] if _has_zero_version(conn, "user") else []
)
tables = [
"notifications",
f"documents ({', '.join(doc_cols)})",
@ -94,9 +98,13 @@ def _resync_zero_publication(tag: str) -> None:
tx = conn.begin_nested() if conn.in_transaction() else conn.begin()
with tx:
conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'"))
conn.execute(
sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'")
)
conn.execute(sa.text(_set_table_ddl(conn)))
conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'"))
conn.execute(
sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'")
)
def upgrade() -> None:
@ -117,7 +125,12 @@ def downgrade() -> None:
if not _column_exists(conn, "searchspaces", "document_summary_llm_id"):
op.add_column(
"searchspaces",
sa.Column("document_summary_llm_id", sa.Integer(), nullable=True, server_default="0"),
sa.Column(
"document_summary_llm_id",
sa.Integer(),
nullable=True,
server_default="0",
),
)
if not _column_exists(conn, "search_source_connectors", "enable_summary"):

179
surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py
View file

@ -40,44 +40,149 @@ class ToolMetadata:
# up in the UI tool picker. This list carries metadata only — wire the actual
# implementation in the relevant builder/registry module.
TOOL_CATALOG: list[ToolMetadata] = [
ToolMetadata(name="generate_podcast", description="Generate an audio podcast from provided content"),
ToolMetadata(name="generate_video_presentation", description="Generate a video presentation with slides and narration from provided content"),
ToolMetadata(name="generate_report", description="Generate a structured report from provided content and export it"),
ToolMetadata(name="generate_resume", description="Generate a professional resume as a Typst document"),
ToolMetadata(name="generate_image", description="Generate images from text descriptions using AI image models"),
ToolMetadata(name="scrape_webpage", description="Scrape and extract the main content from a webpage"),
ToolMetadata(name="web_search", description="Search the web for real-time information using configured search engines"),
ToolMetadata(name="create_automation", description="Draft an automation from an NL intent; user approves the card; tool saves"),
ToolMetadata(name="update_memory", description="Save important long-term facts, preferences, and instructions to the (personal or team) memory"),
ToolMetadata(name="create_notion_page", description="Create a new page in the user's Notion workspace"),
ToolMetadata(name="update_notion_page", description="Append new content to an existing Notion page"),
ToolMetadata(name="delete_notion_page", description="Delete an existing Notion page"),
ToolMetadata(name="create_google_drive_file", description="Create a new Google Doc or Google Sheet in Google Drive"),
ToolMetadata(name="delete_google_drive_file", description="Move an indexed Google Drive file to trash"),
ToolMetadata(name="create_dropbox_file", description="Create a new file in Dropbox"),
ToolMetadata(
name="generate_podcast",
description="Generate an audio podcast from provided content",
),
ToolMetadata(
name="generate_video_presentation",
description="Generate a video presentation with slides and narration from provided content",
),
ToolMetadata(
name="generate_report",
description="Generate a structured report from provided content and export it",
),
ToolMetadata(
name="generate_resume",
description="Generate a professional resume as a Typst document",
),
ToolMetadata(
name="generate_image",
description="Generate images from text descriptions using AI image models",
),
ToolMetadata(
name="scrape_webpage",
description="Scrape and extract the main content from a webpage",
),
ToolMetadata(
name="web_search",
description="Search the web for real-time information using configured search engines",
),
ToolMetadata(
name="create_automation",
description="Draft an automation from an NL intent; user approves the card; tool saves",
),
ToolMetadata(
name="update_memory",
description="Save important long-term facts, preferences, and instructions to the (personal or team) memory",
),
ToolMetadata(
name="create_notion_page",
description="Create a new page in the user's Notion workspace",
),
ToolMetadata(
name="update_notion_page",
description="Append new content to an existing Notion page",
),
ToolMetadata(
name="delete_notion_page", description="Delete an existing Notion page"
),
ToolMetadata(
name="create_google_drive_file",
description="Create a new Google Doc or Google Sheet in Google Drive",
),
ToolMetadata(
name="delete_google_drive_file",
description="Move an indexed Google Drive file to trash",
),
ToolMetadata(
name="create_dropbox_file", description="Create a new file in Dropbox"
),
ToolMetadata(name="delete_dropbox_file", description="Delete a file from Dropbox"),
ToolMetadata(name="create_onedrive_file", description="Create a new file in Microsoft OneDrive"),
ToolMetadata(name="delete_onedrive_file", description="Move a OneDrive file to the recycle bin"),
ToolMetadata(name="search_calendar_events", description="Search Google Calendar events within a date range"),
ToolMetadata(name="create_calendar_event", description="Create a new event on Google Calendar"),
ToolMetadata(name="update_calendar_event", description="Update an existing indexed Google Calendar event"),
ToolMetadata(name="delete_calendar_event", description="Delete an existing indexed Google Calendar event"),
ToolMetadata(name="search_gmail", description="Search emails in Gmail using Gmail search syntax"),
ToolMetadata(name="read_gmail_email", description="Read the full content of a specific Gmail email"),
ToolMetadata(name="create_gmail_draft", description="Create a draft email in Gmail"),
ToolMetadata(
name="create_onedrive_file",
description="Create a new file in Microsoft OneDrive",
),
ToolMetadata(
name="delete_onedrive_file",
description="Move a OneDrive file to the recycle bin",
),
ToolMetadata(
name="search_calendar_events",
description="Search Google Calendar events within a date range",
),
ToolMetadata(
name="create_calendar_event",
description="Create a new event on Google Calendar",
),
ToolMetadata(
name="update_calendar_event",
description="Update an existing indexed Google Calendar event",
),
ToolMetadata(
name="delete_calendar_event",
description="Delete an existing indexed Google Calendar event",
),
ToolMetadata(
name="search_gmail",
description="Search emails in Gmail using Gmail search syntax",
),
ToolMetadata(
name="read_gmail_email",
description="Read the full content of a specific Gmail email",
),
ToolMetadata(
name="create_gmail_draft", description="Create a draft email in Gmail"
),
ToolMetadata(name="send_gmail_email", description="Send an email via Gmail"),
ToolMetadata(name="trash_gmail_email", description="Move an indexed email to trash in Gmail"),
ToolMetadata(name="update_gmail_draft", description="Update an existing Gmail draft"),
ToolMetadata(name="create_confluence_page", description="Create a new page in the user's Confluence space"),
ToolMetadata(name="update_confluence_page", description="Update an existing indexed Confluence page"),
ToolMetadata(name="delete_confluence_page", description="Delete an existing indexed Confluence page"),
ToolMetadata(name="list_discord_channels", description="List text channels in the connected Discord server"),
ToolMetadata(name="read_discord_messages", description="Read recent messages from a Discord text channel"),
ToolMetadata(name="send_discord_message", description="Send a message to a Discord text channel"),
ToolMetadata(name="list_teams_channels", description="List Microsoft Teams and their channels"),
ToolMetadata(name="read_teams_messages", description="Read recent messages from a Microsoft Teams channel"),
ToolMetadata(name="send_teams_message", description="Send a message to a Microsoft Teams channel"),
ToolMetadata(name="list_luma_events", description="List upcoming and recent Luma events"),
ToolMetadata(name="read_luma_event", description="Read detailed information about a specific Luma event"),
ToolMetadata(
name="trash_gmail_email", description="Move an indexed email to trash in Gmail"
),
ToolMetadata(
name="update_gmail_draft", description="Update an existing Gmail draft"
),
ToolMetadata(
name="create_confluence_page",
description="Create a new page in the user's Confluence space",
),
ToolMetadata(
name="update_confluence_page",
description="Update an existing indexed Confluence page",
),
ToolMetadata(
name="delete_confluence_page",
description="Delete an existing indexed Confluence page",
),
ToolMetadata(
name="list_discord_channels",
description="List text channels in the connected Discord server",
),
ToolMetadata(
name="read_discord_messages",
description="Read recent messages from a Discord text channel",
),
ToolMetadata(
name="send_discord_message",
description="Send a message to a Discord text channel",
),
ToolMetadata(
name="list_teams_channels",
description="List Microsoft Teams and their channels",
),
ToolMetadata(
name="read_teams_messages",
description="Read recent messages from a Microsoft Teams channel",
),
ToolMetadata(
name="send_teams_message",
description="Send a message to a Microsoft Teams channel",
),
ToolMetadata(
name="list_luma_events", description="List upcoming and recent Luma events"
),
ToolMetadata(
name="read_luma_event",
description="Read detailed information about a specific Luma event",
),
ToolMetadata(name="create_luma_event", description="Create a new event on Luma"),
]

16
surfsense_backend/app/config/__init__.py
View file

@ -308,12 +308,8 @@ def load_openrouter_integration_settings() -> dict | None:
"anonymous_enabled_free instead. Both new flags have been "
"seeded from the legacy value for back-compat."
)
settings.setdefault(
"anonymous_enabled_paid", settings["anonymous_enabled"]
)
settings.setdefault(
"anonymous_enabled_free", settings["anonymous_enabled"]
)
settings.setdefault("anonymous_enabled_paid", settings["anonymous_enabled"])
settings.setdefault("anonymous_enabled_free", settings["anonymous_enabled"])
# Image generation + vision LLM emission are opt-in (issue L).
# OpenRouter's catalogue contains hundreds of image / vision
@ -622,7 +618,9 @@ class Config:
WHATSAPP_GRAPH_API_VERSION = os.getenv("WHATSAPP_GRAPH_API_VERSION", "v25.0")
WHATSAPP_WEBHOOK_VERIFY_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFY_TOKEN")
WHATSAPP_WEBHOOK_APP_SECRET = os.getenv("WHATSAPP_WEBHOOK_APP_SECRET")
WHATSAPP_BRIDGE_URL = os.getenv("WHATSAPP_BRIDGE_URL", "http://whatsapp-bridge:9929")
WHATSAPP_BRIDGE_URL = os.getenv(
"WHATSAPP_BRIDGE_URL", "http://whatsapp-bridge:9929"
)
GATEWAY_WHATSAPP_INTAKE_MODE = os.getenv(
"GATEWAY_WHATSAPP_INTAKE_MODE", "disabled"
).lower()
@ -632,7 +630,9 @@ class Config:
)
GATEWAY_SLACK_CLIENT_ID = os.getenv("SLACK_CLIENT_ID")
GATEWAY_SLACK_CLIENT_SECRET = os.getenv("SLACK_CLIENT_SECRET")
GATEWAY_SLACK_ENABLED = os.getenv("GATEWAY_SLACK_ENABLED", "FALSE").upper() == "TRUE"
GATEWAY_SLACK_ENABLED = (
os.getenv("GATEWAY_SLACK_ENABLED", "FALSE").upper() == "TRUE"
)
GATEWAY_SLACK_SIGNING_SECRET = os.getenv("GATEWAY_SLACK_SIGNING_SECRET")
GATEWAY_SLACK_REDIRECT_URI = os.getenv("GATEWAY_SLACK_REDIRECT_URI")
GATEWAY_DISCORD_ENABLED = (

20
surfsense_backend/app/connectors/webcrawler_connector.py
View file

@ -105,14 +105,18 @@ class WebCrawlerConnector:
logger.info(f"[webcrawler] Using Scrapling AsyncFetcher for: {url}")
result = await self._crawl_with_async_fetcher(url)
if result:
self._log_tier_outcome("scrapling-static", url, tier_start, "success")
self._log_tier_outcome(
"scrapling-static", url, tier_start, "success"
)
self._log_total(url, "scrapling-static", total_start)
return result, None
errors.append("Scrapling static: empty extraction")
self._log_tier_outcome("scrapling-static", url, tier_start, "empty")
except Exception as exc:
errors.append(f"Scrapling static: {exc!s}")
self._log_tier_outcome("scrapling-static", url, tier_start, "error", exc)
self._log_tier_outcome(
"scrapling-static", url, tier_start, "error", exc
)
# --- 3. Scrapling DynamicFetcher (full browser) ---
tier_start = time.perf_counter()
@ -120,7 +124,9 @@ class WebCrawlerConnector:
logger.info(f"[webcrawler] Using Scrapling DynamicFetcher for: {url}")
result = await self._crawl_with_dynamic(url)
if result:
self._log_tier_outcome("scrapling-dynamic", url, tier_start, "success")
self._log_tier_outcome(
"scrapling-dynamic", url, tier_start, "success"
)
self._log_total(url, "scrapling-dynamic", total_start)
return result, None
errors.append("Scrapling dynamic: empty extraction")
@ -135,7 +141,9 @@ class WebCrawlerConnector:
)
except Exception as exc:
errors.append(f"Scrapling dynamic: {exc!s}")
self._log_tier_outcome("scrapling-dynamic", url, tier_start, "error", exc)
self._log_tier_outcome(
"scrapling-dynamic", url, tier_start, "error", exc
)
# --- 4. Scrapling StealthyFetcher (anti-bot, last resort) ---
tier_start = time.perf_counter()
@ -143,7 +151,9 @@ class WebCrawlerConnector:
logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}")
result = await self._crawl_with_stealthy(url)
if result:
self._log_tier_outcome("scrapling-stealthy", url, tier_start, "success")
self._log_tier_outcome(
"scrapling-stealthy", url, tier_start, "success"
)
self._log_total(url, "scrapling-stealthy", total_start)
return result, None
errors.append("Scrapling stealthy: empty extraction")

28
surfsense_backend/app/db.py
View file

@ -714,7 +714,9 @@ class NewChatThread(BaseModel, TimestampMixin):
# Surface metadata for first-party SurfSense and external chat threads.
# Zero publishes all chat-message sources; the UI can decide which surfaces to render.
source = Column(Text, nullable=False, default="surfsense", server_default="surfsense")
source = Column(
Text, nullable=False, default="surfsense", server_default="surfsense"
)
external_chat_binding_id = Column(
BigInteger,
ForeignKey("external_chat_bindings.id", ondelete="SET NULL"),
@ -802,7 +804,9 @@ class NewChatMessage(BaseModel, TimestampMixin):
# Mirrors the parent thread source for publication-level filtering.
# This denormalization avoids join-dependent logical replication rules.
source = Column(Text, nullable=False, default="surfsense", server_default="surfsense")
source = Column(
Text, nullable=False, default="surfsense", server_default="surfsense"
)
platform_metadata = Column(JSONB, nullable=True)
# Relationships
@ -848,11 +852,15 @@ class ExternalChatAccount(Base, TimestampMixin):
owner_search_space_id = Column(
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=True
)
is_system_account = Column(Boolean, nullable=False, default=False, server_default="false")
is_system_account = Column(
Boolean, nullable=False, default=False, server_default="false"
)
encrypted_credentials = Column(Text, nullable=True)
bot_username = Column(String(255), nullable=True)
webhook_secret = Column(String(64), nullable=True)
cursor_state = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb"))
cursor_state = Column(
JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")
)
health_status = Column(
SQLAlchemyEnum(
ExternalChatHealthStatus,
@ -875,7 +883,9 @@ class ExternalChatAccount(Base, TimestampMixin):
)
owner = relationship("User", foreign_keys=[owner_user_id])
owner_search_space = relationship("SearchSpace", foreign_keys=[owner_search_space_id])
owner_search_space = relationship(
"SearchSpace", foreign_keys=[owner_search_space_id]
)
bindings = relationship(
"ExternalChatBinding",
back_populates="account",
@ -980,7 +990,9 @@ class ExternalChatBinding(Base, TimestampMixin):
external_thread_id = Column(Text, nullable=True)
external_display_name = Column(Text, nullable=True)
external_username = Column(Text, nullable=True)
external_metadata = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb"))
external_metadata = Column(
JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")
)
new_chat_thread_id = Column(
Integer,
ForeignKey("new_chat_threads.id", ondelete="SET NULL"),
@ -1030,7 +1042,9 @@ class ExternalChatBinding(Base, TimestampMixin):
postgresql_where=text("state = 'pending'"),
),
Index("ix_external_chat_bindings_user_state", "user_id", "state"),
Index("ix_external_chat_bindings_search_space_state", "search_space_id", "state"),
Index(
"ix_external_chat_bindings_search_space_state", "search_space_id", "state"
),
)

5
surfsense_backend/app/file_storage/backends/azure.py
View file

@ -2,6 +2,7 @@
from __future__ import annotations
import contextlib
from collections.abc import AsyncIterator
from app.file_storage.backends.base import StorageBackend
@ -43,10 +44,8 @@ class AzureBlobBackend(StorageBackend):
async with self._service() as service:
blob = service.get_blob_client(self._container, key)
try:
with contextlib.suppress(ResourceNotFoundError):
await blob.delete_blob()
except ResourceNotFoundError:
pass
async def exists(self, key: str) -> bool:
async with self._service() as service:

5
surfsense_backend/app/file_storage/backends/local.py
View file

@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
import contextlib
from collections.abc import AsyncIterator
from pathlib import Path
@ -53,10 +54,8 @@ class LocalFileBackend(StorageBackend):
path = self._path_for(key)
def _unlink() -> None:
try:
with contextlib.suppress(FileNotFoundError):
path.unlink()
except FileNotFoundError:
pass
await asyncio.to_thread(_unlink)

9
surfsense_backend/app/gateway/accounts.py
View file

@ -31,7 +31,9 @@ def slack_account_credentials(account: ExternalChatAccount) -> dict:
"""Decrypt Slack gateway credentials stored as encrypted JSON."""
if not account.encrypted_credentials:
return {}
raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token(account.encrypted_credentials)
raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token(
account.encrypted_credentials
)
try:
data = json.loads(raw)
except json.JSONDecodeError:
@ -44,7 +46,9 @@ def discord_account_credentials(account: ExternalChatAccount) -> dict:
"""Decrypt Discord gateway credentials stored as encrypted JSON."""
if not account.encrypted_credentials:
return {}
raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token(account.encrypted_credentials)
raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token(
account.encrypted_credentials
)
try:
data = json.loads(raw)
except json.JSONDecodeError:
@ -135,4 +139,3 @@ async def get_discord_account_by_guild(
)
)
return result.scalars().first()

5
surfsense_backend/app/gateway/agent_invoke.py
View file

@ -21,7 +21,9 @@ from app.tasks.chat.streaming.flows import stream_new_chat
logger = logging.getLogger(__name__)
async def _events_from_sse(chunks: AsyncIterator[str]) -> AsyncIterator[GatewayStreamEvent]:
async def _events_from_sse(
chunks: AsyncIterator[str],
) -> AsyncIterator[GatewayStreamEvent]:
saw_text = False
async for chunk in chunks:
for raw_line in chunk.splitlines():
@ -98,4 +100,3 @@ async def call_agent_for_gateway(
record_gateway_turn_latency(0, platform=platform_label)
finally:
release_thread_lock(thread.id)

1
surfsense_backend/app/gateway/auth_invariant.py
View file

@ -52,4 +52,3 @@ async def assert_authorization_invariant(
await _fail(session, binding, f"rbac_{exc.status_code}")
return user

1
surfsense_backend/app/gateway/base/__init__.py
View file

@ -1,2 +1 @@
"""Base gateway interfaces."""

5
surfsense_backend/app/gateway/base/adapter.py
View file

@ -62,9 +62,10 @@ class BasePlatformAdapter(ABC):
async def validate_credentials(self) -> dict[str, Any]:
"""Validate configured credentials and return account metadata."""
async def fetch_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]:
async def fetch_updates(
self, *, offset: int | None
) -> AsyncIterator[dict[str, Any]]:
"""Yield provider updates for long-polling adapters."""
if False:
yield {} # pragma: no cover
raise NotImplementedError("This adapter does not support long-polling")

1
surfsense_backend/app/gateway/base/identity.py
View file

@ -16,4 +16,3 @@ def hash_external_id(value: str | int | None) -> str | None:
if not normalized:
return None
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()

1
surfsense_backend/app/gateway/base/translator.py
View file

@ -25,4 +25,3 @@ class BaseStreamTranslator(ABC):
@abstractmethod
async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None:
"""Consume agent stream events and emit platform messages."""

1
surfsense_backend/app/gateway/bindings.py
View file

@ -64,4 +64,3 @@ def resume_binding(binding: ExternalChatBinding) -> None:
binding.state = ExternalChatBindingState.BOUND
binding.suspended_at = None
binding.suspended_reason = None

19
surfsense_backend/app/gateway/byo_long_poll.py
View file

@ -58,8 +58,10 @@ async def _whatsapp_baileys_supervisor() -> None:
async with async_session_maker() as session:
result = await session.execute(
select(ExternalChatAccount).where(
ExternalChatAccount.platform == ExternalChatPlatform.WHATSAPP,
ExternalChatAccount.mode == ExternalChatAccountMode.SELF_HOST_BYO,
ExternalChatAccount.platform
== ExternalChatPlatform.WHATSAPP,
ExternalChatAccount.mode
== ExternalChatAccountMode.SELF_HOST_BYO,
ExternalChatAccount.is_system_account.is_(False),
ExternalChatAccount.suspended_at.is_(None),
)
@ -128,7 +130,9 @@ async def start_byo_long_poll_supervisors() -> None:
)
_tasks.add(task)
task.add_done_callback(_tasks.discard)
logger.info("Started BYO Telegram long-poll supervisor account_id=%s", account.id)
logger.info(
"Started BYO Telegram long-poll supervisor account_id=%s", account.id
)
if config.GATEWAY_WHATSAPP_INTAKE_MODE == "baileys":
task = asyncio.create_task(
@ -151,9 +155,12 @@ async def stop_byo_long_poll_supervisors() -> None:
task.cancel()
if tasks:
try:
await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=10)
await asyncio.wait_for(
asyncio.gather(*tasks, return_exceptions=True), timeout=10
)
except TimeoutError:
logger.warning("Timed out waiting for BYO Telegram long-poll supervisors to stop")
logger.warning(
"Timed out waiting for BYO Telegram long-poll supervisors to stop"
)
_tasks.clear()
_shutdown_event = None

19
surfsense_backend/app/gateway/discord/intake.py
View file

@ -39,7 +39,9 @@ def _message_reference_payload(message: discord.Message) -> dict[str, Any] | Non
}
def _serialize_message(message: discord.Message, *, bot_user_id: str | None) -> dict[str, Any]:
def _serialize_message(
message: discord.Message, *, bot_user_id: str | None
) -> dict[str, Any]:
guild = message.guild
channel = message.channel
thread_id = str(channel.id) if isinstance(channel, discord.Thread) else None
@ -62,8 +64,7 @@ def _serialize_message(message: discord.Message, *, bot_user_id: str | None) ->
"bot": message.author.bot,
},
"mentions": [
{"id": str(user.id), "username": user.name}
for user in message.mentions
{"id": str(user.id), "username": user.name} for user in message.mentions
],
"message_reference": _message_reference_payload(message),
"created_at": message.created_at.isoformat()
@ -73,7 +74,9 @@ def _serialize_message(message: discord.Message, *, bot_user_id: str | None) ->
}
async def _persist_message(message: discord.Message, *, bot_user_id: str | None) -> None:
async def _persist_message(
message: discord.Message, *, bot_user_id: str | None
) -> None:
if message.guild is None:
return
guild_id = str(message.guild.id)
@ -82,7 +85,9 @@ async def _persist_message(message: discord.Message, *, bot_user_id: str | None)
async with async_session_maker() as session:
account = await get_discord_account_by_guild(session, guild_id=guild_id)
if account is None:
logger.info("Ignoring Discord message for uninstalled guild_id=%s", guild_id)
logger.info(
"Ignoring Discord message for uninstalled guild_id=%s", guild_id
)
return
inbox_id = await persist_inbound_event(
@ -144,7 +149,9 @@ def _build_client() -> discord.Client:
try:
await _persist_message(message, bot_user_id=bot_user_id)
except Exception:
logger.exception("Discord gateway failed to persist message_id=%s", message.id)
logger.exception(
"Discord gateway failed to persist message_id=%s", message.id
)
return client

8
surfsense_backend/app/gateway/discord/translator.py
View file

@ -41,7 +41,9 @@ class DiscordStreamTranslator(BaseStreamTranslator):
async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None:
async for event in events:
if event.type in {"text-delta", "text_delta", "text"}:
self._buffer += str(event.data.get("text") or event.data.get("delta") or "")
self._buffer += str(
event.data.get("text") or event.data.get("delta") or ""
)
elif event.type in {"data-interrupt-request", "interrupt"}:
await self._handle_hitl_interrupt()
return
@ -53,7 +55,9 @@ class DiscordStreamTranslator(BaseStreamTranslator):
async def _flush_final(self) -> None:
if not self._buffer:
return
for chunk in split_text_message(self._buffer, max_chars=DISCORD_MAX_MESSAGE_CHARS):
for chunk in split_text_message(
self._buffer, max_chars=DISCORD_MAX_MESSAGE_CHARS
):
await self._send_text(chunk)
async def _send_text(self, text: str) -> PlatformSendResult:

1
surfsense_backend/app/gateway/hitl_filter.py
View file

@ -32,4 +32,3 @@ def filter_hitl_tools(
return None
blocked = blocked_names or DEFAULT_HITL_TOOL_NAMES
return [tool for tool in toolkit if (_tool_name(tool) or "") not in blocked]

1
surfsense_backend/app/gateway/inbox.py
View file

@ -51,4 +51,3 @@ async def persist_inbound_event(
)
result = await session.execute(stmt)
return result.scalar_one_or_none()

22
surfsense_backend/app/gateway/inbox_processor.py
View file

@ -128,7 +128,9 @@ async def process_inbound_event(
event.status = ExternalChatEventStatus.PROCESSED
event.processed_at = datetime.now(UTC)
await session.commit()
record_gateway_inbox_processed(platform=event.platform.value, status="processed")
record_gateway_inbox_processed(
platform=event.platform.value, status="processed"
)
async def _mark_failed(
@ -173,7 +175,9 @@ async def _resolve_slack_thread_binding(
parsed,
) -> ExternalChatBinding | None:
user_peer_id = parsed.metadata.get("slack_user_peer_id")
thread_peer_id = parsed.metadata.get("slack_thread_peer_id") or parsed.external_peer_id
thread_peer_id = (
parsed.metadata.get("slack_thread_peer_id") or parsed.external_peer_id
)
if not user_peer_id or not thread_peer_id:
return None
@ -233,7 +237,9 @@ async def _resolve_discord_thread_binding(
parsed,
) -> ExternalChatBinding | None:
user_peer_id = parsed.metadata.get("discord_user_peer_id")
thread_peer_id = parsed.metadata.get("discord_thread_peer_id") or parsed.external_peer_id
thread_peer_id = (
parsed.metadata.get("discord_thread_peer_id") or parsed.external_peer_id
)
if not user_peer_id or not thread_peer_id:
return None
@ -357,7 +363,11 @@ async def _dispatch_inbound_event(
return
if binding is None:
if bundle.auto_bind_owner and account.owner_user_id and account.owner_search_space_id:
if (
bundle.auto_bind_owner
and account.owner_user_id
and account.owner_search_space_id
):
binding = ExternalChatBinding(
account_id=account.id,
user_id=account.owner_user_id,
@ -385,7 +395,9 @@ async def _dispatch_inbound_event(
event.external_chat_binding_id = binding.id
if cmd == "/help":
handled = await bundle.commands.handle_help_command(adapter=adapter, event=parsed)
handled = await bundle.commands.handle_help_command(
adapter=adapter, event=parsed
)
if handled:
event.status = ExternalChatEventStatus.PROCESSED
await session.commit()

1
surfsense_backend/app/gateway/inbox_worker.py
View file

@ -55,4 +55,3 @@ async def stop_gateway_inbox_worker() -> None:
with suppress(TimeoutError, asyncio.CancelledError):
await asyncio.wait_for(_task, timeout=10)
_task = None

3
surfsense_backend/app/gateway/pairing.py
View file

@ -8,7 +8,7 @@ from datetime import UTC, datetime, timedelta
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import ExternalChatBindingState, ExternalChatBinding
from app.db import ExternalChatBinding, ExternalChatBindingState
PAIRING_CODE_TTL = timedelta(minutes=10)
@ -51,4 +51,3 @@ async def redeem_pairing_code(
binding.external_username = external_username
binding.external_metadata = external_metadata or {}
return binding

1
surfsense_backend/app/gateway/ratelimit.py
View file

@ -133,4 +133,3 @@ async def wait_for_token(
if wait_ms > 0:
await asyncio.sleep(wait_ms / 1000)
return wait_ms

4
surfsense_backend/app/gateway/registry.py
View file

@ -186,4 +186,6 @@ def resolve_platform_bundle(account: ExternalChatAccount) -> PlatformBundle:
auto_bind_owner=False,
)
raise RuntimeError(f"unsupported_gateway_platform:{account.platform.value}:{account.mode.value}")
raise RuntimeError(
f"unsupported_gateway_platform:{account.platform.value}:{account.mode.value}"
)

20
surfsense_backend/app/gateway/runner.py
View file

@ -8,7 +8,12 @@ import uuid
from sqlalchemy import text
from app.db import ExternalChatPlatform, ExternalChatAccount, async_session_maker, engine
from app.db import (
ExternalChatAccount,
ExternalChatPlatform,
async_session_maker,
engine,
)
from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key
from app.gateway.telegram.adapter import TelegramAdapter
from app.observability.metrics import record_gateway_byo_longpoll_running_delta
@ -39,7 +44,9 @@ async def _run_telegram_account(account_id: int, token: str) -> None:
account = await session.get(ExternalChatAccount, account_id)
offset = None
if account is not None:
offset = int((account.cursor_state or {}).get("last_update_id", 0)) + 1
offset = (
int((account.cursor_state or {}).get("last_update_id", 0)) + 1
)
async for update in adapter.fetch_updates(offset=offset):
request_id = f"gateway_{uuid.uuid4().hex[:16]}"
@ -58,8 +65,11 @@ async def _run_telegram_account(account_id: int, token: str) -> None:
)
await session.commit()
if inbox_id is not None:
logger.debug("Persisted Telegram polling update inbox_id=%s", inbox_id)
logger.debug(
"Persisted Telegram polling update inbox_id=%s", inbox_id
)
finally:
record_gateway_byo_longpoll_running_delta(-1, account_id=account_id)
await conn.execute(text("SELECT pg_advisory_unlock(:key)"), {"key": lock_key})
await conn.execute(
text("SELECT pg_advisory_unlock(:key)"), {"key": lock_key}
)

4
surfsense_backend/app/gateway/slack/adapter.py
View file

@ -38,7 +38,9 @@ class SlackAdapter(BasePlatformAdapter):
slack_user_id = str(event.get("user") or "")
message_ts = str(event.get("ts") or "")
thread_ts = str(event.get("thread_ts") or message_ts)
bot_user_id = self.bot_user_id or str(raw_payload.get("authorizations", [{}])[0].get("user_id") or "")
bot_user_id = self.bot_user_id or str(
raw_payload.get("authorizations", [{}])[0].get("user_id") or ""
)
if not channel_id or not slack_user_id or not message_ts:
return ParsedInboundEvent(

8
surfsense_backend/app/gateway/slack/client.py
View file

@ -15,7 +15,9 @@ class SlackGatewayClient:
def __init__(self, bot_token: str) -> None:
self.bot_token = bot_token
async def api_call(self, method: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
async def api_call(
self, method: str, payload: dict[str, Any] | None = None
) -> dict[str, Any]:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.post(
f"{SLACK_API}/{method}",
@ -55,7 +57,9 @@ class SlackGatewayClient:
ts: str,
text: str,
) -> PlatformSendResult:
data = await self.api_call("chat.update", {"channel": channel, "ts": ts, "text": text})
data = await self.api_call(
"chat.update", {"channel": channel, "ts": ts, "text": text}
)
return PlatformSendResult(
external_message_id=str(data.get("ts") or ts),
raw_response=data,

8
surfsense_backend/app/gateway/slack/translator.py
View file

@ -41,7 +41,9 @@ class SlackStreamTranslator(BaseStreamTranslator):
async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None:
async for event in events:
if event.type in {"text-delta", "text_delta", "text"}:
self._buffer += str(event.data.get("text") or event.data.get("delta") or "")
self._buffer += str(
event.data.get("text") or event.data.get("delta") or ""
)
elif event.type in {"data-interrupt-request", "interrupt"}:
await self._handle_hitl_interrupt()
return
@ -53,7 +55,9 @@ class SlackStreamTranslator(BaseStreamTranslator):
async def _flush_final(self) -> None:
if not self._buffer:
return
for chunk in split_text_message(self._buffer, max_chars=SLACK_MAX_MESSAGE_CHARS):
for chunk in split_text_message(
self._buffer, max_chars=SLACK_MAX_MESSAGE_CHARS
):
await self._send_text(chunk)
async def _send_text(self, text: str) -> PlatformSendResult:

1
surfsense_backend/app/gateway/telegram/__init__.py
View file

@ -1,2 +1 @@
"""Telegram gateway adapter."""

22
surfsense_backend/app/gateway/telegram/adapter.py
View file

@ -51,9 +51,7 @@ class TelegramAdapter(BasePlatformAdapter):
"channel": "channel",
}.get(chat_type, "unknown")
display_name = chat.get("title") or " ".join(
part
for part in (sender.get("first_name"), sender.get("last_name"))
if part
part for part in (sender.get("first_name"), sender.get("last_name")) if part
)
return ParsedInboundEvent(
@ -62,14 +60,21 @@ class TelegramAdapter(BasePlatformAdapter):
external_peer_id=str(chat["id"]) if chat.get("id") is not None else None,
external_peer_kind=peer_kind,
external_message_id=(
str(message["message_id"]) if message.get("message_id") is not None else None
str(message["message_id"])
if message.get("message_id") is not None
else None
),
external_user_id=str(sender["id"]) if sender.get("id") is not None else None,
external_user_id=str(sender["id"])
if sender.get("id") is not None
else None,
text=message.get("text") or message.get("caption"),
raw_payload=raw_payload,
display_name=display_name or None,
username=sender.get("username") or chat.get("username"),
metadata={"chat_type": chat_type, "update_id": raw_payload.get("update_id")},
metadata={
"chat_type": chat_type,
"update_id": raw_payload.get("update_id"),
},
)
async def send_message(
@ -108,7 +113,8 @@ class TelegramAdapter(BasePlatformAdapter):
async def leave_chat(self, *, external_peer_id: str) -> None:
await self.client.leave_chat(chat_id=external_peer_id)
async def fetch_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]:
async def fetch_updates(
self, *, offset: int | None
) -> AsyncIterator[dict[str, Any]]:
async for update in self.client.get_updates(offset=offset):
yield update

1
surfsense_backend/app/gateway/telegram/client.py
View file

@ -106,4 +106,3 @@ async def retry_plaintext_on_bad_markdown(call, *args, **kwargs) -> PlatformSend
raise
kwargs["parse_mode"] = None
return await call(*args, **kwargs)

6
surfsense_backend/app/gateway/telegram/commands.py
View file

@ -54,7 +54,9 @@ async def handle_start_command(
return True
async def handle_help_command(*, adapter: TelegramAdapter, event: ParsedInboundEvent) -> bool:
async def handle_help_command(
*, adapter: TelegramAdapter, event: ParsedInboundEvent
) -> bool:
if not event.external_peer_id:
return True
await adapter.send_message(external_peer_id=event.external_peer_id, text=HELP_TEXT)
@ -114,4 +116,4 @@ class TelegramGatewayCommands(BaseGatewayCommands):
adapter=adapter,
event=event,
dashboard_url=dashboard_url,
)
)

9
surfsense_backend/app/gateway/telegram/formatting.py
View file

@ -32,9 +32,13 @@ def _split_at_boundary(text: str, max_units: int) -> tuple[str, str]:
end -= 1
candidate = text[:end]
boundary = max(candidate.rfind("\n\n"), candidate.rfind(". "), candidate.rfind("\n"))
boundary = max(
candidate.rfind("\n\n"), candidate.rfind(". "), candidate.rfind("\n")
)
if boundary > max(200, end // 2):
end = boundary + (2 if candidate[boundary : boundary + 2] in {"\n\n", ". "} else 1)
end = boundary + (
2 if candidate[boundary : boundary + 2] in {"\n\n", ". "} else 1
)
return text[:end], text[end:]
@ -56,4 +60,3 @@ def chunk_message(
chunks.append(chunk)
return chunks
return split_text_message(text, max_chars=max_units)

9
surfsense_backend/app/gateway/telegram/translator.py
View file

@ -49,7 +49,9 @@ class TelegramStreamTranslator(BaseStreamTranslator):
async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None:
async for event in events:
if event.type in {"text-delta", "text_delta", "text"}:
self._buffer += str(event.data.get("text") or event.data.get("delta") or "")
self._buffer += str(
event.data.get("text") or event.data.get("delta") or ""
)
await self._maybe_flush()
elif event.type in {"data-interrupt-request", "interrupt"}:
await self._handle_hitl_interrupt()
@ -159,7 +161,9 @@ class TelegramStreamTranslator(BaseStreamTranslator):
)
if chat_wait:
record_gateway_rate_limit_hit(bucket="tg:chat")
global_wait = await wait_for_token("tg:global", capacity=25, refill_per_sec=25.0)
global_wait = await wait_for_token(
"tg:global", capacity=25, refill_per_sec=25.0
)
if global_wait:
record_gateway_rate_limit_hit(bucket="tg:global")
@ -168,4 +172,3 @@ class TelegramStreamTranslator(BaseStreamTranslator):
await self._flush(final=False)
await self._send_text(HITL_UNSUPPORTED_MESSAGE)
record_gateway_hitl_aborted(platform="telegram")

5
surfsense_backend/app/gateway/thread_lock.py
View file

@ -36,5 +36,6 @@ def release_thread_lock(thread_id: int) -> None:
try:
_redis().delete(_lock_key(thread_id))
except redis.RedisError as exc:
logger.warning("Failed to release gateway thread lock for %s: %s", thread_id, exc)
logger.warning(
"Failed to release gateway thread lock for %s: %s", thread_id, exc
)

7
surfsense_backend/app/gateway/whatsapp/adapter_baileys.py
View file

@ -36,7 +36,8 @@ class WhatsAppBaileysAdapter(BasePlatformAdapter):
external_user_id=sender_id or None,
text=str(body) if body is not None else None,
raw_payload=raw_payload,
display_name=str(raw_payload.get("chatName") or sender_id or chat_id) or None,
display_name=str(raw_payload.get("chatName") or sender_id or chat_id)
or None,
username=None,
metadata={
"sender_id": sender_id,
@ -92,7 +93,9 @@ class WhatsAppBaileysAdapter(BasePlatformAdapter):
response.raise_for_status()
return response.json()
async def fetch_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]:
async def fetch_updates(
self, *, offset: int | None
) -> AsyncIterator[dict[str, Any]]:
async with httpx.AsyncClient(timeout=35) as client:
response = await client.get(f"{self.bridge_url}/messages")
response.raise_for_status()

8
surfsense_backend/app/gateway/whatsapp/adapter_cloud.py
View file

@ -54,7 +54,9 @@ class WhatsAppCloudAdapter(BasePlatformAdapter):
username=None,
metadata={
"phone_number_id": _metadata(raw_payload).get("phone_number_id"),
"display_phone_number": _metadata(raw_payload).get("display_phone_number"),
"display_phone_number": _metadata(raw_payload).get(
"display_phone_number"
),
"timestamp": message.get("timestamp"),
"message_type": message.get("type"),
},
@ -96,7 +98,9 @@ def _changes(raw_payload: dict[str, Any]) -> list[dict[str, Any]]:
for entry in raw_payload.get("entry") or []:
if isinstance(entry, dict):
changes.extend(
change for change in (entry.get("changes") or []) if isinstance(change, dict)
change
for change in (entry.get("changes") or [])
if isinstance(change, dict)
)
return changes

3
surfsense_backend/app/gateway/whatsapp/credentials.py
View file

@ -18,8 +18,7 @@ class WhatsAppCredentials(TypedDict, total=False):
def load_system_whatsapp_credentials() -> WhatsAppCredentials:
if not (
config.WHATSAPP_SHARED_BUSINESS_TOKEN
and config.WHATSAPP_SHARED_PHONE_NUMBER_ID
config.WHATSAPP_SHARED_BUSINESS_TOKEN and config.WHATSAPP_SHARED_PHONE_NUMBER_ID
):
raise RuntimeError("whatsapp_system_credentials_not_configured")

4
surfsense_backend/app/gateway/whatsapp/translator.py
View file

@ -41,7 +41,9 @@ class WhatsAppCloudStreamTranslator(BaseStreamTranslator):
if event.type in {"text-delta", "text_delta", "text"}:
if not self._typing_sent:
await self._send_typing_indicator()
self._buffer += str(event.data.get("text") or event.data.get("delta") or "")
self._buffer += str(
event.data.get("text") or event.data.get("delta") or ""
)
elif event.type in {"data-interrupt-request", "interrupt"}:
await self._handle_hitl_interrupt()
return

8
surfsense_backend/app/gateway/whatsapp/translator_baileys.py
View file

@ -42,7 +42,9 @@ class WhatsAppBaileysStreamTranslator(BaseStreamTranslator):
await self._send_typing_indicator()
async for event in events:
if event.type in {"text-delta", "text_delta", "text"}:
self._buffer += str(event.data.get("text") or event.data.get("delta") or "")
self._buffer += str(
event.data.get("text") or event.data.get("delta") or ""
)
await self._maybe_flush()
elif event.type in {"data-interrupt-request", "interrupt"}:
await self._handle_hitl_interrupt()
@ -86,7 +88,9 @@ class WhatsAppBaileysStreamTranslator(BaseStreamTranslator):
if not isinstance(self.adapter, WhatsAppBaileysAdapter):
return
try:
await self.adapter.send_typing_indicator(external_peer_id=self.external_peer_id)
await self.adapter.send_typing_indicator(
external_peer_id=self.external_peer_id
)
record_gateway_outbound(platform="whatsapp", kind="typing", status="sent")
except Exception:
logger.debug("WhatsApp Baileys typing indicator failed", exc_info=True)

8
surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py
View file

@ -202,7 +202,9 @@ class IndexingPipelineService:
await self.session.commit()
async def index_batch(self, connector_docs: list[ConnectorDocument]) -> list[Document]:
async def index_batch(
self, connector_docs: list[ConnectorDocument]
) -> list[Document]:
"""Convenience method: prepare_for_indexing then index each document.
Indexers that need heartbeat callbacks or custom per-document logic
@ -347,7 +349,9 @@ class IndexingPipelineService:
await self.session.rollback()
return []
async def index(self, document: Document, connector_doc: ConnectorDocument) -> Document:
async def index(
self, document: Document, connector_doc: ConnectorDocument
) -> Document:
"""
Run deterministic content storage, embedding, and chunking for a document.
"""

1
surfsense_backend/app/notifications/__init__.py
View file

@ -9,7 +9,6 @@ from __future__ import annotations
# Initialize app.db first to avoid a partial-init circular import when this
# package is the entry point (e.g. Celery loading it before any ORM code).
import app.db # noqa: F401
from app.notifications.persistence import Notification
from app.notifications.service import NotificationService

12
surfsense_backend/app/routes/__init__.py
View file

@ -1,8 +1,9 @@
from fastapi import APIRouter, Depends
from app.automations.api import router as automations_router
from app.gateway import require_gateway_enabled
from app.file_storage.api import router as file_storage_router
from app.gateway import require_gateway_enabled
from app.notifications.api import router as notifications_router
from .agent_action_log_route import router as agent_action_log_router
from .agent_flags_route import router as agent_flags_router
@ -46,7 +47,6 @@ from .model_list_routes import router as model_list_router
from .new_chat_routes import router as new_chat_router
from .new_llm_config_routes import router as new_llm_config_router
from .notes_routes import router as notes_router
from app.notifications.api import router as notifications_router
from .notion_add_connector_route import router as notion_add_connector_router
from .obsidian_plugin_routes import router as obsidian_plugin_router
from .onedrive_add_connector_route import router as onedrive_add_connector_router
@ -76,8 +76,12 @@ router.include_router(documents_router)
router.include_router(folders_router)
_gateway_enabled_dep = [Depends(require_gateway_enabled)]
router.include_router(gateway_router, dependencies=_gateway_enabled_dep)
router.include_router(gateway_whatsapp_webhook_router, dependencies=_gateway_enabled_dep)
router.include_router(gateway_whatsapp_baileys_router, dependencies=_gateway_enabled_dep)
router.include_router(
gateway_whatsapp_webhook_router, dependencies=_gateway_enabled_dep
)
router.include_router(
gateway_whatsapp_baileys_router, dependencies=_gateway_enabled_dep
)
router.include_router(notes_router)
router.include_router(new_chat_router) # Chat with assistant-ui persistence
router.include_router(agent_revert_router) # POST /threads/{id}/revert/{action_id}

110
surfsense_backend/app/routes/gateway_webhook_routes.py
View file

@ -119,21 +119,35 @@ def _discord_redirect_uri() -> str:
return f"{base.rstrip('/')}/api/v1/gateway/discord/callback"
def _slack_frontend_redirect(space_id: int, *, success: bool = False, error: str | None = None) -> RedirectResponse:
qs = "slack_gateway=connected" if success else f"error={error or 'slack_gateway_failed'}"
def _slack_frontend_redirect(
space_id: int, *, success: bool = False, error: str | None = None
) -> RedirectResponse:
qs = (
"slack_gateway=connected"
if success
else f"error={error or 'slack_gateway_failed'}"
)
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/user-settings?{qs}"
)
def _discord_frontend_redirect(space_id: int, *, success: bool = False, error: str | None = None) -> RedirectResponse:
qs = "discord_gateway=connected" if success else f"error={error or 'discord_gateway_failed'}"
def _discord_frontend_redirect(
space_id: int, *, success: bool = False, error: str | None = None
) -> RedirectResponse:
qs = (
"discord_gateway=connected"
if success
else f"error={error or 'discord_gateway_failed'}"
)
return RedirectResponse(
url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/user-settings?{qs}"
)
def verify_slack_signature(*, signing_secret: str, timestamp: str | None, signature: str | None, body: bytes) -> bool:
def verify_slack_signature(
*, signing_secret: str, timestamp: str | None, signature: str | None, body: bytes
) -> bool:
if not signing_secret or not timestamp or not signature:
return False
try:
@ -239,7 +253,9 @@ async def install_slack_gateway(
session: AsyncSession = Depends(get_async_session),
) -> dict[str, str]:
if not _slack_gateway_enabled():
raise HTTPException(status_code=500, detail="Slack gateway OAuth is not configured")
raise HTTPException(
status_code=500, detail="Slack gateway OAuth is not configured"
)
await check_search_space_access(session, user, search_space_id)
state = _get_state_manager().generate_secure_state(search_space_id, user.id)
auth_params = {
@ -269,11 +285,17 @@ async def slack_gateway_callback(
state_data = None
if error:
return _slack_frontend_redirect(space_id or 0, error="slack_gateway_oauth_denied")
return _slack_frontend_redirect(
space_id or 0, error="slack_gateway_oauth_denied"
)
if not code or state_data is None:
raise HTTPException(status_code=400, detail="Invalid Slack gateway OAuth callback")
raise HTTPException(
status_code=400, detail="Invalid Slack gateway OAuth callback"
)
if not _slack_gateway_enabled():
raise HTTPException(status_code=500, detail="Slack gateway OAuth is not configured")
raise HTTPException(
status_code=500, detail="Slack gateway OAuth is not configured"
)
user_id = UUID(state_data["user_id"])
token_payload = {
@ -300,7 +322,9 @@ async def slack_gateway_callback(
team = token_json.get("team") or {}
team_id = team.get("id")
if not bot_token or not team_id:
raise HTTPException(status_code=400, detail="Slack gateway OAuth returned incomplete data")
raise HTTPException(
status_code=400, detail="Slack gateway OAuth returned incomplete data"
)
bot_user_id = token_json.get("bot_user_id")
app_id = token_json.get("app_id")
@ -388,7 +412,9 @@ async def install_discord_gateway(
session: AsyncSession = Depends(get_async_session),
) -> dict[str, str]:
if not _discord_gateway_enabled():
raise HTTPException(status_code=500, detail="Discord gateway OAuth is not configured")
raise HTTPException(
status_code=500, detail="Discord gateway OAuth is not configured"
)
await check_search_space_access(session, user, search_space_id)
state = _get_state_manager().generate_secure_state(search_space_id, user.id)
auth_params = {
@ -420,11 +446,17 @@ async def discord_gateway_callback(
state_data = None
if error:
return _discord_frontend_redirect(space_id or 0, error="discord_gateway_oauth_denied")
return _discord_frontend_redirect(
space_id or 0, error="discord_gateway_oauth_denied"
)
if not code or state_data is None:
raise HTTPException(status_code=400, detail="Invalid Discord gateway OAuth callback")
raise HTTPException(
status_code=400, detail="Invalid Discord gateway OAuth callback"
)
if not _discord_gateway_enabled():
raise HTTPException(status_code=500, detail="Discord gateway OAuth is not configured")
raise HTTPException(
status_code=500, detail="Discord gateway OAuth is not configured"
)
user_id = UUID(state_data["user_id"])
token_payload = {
@ -535,7 +567,10 @@ async def discord_gateway_callback(
elif binding.user_id == user_id:
binding.search_space_id = space_id
binding.external_username = discord_username or binding.external_username
binding.external_metadata = {**(binding.external_metadata or {}), **metadata}
binding.external_metadata = {
**(binding.external_metadata or {}),
**metadata,
}
await session.commit()
return _discord_frontend_redirect(space_id, success=True)
@ -614,7 +649,9 @@ async def _resolve_webhook_account(
if account is None or account.platform != ExternalChatPlatform.TELEGRAM:
raise HTTPException(status_code=404, detail="Gateway account not found")
expected_secret = account.webhook_secret or ""
if not expected_secret or not hmac.compare_digest(header_secret or "", expected_secret):
if not expected_secret or not hmac.compare_digest(
header_secret or "", expected_secret
):
raise HTTPException(status_code=403, detail="Invalid Telegram webhook secret")
return account
@ -654,7 +691,9 @@ async def telegram_webhook(
event_dedupe_key=telegram_event_dedupe_key(update_id),
external_event_id=str(update_id),
external_message_id=(
str(message["message_id"]) if message.get("message_id") is not None else None
str(message["message_id"])
if message.get("message_id") is not None
else None
),
event_kind=_classify_telegram_event(payload),
raw_payload=payload,
@ -739,7 +778,10 @@ async def list_bindings(
) -> list[dict[str, Any]]:
result = await session.execute(
select(ExternalChatBinding, ExternalChatAccount)
.join(ExternalChatAccount, ExternalChatBinding.account_id == ExternalChatAccount.id)
.join(
ExternalChatAccount,
ExternalChatBinding.account_id == ExternalChatAccount.id,
)
.where(ExternalChatBinding.user_id == user.id)
)
return [
@ -777,13 +819,20 @@ async def list_connections(
]
if platform is not None:
filters.append(ExternalChatAccount.platform == platform)
if platform == ExternalChatPlatform.WHATSAPP and active_whatsapp_mode is not None:
if (
platform == ExternalChatPlatform.WHATSAPP
and active_whatsapp_mode is not None
):
filters.append(ExternalChatAccount.mode == active_whatsapp_mode)
else:
if not _telegram_gateway_enabled():
filters.append(ExternalChatAccount.platform != ExternalChatPlatform.TELEGRAM)
filters.append(
ExternalChatAccount.platform != ExternalChatPlatform.TELEGRAM
)
if active_whatsapp_mode is None:
filters.append(ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP)
filters.append(
ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP
)
else:
filters.append(
or_(
@ -794,7 +843,10 @@ async def list_connections(
result = await session.execute(
select(ExternalChatBinding, ExternalChatAccount)
.join(ExternalChatAccount, ExternalChatBinding.account_id == ExternalChatAccount.id)
.join(
ExternalChatAccount,
ExternalChatBinding.account_id == ExternalChatAccount.id,
)
.where(*filters)
)
@ -828,7 +880,9 @@ async def list_connections(
baileys_account_ids.add(int(account.id))
route_type = "account"
connection_id = account.id
search_space_id = account.owner_search_space_id or binding.search_space_id
search_space_id = (
account.owner_search_space_id or binding.search_space_id
)
display_name = "WhatsApp Bridge"
connections.append(
@ -853,9 +907,8 @@ async def list_connections(
}
)
if (
active_whatsapp_mode == ExternalChatAccountMode.SELF_HOST_BYO
and (platform is None or platform == ExternalChatPlatform.WHATSAPP)
if active_whatsapp_mode == ExternalChatAccountMode.SELF_HOST_BYO and (
platform is None or platform == ExternalChatPlatform.WHATSAPP
):
account_result = await session.execute(
select(ExternalChatAccount).where(
@ -940,7 +993,9 @@ async def update_binding_search_space(
ExternalChatBindingState.BOUND,
ExternalChatBindingState.SUSPENDED,
}:
raise HTTPException(status_code=400, detail="Only active bindings can be routed")
raise HTTPException(
status_code=400, detail="Only active bindings can be routed"
)
account = await session.get(ExternalChatAccount, binding.account_id)
if account is None or _is_inactive_whatsapp_account(account):
raise HTTPException(status_code=404, detail="Binding not found")
@ -1062,4 +1117,3 @@ async def resume_external_chat_binding(
binding.updated_at = datetime.now(UTC)
await session.commit()
return {"ok": True}

4
surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py
View file

@ -33,7 +33,9 @@ class BaileysPairRequest(BaseModel):
def _ensure_baileys_enabled() -> None:
if config.GATEWAY_WHATSAPP_INTAKE_MODE != "baileys":
raise HTTPException(status_code=404, detail="WhatsApp Baileys gateway is disabled")
raise HTTPException(
status_code=404, detail="WhatsApp Baileys gateway is disabled"
)
if config.is_cloud():
raise HTTPException(
status_code=403,

12
surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py
View file

@ -79,7 +79,9 @@ async def whatsapp_webhook(
def _verify_signature(raw_body: bytes, header_signature: str | None) -> None:
if not config.WHATSAPP_WEBHOOK_APP_SECRET:
raise HTTPException(status_code=500, detail="WhatsApp app secret is not configured")
raise HTTPException(
status_code=500, detail="WhatsApp app secret is not configured"
)
received = (header_signature or "").removeprefix("sha256=")
expected = hmac.new(
config.WHATSAPP_WEBHOOK_APP_SECRET.encode(),
@ -87,7 +89,9 @@ def _verify_signature(raw_body: bytes, header_signature: str | None) -> None:
hashlib.sha256,
).hexdigest()
if not received or not hmac.compare_digest(received, expected):
raise HTTPException(status_code=403, detail="Invalid WhatsApp webhook signature")
raise HTTPException(
status_code=403, detail="Invalid WhatsApp webhook signature"
)
async def _process_payload(session: AsyncSession, payload: dict[str, Any]) -> None:
@ -114,7 +118,9 @@ async def _process_messages_change(
change: dict[str, Any],
value: dict[str, Any],
) -> None:
statuses = [status for status in value.get("statuses") or [] if isinstance(status, dict)]
statuses = [
status for status in value.get("statuses") or [] if isinstance(status, dict)
]
for status in statuses:
record_gateway_outbound(
platform="whatsapp",

2
surfsense_backend/app/routes/obsidian_plugin_routes.py
View file

@ -25,6 +25,7 @@ from app.db import (
User,
get_async_session,
)
from app.notifications.service import NotificationService
from app.schemas.obsidian_plugin import (
ALLOWED_ATTACHMENT_EXTENSIONS,
ATTACHMENT_MIME_TYPES,
@ -43,7 +44,6 @@ from app.schemas.obsidian_plugin import (
SyncAckItem,
SyncBatchRequest,
)
from app.notifications.service import NotificationService
from app.services.obsidian_plugin_indexer import (
delete_note,
get_manifest,

2
surfsense_backend/app/services/chat_comments_service.py
View file

@ -20,6 +20,7 @@ from app.db import (
User,
has_permission,
)
from app.notifications.service import NotificationService
from app.schemas.chat_comments import (
AuthorResponse,
CommentBatchResponse,
@ -31,7 +32,6 @@ from app.schemas.chat_comments import (
MentionListResponse,
MentionResponse,
)
from app.notifications.service import NotificationService
from app.utils.chat_comments import parse_mentions, render_mentions
from app.utils.rbac import check_permission, get_user_permissions

5
surfsense_backend/app/services/confluence/kb_sync_service.py
View file

@ -64,9 +64,6 @@ class ConfluenceKBSyncService:
if dup:
content_hash = unique_hash
summary_content = f"Confluence Page: {page_title}\n\n{page_content}"
summary_embedding = embed_text(summary_content)
@ -166,8 +163,6 @@ class ConfluenceKBSyncService:
space_id = (document.document_metadata or {}).get("space_id", "")
summary_content = f"Confluence Page: {page_title}\n\n{page_content}"
summary_embedding = embed_text(summary_content)

3
surfsense_backend/app/services/dropbox/kb_sync_service.py
View file

@ -71,9 +71,6 @@ class DropboxKBSyncService:
)
content_hash = unique_hash
summary_content = f"Dropbox File: {file_name}\n\n{indexable_content}"
summary_embedding = embed_text(summary_content)

3
surfsense_backend/app/services/gmail/kb_sync_service.py
View file

@ -77,9 +77,6 @@ class GmailKBSyncService:
)
content_hash = unique_hash
summary_content = f"Gmail Message: {subject}\n\n{indexable_content}"
summary_embedding = await asyncio.to_thread(embed_text, summary_content)

6
surfsense_backend/app/services/google_calendar/kb_sync_service.py
View file

@ -89,9 +89,6 @@ class GoogleCalendarKBSyncService:
)
content_hash = unique_hash
summary_content = (
f"Google Calendar Event: {event_summary}\n\n{indexable_content}"
)
@ -252,9 +249,6 @@ class GoogleCalendarKBSyncService:
if not indexable_content:
return {"status": "error", "message": "Event produced empty content"}
summary_content = (
f"Google Calendar Event: {event_summary}\n\n{indexable_content}"
)

7
surfsense_backend/app/services/google_drive/kb_sync_service.py
View file

@ -73,12 +73,7 @@ class GoogleDriveKBSyncService:
)
content_hash = unique_hash
summary_content = (
f"Google Drive File: {file_name}\n\n{indexable_content}"
)
summary_content = f"Google Drive File: {file_name}\n\n{indexable_content}"
summary_embedding = embed_text(summary_content)
chunks = await create_document_chunks(indexable_content)

5
surfsense_backend/app/services/linear/kb_sync_service.py
View file

@ -83,9 +83,6 @@ class LinearKBSyncService:
)
content_hash = unique_hash
summary_content = (
f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}"
)
@ -207,8 +204,6 @@ class LinearKBSyncService:
comment_count = len(formatted_issue.get("comments", []))
formatted_issue.get("description", "")
summary_content = (
f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}"
)

4
surfsense_backend/app/services/notion/kb_sync_service.py
View file

@ -72,9 +72,6 @@ class NotionKBSyncService:
)
content_hash = unique_hash
summary_content = f"Notion Page: {page_title}\n\n{markdown_content}"
summary_embedding = embed_text(summary_content)
@ -225,7 +222,6 @@ class NotionKBSyncService:
f"Final content length: {len(full_content)} chars, verified={content_verified}"
)
logger.debug("Generating summary and embeddings")
summary_content = f"Notion Page: {document.document_metadata.get('page_title')}\n\n{full_content}"

3
surfsense_backend/app/services/onedrive/kb_sync_service.py
View file

@ -72,9 +72,6 @@ class OneDriveKBSyncService:
)
content_hash = unique_hash
summary_content = f"OneDrive File: {file_name}\n\n{indexable_content}"
summary_embedding = await asyncio.to_thread(embed_text, summary_content)

6
surfsense_backend/app/tasks/celery_tasks/document_tasks.py
View file

@ -9,8 +9,8 @@ from uuid import UUID
from app.celery_app import celery_app
from app.config import config
from app.observability import metrics as ot_metrics
from app.notifications.service import NotificationService
from app.observability import metrics as ot_metrics
from app.services.task_logging_service import TaskLoggingService
from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task
from app.tasks.connector_indexers.local_folder_indexer import (
@ -1335,7 +1335,7 @@ async def _index_local_folder_async(
exclude_patterns=exclude_patterns,
file_extensions=file_extensions,
root_folder_id=root_folder_id,
target_file_paths=target_file_paths,
target_file_paths=target_file_paths,
on_heartbeat_callback=_heartbeat_progress
if (is_batch or is_full_scan)
else None,
@ -1463,7 +1463,7 @@ async def _index_uploaded_folder_files_async(
user_id=user_id,
folder_name=folder_name,
root_folder_id=root_folder_id,
file_mappings=file_mappings,
file_mappings=file_mappings,
on_heartbeat_callback=_heartbeat_progress,
use_vision_llm=use_vision_llm,
processing_mode=processing_mode,

4
surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py
View file

@ -46,7 +46,8 @@ def reconcile_inbox_task() -> None:
result = await session.execute(
update(ExternalChatInboundEvent)
.where(
ExternalChatInboundEvent.status == ExternalChatEventStatus.PROCESSING,
ExternalChatInboundEvent.status
== ExternalChatEventStatus.PROCESSING,
ExternalChatInboundEvent.received_at < stale_threshold,
)
.values(
@ -163,4 +164,3 @@ async def enqueue_telegram_update(account_id: int, raw_update: dict) -> int | No
)
await session.commit()
return inbox_id

2
surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py
View file

@ -260,7 +260,7 @@ async def index_confluence_pages(
connector_id=connector_id,
search_space_id=search_space_id,
user_id=user_id,
)
)
with session.no_autoflush:
duplicate_by_content = await check_duplicate_document_by_hash(

2
surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py
View file

@ -414,7 +414,7 @@ async def index_google_calendar_events(
connector_id=connector_id,
search_space_id=search_space_id,
user_id=user_id,
)
)
with session.no_autoflush:
duplicate = await check_duplicate_document_by_hash(

2
surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py
View file

@ -552,7 +552,7 @@ async def _process_single_file(
connector_id=connector_id,
search_space_id=search_space_id,
user_id=user_id,
)
)
pipeline = IndexingPipelineService(session)
documents = await pipeline.prepare_for_indexing([doc])

2
surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py
View file

@ -444,7 +444,7 @@ async def index_google_gmail_messages(
connector_id=connector_id,
search_space_id=search_space_id,
user_id=user_id,
)
)
with session.no_autoflush:
duplicate = await check_duplicate_document_by_hash(

2
surfsense_backend/app/tasks/connector_indexers/linear_indexer.py
View file

@ -268,7 +268,7 @@ async def index_linear_issues(
connector_id=connector_id,
search_space_id=search_space_id,
user_id=user_id,
)
)
with session.no_autoflush:
duplicate = await check_duplicate_document_by_hash(

12
surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py
View file

@ -568,7 +568,7 @@ async def index_local_folder(
folder_path=folder_path,
folder_name=folder_name,
target_file_path=target_file_paths[0],
root_folder_id=root_folder_id,
root_folder_id=root_folder_id,
task_logger=task_logger,
log_entry=log_entry,
)
@ -580,7 +580,7 @@ async def index_local_folder(
folder_path=folder_path,
folder_name=folder_name,
target_file_paths=target_file_paths,
root_folder_id=root_folder_id,
root_folder_id=root_folder_id,
on_progress_callback=on_heartbeat_callback,
)
if err:
@ -766,7 +766,7 @@ async def index_local_folder(
folder_name=folder_name,
search_space_id=search_space_id,
user_id=user_id,
)
)
connector_docs.append(doc)
file_meta_map[unique_identifier] = {
"relative_path": relative_path,
@ -983,7 +983,7 @@ async def _index_batch_files(
folder_path=folder_path,
folder_name=folder_name,
target_file_path=file_path,
root_folder_id=root_folder_id,
root_folder_id=root_folder_id,
task_logger=task_logger,
log_entry=log_entry,
)
@ -1111,7 +1111,7 @@ async def _index_single_file(
folder_name=folder_name,
search_space_id=search_space_id,
user_id=user_id,
)
)
if root_folder_id:
connector_doc.folder_id = await _resolve_folder_for_file(
@ -1396,7 +1396,7 @@ async def index_uploaded_files(
folder_name=folder_name,
search_space_id=search_space_id,
user_id=user_id,
)
)
connector_doc.folder_id = await _resolve_folder_for_file(
session,

4
surfsense_backend/app/tasks/connector_indexers/luma_indexer.py
View file

@ -440,9 +440,7 @@ async def index_luma_events(
summary_content = (
f"Luma Event: {item['event_name']}\n\n{item['event_markdown']}"
)
summary_embedding = await asyncio.to_thread(
embed_text, summary_content
)
summary_embedding = await asyncio.to_thread(embed_text, summary_content)
chunks = await create_document_chunks(item["event_markdown"])

2
surfsense_backend/app/tasks/connector_indexers/notion_indexer.py
View file

@ -308,7 +308,7 @@ async def index_notion_pages(
connector_id=connector_id,
search_space_id=search_space_id,
user_id=user_id,
)
)
with session.no_autoflush:
duplicate = await check_duplicate_document_by_hash(

8
surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py
View file

@ -318,9 +318,7 @@ async def index_crawled_urls(
continue
# Format content as structured document for summary generation
structured_document = crawler.format_to_structured_document(
crawl_result
)
crawler.format_to_structured_document(crawl_result)
# Generate content hash using a version WITHOUT metadata
structured_document_for_hash = crawler.format_to_structured_document(
@ -332,8 +330,8 @@ async def index_crawled_urls(
# Extract useful metadata
title = metadata.get("title", url)
description = metadata.get("description", "")
language = metadata.get("language", "")
metadata.get("description", "")
metadata.get("language", "")
# Update title immediately for better UX
document.title = title

6
surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py
View file

@ -34,9 +34,9 @@ class AnonymousProxiesProvider(ProxyProvider):
"l": Config.RESIDENTIAL_PROXY_LOCATION,
"t": Config.RESIDENTIAL_PROXY_TYPE,
}
return base64.b64encode(
json.dumps(password_dict).encode("utf-8")
).decode("utf-8")
return base64.b64encode(json.dumps(password_dict).encode("utf-8")).decode(
"utf-8"
)
def get_proxy_url(self) -> str | None:
username = Config.RESIDENTIAL_PROXY_USERNAME

13
surfsense_backend/app/zero_publication.py
View file

@ -110,7 +110,9 @@ def _format_table_entry(conn: Connection, table: str) -> str:
def build_set_table_sql(conn: Connection) -> str:
"""Build the canonical plain SET TABLE statement for Zero's event triggers."""
table_list = ", ".join(_format_table_entry(conn, table) for table in ZERO_PUBLICATION)
table_list = ", ".join(
_format_table_entry(conn, table) for table in ZERO_PUBLICATION
)
return f"ALTER PUBLICATION {_quote_identifier(PUBLICATION_NAME)} SET TABLE {table_list}"
@ -175,7 +177,9 @@ def verify_publication(conn: Connection) -> list[str]:
actual_columns = actual[table]
actual_key = sorted(actual_columns) if actual_columns is not None else None
expected_key = sorted(expected_columns) if expected_columns is not None else None
expected_key = (
sorted(expected_columns) if expected_columns is not None else None
)
if actual_key != expected_key:
mismatches.append(
f"{table}: expected columns {expected_columns or 'ALL'}, "
@ -196,6 +200,7 @@ async def _verify_cli() -> int:
engine = create_async_engine(database_url)
async with engine.connect() as async_conn:
def run_verify(sync_conn: Connection) -> list[str]:
return verify_publication(sync_conn)
@ -215,7 +220,9 @@ async def _verify_cli() -> int:
def main() -> int:
parser = argparse.ArgumentParser(description="Manage SurfSense's Zero publication")
parser.add_argument("--verify", action="store_true", help="verify zero_publication shape")
parser.add_argument(
"--verify", action="store_true", help="verify zero_publication shape"
)
args = parser.parse_args()
if args.verify:

5
surfsense_backend/scripts/register_webhook.py
View file

@ -41,7 +41,9 @@ async def main() -> int:
await session.commit()
account_id = int(account.id)
webhook_url = f"{base_url.rstrip('/')}/api/v1/gateway/webhooks/telegram/{account_id}"
webhook_url = (
f"{base_url.rstrip('/')}/api/v1/gateway/webhooks/telegram/{account_id}"
)
bot = Bot(token=token)
ok = await bot.set_webhook(
url=webhook_url,
@ -58,4 +60,3 @@ async def main() -> int:
if __name__ == "__main__":
raise SystemExit(asyncio.run(main()))

5
surfsense_backend/tests/e2e/fakes/mcp_runtime.py
View file

@ -140,7 +140,10 @@ def install(active_patches: list[Any]) -> None:
"app.agents.chat.multi_agent_chat.shared.tools.mcp.tool.streamablehttp_client",
_fake_streamablehttp_client,
),
("app.agents.chat.multi_agent_chat.shared.tools.mcp.tool.ClientSession", _FakeClientSession),
(
"app.agents.chat.multi_agent_chat.shared.tools.mcp.tool.ClientSession",
_FakeClientSession,
),
]
for target, replacement in targets:
p = patch(target, replacement)

4
surfsense_backend/tests/integration/agents/multi_agent_chat/test_agent_turn.py
View file

@ -135,8 +135,6 @@ async def test_agent_checkpoint_round_trips_across_turns(
{"messages": [HumanMessage(content="second turn")]}, config
)
texts = [
m.content for m in second["messages"] if isinstance(m, HumanMessage)
]
texts = [m.content for m in second["messages"] if isinstance(m, HumanMessage)]
assert "remember apple" in texts, "turn 1 history not reloaded from checkpoint"
assert len(second["messages"]) > len(first["messages"])

6
surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py
View file

@ -45,9 +45,7 @@ def _build_desktop_fs_mw(root: Path):
"""Build the production filesystem middleware bound to a real local folder."""
selection = FilesystemSelection(
mode=FilesystemMode.DESKTOP_LOCAL_FOLDER,
local_mounts=(
LocalFilesystemMount(mount_id=_MOUNT_ID, root_path=str(root)),
),
local_mounts=(LocalFilesystemMount(mount_id=_MOUNT_ID, root_path=str(root)),),
)
resolver = build_backend_resolver(selection)
return build_filesystem_mw(
@ -157,7 +155,7 @@ async def test_write_then_ls_lists_file(tmp_path: Path):
async def test_edit_file_rewrites_on_disk(tmp_path: Path):
"""edit_file applies a real string replacement to the on-disk file."""
result = await _run(
await _run(
tmp_path,
[
ScriptedTurn(

28
surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py
View file

@ -7,9 +7,7 @@ from app.indexing_pipeline.adapters.file_upload_adapter import UploadDocumentAda
pytestmark = pytest.mark.integration
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_sets_status_ready(db_session, db_search_space, db_user, mocker):
"""Document status is READY after successful indexing."""
adapter = UploadDocumentAdapter(db_session)
@ -29,9 +27,7 @@ async def test_sets_status_ready(db_session, db_search_space, db_user, mocker):
assert DocumentStatus.is_state(document.status, DocumentStatus.READY)
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_content_is_source_markdown(db_session, db_search_space, db_user, mocker):
"""Document content is set to the extracted source markdown."""
adapter = UploadDocumentAdapter(db_session)
@ -51,9 +47,7 @@ async def test_content_is_source_markdown(db_session, db_search_space, db_user,
assert document.content == "## Hello\n\nSome content."
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_chunks_written_to_db(db_session, db_search_space, db_user, mocker):
"""Chunks derived from the source markdown are persisted in the DB."""
adapter = UploadDocumentAdapter(db_session)
@ -98,9 +92,7 @@ async def test_raises_on_indexing_failure(db_session, db_search_space, db_user,
# ---------------------------------------------------------------------------
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_reindex_updates_content(db_session, db_search_space, db_user, mocker):
"""Document content is updated to the new source markdown after reindexing."""
adapter = UploadDocumentAdapter(db_session)
@ -126,9 +118,7 @@ async def test_reindex_updates_content(db_session, db_search_space, db_user, moc
assert document.content == "## Edited\n\nNew content after user edit."
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_reindex_updates_content_hash(
db_session, db_search_space, db_user, mocker
):
@ -157,9 +147,7 @@ async def test_reindex_updates_content_hash(
assert document.content_hash != original_hash
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_reindex_sets_status_ready(db_session, db_search_space, db_user, mocker):
"""Document status is READY after successful reindexing."""
adapter = UploadDocumentAdapter(db_session)
@ -222,9 +210,7 @@ async def test_reindex_replaces_chunks(db_session, db_search_space, db_user, moc
assert chunks[0].content == "Updated chunk."
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_reindex_clears_reindexing_flag(
db_session, db_search_space, db_user, mocker
):

8
surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py
View file

@ -34,9 +34,7 @@ def _cal_doc(
)
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_calendar_pipeline_creates_ready_document(
db_session, db_search_space, db_connector, db_user, mocker
):
@ -65,9 +63,7 @@ async def test_calendar_pipeline_creates_ready_document(
assert DocumentStatus.is_state(row.status, DocumentStatus.READY)
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_calendar_legacy_doc_migrated(
db_session, db_search_space, db_connector, db_user, mocker
):

8
surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py
View file

@ -33,9 +33,7 @@ def _drive_doc(
)
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_drive_pipeline_creates_ready_document(
db_session, db_search_space, db_connector, db_user, mocker
):
@ -64,9 +62,7 @@ async def test_drive_pipeline_creates_ready_document(
assert DocumentStatus.is_state(row.status, DocumentStatus.READY)
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_drive_legacy_doc_migrated(
db_session, db_search_space, db_connector, db_user, mocker
):

8
surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py
View file

@ -32,9 +32,7 @@ def _dropbox_doc(
)
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_dropbox_pipeline_creates_ready_document(
db_session, db_search_space, db_connector, db_user, mocker
):
@ -63,9 +61,7 @@ async def test_dropbox_pipeline_creates_ready_document(
assert DocumentStatus.is_state(row.status, DocumentStatus.READY)
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_dropbox_duplicate_content_skipped(
db_session, db_search_space, db_connector, db_user, mocker
):

8
surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py
View file

@ -36,9 +36,7 @@ def _gmail_doc(
)
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_gmail_pipeline_creates_ready_document(
db_session, db_search_space, db_connector, db_user, mocker
):
@ -68,9 +66,7 @@ async def test_gmail_pipeline_creates_ready_document(
assert row.source_markdown == doc.source_markdown
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_gmail_legacy_doc_migrated_then_reused(
db_session, db_search_space, db_connector, db_user, mocker
):

8
surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py
View file

@ -9,9 +9,7 @@ from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineServ
pytestmark = pytest.mark.integration
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_index_batch_creates_ready_documents(
db_session, db_search_space, make_connector_document, mocker
):
@ -49,9 +47,7 @@ async def test_index_batch_creates_ready_documents(
assert row.embedding is not None
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_index_batch_empty_returns_empty(db_session, mocker):
"""index_batch with empty input returns an empty list."""
service = IndexingPipelineService(session=db_session)

8
surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py
View file

@ -32,9 +32,7 @@ def _onedrive_doc(
)
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_onedrive_pipeline_creates_ready_document(
db_session, db_search_space, db_connector, db_user, mocker
):
@ -63,9 +61,7 @@ async def test_onedrive_pipeline_creates_ready_document(
assert DocumentStatus.is_state(row.status, DocumentStatus.READY)
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_onedrive_duplicate_content_skipped(
db_session, db_search_space, db_connector, db_user, mocker
):

8
surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py
View file

@ -32,9 +32,7 @@ async def test_new_document_is_persisted_with_pending_status(
assert reloaded.source_markdown == doc.source_markdown
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_unchanged_ready_document_is_skipped(
db_session,
db_search_space,
@ -55,9 +53,7 @@ async def test_unchanged_ready_document_is_skipped(
assert results == []
@pytest.mark.usefixtures(
"patched_embed_texts", "patched_chunk_text"
)
@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text")
async def test_title_only_change_updates_title_in_db(
db_session,
db_search_space,

4
surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py
View file

@ -47,7 +47,9 @@ async def test_comment_reply_truncates_long_preview(
db_session: AsyncSession, db_user: User, db_search_space: SearchSpace
):
"""A long comment preview is truncated in the reply message."""
notification = await _notify(db_session, db_user, db_search_space, preview="y" * 150)
notification = await _notify(
db_session, db_user, db_search_space, preview="y" * 150
)
assert notification.message == "y" * 100 + "..."

2
surfsense_backend/tests/integration/notifications/test_inbox_api.py
View file

@ -12,7 +12,7 @@ from datetime import UTC, datetime, timedelta
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import SearchSpace, User
from app.db import User
from app.notifications.persistence import Notification
pytestmark = pytest.mark.integration

4
surfsense_backend/tests/integration/notifications/test_mention_handler.py
View file

@ -47,7 +47,9 @@ async def test_new_mention_truncates_long_preview(
db_session: AsyncSession, db_user: User, db_search_space: SearchSpace
):
"""A long comment preview is truncated in the mention message."""
notification = await _notify(db_session, db_user, db_search_space, preview="x" * 150)
notification = await _notify(
db_session, db_user, db_search_space, preview="x" * 150
)
assert notification.message == "x" * 100 + "..."

5
surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py
View file

@ -137,7 +137,10 @@ class TestPluginLoaderIsolation:
_FakeEntryPoint("crashing", crashing_factory),
_FakeEntryPoint("ok", year_substituter_factory),
]
with patch("app.agents.chat.multi_agent_chat.main_agent.plugins.loader.entry_points", return_value=eps):
with patch(
"app.agents.chat.multi_agent_chat.main_agent.plugins.loader.entry_points",
return_value=eps,
):
result = load_plugin_middlewares(
_ctx(), allowed_plugin_names={"crashing", "ok"}
)

28
surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py
View file

@ -5,8 +5,7 @@ import asyncio
import pytest
import pytest_asyncio
from app.gateway import byo_long_poll
from app.gateway import runner
from app.gateway import byo_long_poll, runner
class ScalarResult:
@ -48,7 +47,9 @@ async def test_start_byo_long_poll_noops_when_mode_is_webhook(monkeypatch):
@pytest.mark.asyncio
async def test_start_byo_long_poll_noops_when_no_byo_accounts(mocker, monkeypatch):
monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll")
monkeypatch.setattr(
byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll"
)
session = mocker.AsyncMock()
session.execute.return_value = ScalarResult([])
monkeypatch.setattr(
@ -63,8 +64,12 @@ async def test_start_byo_long_poll_noops_when_no_byo_accounts(mocker, monkeypatc
@pytest.mark.asyncio
async def test_start_byo_long_poll_spawns_one_supervisor_per_account(mocker, monkeypatch):
monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll")
async def test_start_byo_long_poll_spawns_one_supervisor_per_account(
mocker, monkeypatch
):
monkeypatch.setattr(
byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll"
)
accounts = [mocker.Mock(id=1), mocker.Mock(id=2)]
session = mocker.AsyncMock()
session.execute.return_value = ScalarResult(accounts)
@ -73,7 +78,9 @@ async def test_start_byo_long_poll_spawns_one_supervisor_per_account(mocker, mon
"async_session_maker",
lambda: SessionContext(session),
)
monkeypatch.setattr(byo_long_poll, "account_token", lambda account: f"token-{account.id}")
monkeypatch.setattr(
byo_long_poll, "account_token", lambda account: f"token-{account.id}"
)
async def forever(_account_id: int, _token: str) -> None:
await asyncio.Event().wait()
@ -108,7 +115,9 @@ async def test_supervisor_retries_after_run_returns(mocker, monkeypatch):
@pytest.mark.asyncio
async def test_shutdown_cancels_running_supervisors(mocker, monkeypatch):
monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll")
monkeypatch.setattr(
byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll"
)
session = mocker.AsyncMock()
session.execute.return_value = ScalarResult([mocker.Mock(id=1)])
monkeypatch.setattr(
@ -130,7 +139,9 @@ async def test_shutdown_cancels_running_supervisors(mocker, monkeypatch):
@pytest.mark.asyncio
async def test_run_telegram_account_persists_for_fastapi_inbox_worker(mocker, monkeypatch):
async def test_run_telegram_account_persists_for_fastapi_inbox_worker(
mocker, monkeypatch
):
class ConnectionContext:
async def __aenter__(self):
conn = mocker.AsyncMock()
@ -169,4 +180,3 @@ async def test_run_telegram_account_persists_for_fastapi_inbox_worker(mocker, mo
second_session.commit.assert_awaited_once()
persist.assert_awaited_once()
assert persist.await_args.kwargs["request_id"].startswith("gateway_")

5
surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py
View file

@ -5,7 +5,9 @@ from app.tasks.celery_tasks import gateway_tasks
def test_enqueue_received_sweep_is_noop_guard(mocker):
apply_async = mocker.Mock()
mocker.patch.object(gateway_tasks.process_inbound_event_task, "apply_async", apply_async)
mocker.patch.object(
gateway_tasks.process_inbound_event_task, "apply_async", apply_async
)
info = mocker.patch.object(gateway_tasks.logger, "info")
replayed = gateway_tasks.enqueue_received_sweep_task.run()
@ -13,4 +15,3 @@ def test_enqueue_received_sweep_is_noop_guard(mocker):
apply_async.assert_not_called()
assert replayed == 0
info.assert_called_once()

1
surfsense_backend/tests/unit/gateway/test_formatting.py
View file

@ -15,4 +15,3 @@ def test_chunk_message_preserves_content_and_limits_size():
assert "".join(chunks) == text
assert len(chunks) > 1
assert all(len(chunk.encode("utf-16-le")) // 2 <= 4096 for chunk in chunks)

1
surfsense_backend/tests/unit/gateway/test_hitl_filter.py
View file

@ -12,4 +12,3 @@ def test_filter_hitl_tools_removes_known_approval_tools():
filtered = filter_hitl_tools(tools)
assert [getattr(tool, "name", tool) for tool in filtered] == ["search", "summarize"]

5
surfsense_backend/tests/unit/gateway/test_inbox_worker.py
View file

@ -8,7 +8,9 @@ from app.gateway import inbox_worker
@pytest.mark.asyncio
async def test_inbox_worker_claims_and_processes_in_fastapi_process(mocker, monkeypatch):
async def test_inbox_worker_claims_and_processes_in_fastapi_process(
mocker, monkeypatch
):
claim = mocker.AsyncMock(return_value=7)
process = mocker.AsyncMock(side_effect=asyncio.CancelledError)
monkeypatch.setattr(inbox_worker, "claim_next_inbound_event", claim)
@ -42,4 +44,3 @@ async def test_start_stop_gateway_inbox_worker(mocker, monkeypatch):
assert stopped.is_set()
assert inbox_worker._task is None

1
surfsense_backend/tests/unit/gateway/test_pairing.py
View file

@ -38,4 +38,3 @@ async def test_redeem_pairing_code_binds_pending_row(mocker):
assert binding.state == ExternalChatBindingState.BOUND
assert binding.external_peer_id == "telegram:123"
assert binding.pairing_code is None

5
surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py
View file

@ -9,5 +9,6 @@ def test_process_inbound_event_task_is_noop_guard(mocker):
assert gateway_tasks.process_inbound_event_task.run(123) is None
warning.assert_called_once()
assert "FastAPI owns external chat agent turn processing" in warning.call_args.args[0]
assert (
"FastAPI owns external chat agent turn processing" in warning.call_args.args[0]
)

29
surfsense_backend/tests/unit/gateway/test_webhook_routes.py
View file

@ -23,7 +23,9 @@ def _enable_gateways(monkeypatch):
monkeypatch.setattr(routes.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "webhook")
monkeypatch.setattr(routes.config, "TELEGRAM_SHARED_BOT_TOKEN", "telegram-token")
monkeypatch.setattr(routes.config, "TELEGRAM_SHARED_BOT_USERNAME", "surf_bot")
monkeypatch.setattr(routes.config, "TELEGRAM_WEBHOOK_SECRET", "telegram-webhook-secret")
monkeypatch.setattr(
routes.config, "TELEGRAM_WEBHOOK_SECRET", "telegram-webhook-secret"
)
monkeypatch.setattr(routes.config, "GATEWAY_SLACK_ENABLED", True)
monkeypatch.setattr(routes.config, "GATEWAY_SLACK_CLIENT_ID", "slack-client")
@ -37,7 +39,9 @@ def _enable_gateways(monkeypatch):
class RequestStub:
def __init__(self, payload=None, *, headers=None, json_exc: Exception | None = None):
def __init__(
self, payload=None, *, headers=None, json_exc: Exception | None = None
):
self.headers = headers or {}
self._payload = payload
self._json_exc = json_exc
@ -70,7 +74,9 @@ def _slack_account() -> ExternalChatAccount:
)
def _signed_slack_request(payload: dict, *, secret: str = "signing-secret") -> RequestStub:
def _signed_slack_request(
payload: dict, *, secret: str = "signing-secret"
) -> RequestStub:
body = json.dumps(payload).encode()
timestamp = str(int(time.time()))
digest = hmac.new(
@ -195,7 +201,9 @@ async def test_telegram_webhook_persists_for_fastapi_inbox_worker(mocker, monkey
async def test_telegram_webhook_commits_dedup_without_enqueue(mocker, monkeypatch):
session = mocker.AsyncMock()
session.get.return_value = _account()
monkeypatch.setattr(routes, "persist_inbound_event", mocker.AsyncMock(return_value=None))
monkeypatch.setattr(
routes, "persist_inbound_event", mocker.AsyncMock(return_value=None)
)
request = RequestStub(
{"update_id": 10, "message": {"message_id": 7}},
@ -250,7 +258,11 @@ async def test_slack_webhook_url_verification(monkeypatch, mocker):
async def test_slack_webhook_persists_event(monkeypatch, mocker):
_enable_slack_gateway(monkeypatch)
session = mocker.AsyncMock()
monkeypatch.setattr(routes, "get_slack_account_by_team", mocker.AsyncMock(return_value=_slack_account()))
monkeypatch.setattr(
routes,
"get_slack_account_by_team",
mocker.AsyncMock(return_value=_slack_account()),
)
persist = mocker.AsyncMock(return_value=100)
monkeypatch.setattr(routes, "persist_inbound_event", persist)
payload = {
@ -280,7 +292,11 @@ async def test_slack_webhook_persists_event(monkeypatch, mocker):
async def test_slack_webhook_ignores_self_event(monkeypatch, mocker):
_enable_slack_gateway(monkeypatch)
session = mocker.AsyncMock()
monkeypatch.setattr(routes, "get_slack_account_by_team", mocker.AsyncMock(return_value=_slack_account()))
monkeypatch.setattr(
routes,
"get_slack_account_by_team",
mocker.AsyncMock(return_value=_slack_account()),
)
persist = mocker.AsyncMock(return_value=100)
monkeypatch.setattr(routes, "persist_inbound_event", persist)
request = _signed_slack_request(
@ -331,4 +347,3 @@ def test_discord_gateway_callback_does_not_create_search_source_connector():
callback_source = inspect.getsource(routes.discord_gateway_callback)
assert "SearchSourceConnector" not in callback_source

10
surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py
View file

@ -81,7 +81,9 @@ class TestCwdDefaults:
class TestRelativePathResolution:
def test_relative_path_resolves_against_cwd(self):
assert (
resolve_relative(_mw(), "notes.md", _runtime({"cwd": "/documents/projects"}))
resolve_relative(
_mw(), "notes.md", _runtime({"cwd": "/documents/projects"})
)
== "/documents/projects/notes.md"
)
@ -281,7 +283,11 @@ class TestNormalizeLocalMountPath:
_desktop_mw(backend),
"/brand-new-note.md",
_runtime(
{"file_operation_contract": {"suggested_path": "/root_b/notes/context.md"}}
{
"file_operation_contract": {
"suggested_path": "/root_b/notes/context.md"
}
}
),
)
assert resolved == "/root_b/brand-new-note.md"

4
surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py
View file

@ -15,7 +15,9 @@ from unittest.mock import AsyncMock
import numpy as np
import pytest
from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence import middleware as kb_persistence
from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence import (
middleware as kb_persistence,
)
from app.db import Document

Some files were not shown because too many files have changed in this diff Show more