diff --git a/surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py b/surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py index 1b25be753..e23f3a371 100644 --- a/surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py +++ b/surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py @@ -165,9 +165,7 @@ def downgrade() -> None: tx = conn.begin_nested() if conn.in_transaction() else conn.begin() with tx: conn.execute( - sa.text( - f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-148-downgrade'" - ) + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-148-downgrade'") ) conn.execute(sa.text(ddl)) conn.execute( diff --git a/surfsense_backend/alembic/versions/149_add_gateway_tables.py b/surfsense_backend/alembic/versions/149_add_gateway_tables.py index a77e6a69b..eee4a45b6 100644 --- a/surfsense_backend/alembic/versions/149_add_gateway_tables.py +++ b/surfsense_backend/alembic/versions/149_add_gateway_tables.py @@ -65,6 +65,7 @@ AUTOMATION_RUN_COLS = [ "created_at", ] + def _has_zero_version(conn, table: str) -> bool: return ( conn.execute( @@ -190,7 +191,8 @@ def upgrade() -> None: "external_chat_peer_kind", ("direct", "group", "channel", "unknown") ) external_chat_event_kind_enum = _create_enum( - "external_chat_event_kind", ("message", "edited_message", "callback_query", "other") + "external_chat_event_kind", + ("message", "edited_message", "callback_query", "other"), ) external_chat_event_status_enum = _create_enum( "external_chat_event_status", @@ -205,7 +207,12 @@ def upgrade() -> None: sa.Column("mode", external_chat_account_mode_enum, nullable=False), sa.Column("owner_user_id", postgresql.UUID(as_uuid=True), nullable=True), sa.Column("owner_search_space_id", sa.Integer(), nullable=True), - sa.Column("is_system_account", sa.Boolean(), nullable=False, server_default="false"), + sa.Column( + "is_system_account", + sa.Boolean(), + nullable=False, + server_default="false", + ), sa.Column("encrypted_credentials", sa.Text(), nullable=True), sa.Column("bot_username", sa.String(255), nullable=True), sa.Column("webhook_secret", sa.String(64), nullable=True), @@ -221,7 +228,9 @@ def upgrade() -> None: nullable=False, server_default="unknown", ), - sa.Column("last_health_check_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column( + "last_health_check_at", sa.TIMESTAMP(timezone=True), nullable=True + ), sa.Column("suspended_at", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column("suspended_reason", sa.Text(), nullable=True), sa.Column( @@ -285,7 +294,9 @@ def upgrade() -> None: server_default="pending", ), sa.Column("pairing_code", sa.Text(), nullable=True), - sa.Column("pairing_code_expires_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column( + "pairing_code_expires_at", sa.TIMESTAMP(timezone=True), nullable=True + ), sa.Column("external_peer_id", sa.Text(), nullable=True), sa.Column( "external_peer_kind", @@ -327,7 +338,9 @@ def upgrade() -> None: ["account_id"], ["external_chat_accounts.id"], ondelete="CASCADE" ), sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["search_space_id"], ["searchspaces.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["search_space_id"], ["searchspaces.id"], ondelete="CASCADE" + ), sa.ForeignKeyConstraint( ["new_chat_thread_id"], ["new_chat_threads.id"], ondelete="SET NULL" ), @@ -386,7 +399,9 @@ def upgrade() -> None: nullable=False, server_default="received", ), - sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column( + "attempt_count", sa.Integer(), nullable=False, server_default="0" + ), sa.Column("last_error", sa.Text(), nullable=True), sa.Column( "received_at", @@ -405,7 +420,9 @@ def upgrade() -> None: ["account_id"], ["external_chat_accounts.id"], ondelete="CASCADE" ), sa.ForeignKeyConstraint( - ["external_chat_binding_id"], ["external_chat_bindings.id"], ondelete="SET NULL" + ["external_chat_binding_id"], + ["external_chat_bindings.id"], + ondelete="SET NULL", ), sa.UniqueConstraint( "account_id", @@ -445,7 +462,9 @@ def upgrade() -> None: sa.Column("external_chat_binding_id", sa.BigInteger(), nullable=True), ) if not _constraint_exists( - conn, "new_chat_threads", "fk_new_chat_threads_external_chat_external_chat_binding_id" + conn, + "new_chat_threads", + "fk_new_chat_threads_external_chat_external_chat_binding_id", ): op.create_foreign_key( "fk_new_chat_threads_external_chat_external_chat_binding_id", @@ -455,7 +474,9 @@ def upgrade() -> None: ["id"], ondelete="SET NULL", ) - op.create_index("ix_new_chat_threads_source", "new_chat_threads", ["source"], if_not_exists=True) + op.create_index( + "ix_new_chat_threads_source", "new_chat_threads", ["source"], if_not_exists=True + ) op.create_index( "ix_new_chat_threads_external_chat_binding_id", "new_chat_threads", @@ -472,7 +493,11 @@ def upgrade() -> None: if not _column_exists(conn, "new_chat_messages", "platform_metadata"): op.add_column( "new_chat_messages", - sa.Column("platform_metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column( + "platform_metadata", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + ), ) op.create_index( "ix_new_chat_messages_source", @@ -553,11 +578,15 @@ def downgrade() -> None: tx = conn.begin_nested() if conn.in_transaction() else conn.begin() with tx: conn.execute( - sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-144-downgrade'") + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-144-downgrade'" + ) ) conn.execute(sa.text(ddl)) conn.execute( - sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-144-downgrade'") + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-144-downgrade'" + ) ) if _column_exists(conn, "new_chat_messages", "source"): @@ -567,10 +596,14 @@ def downgrade() -> None: _drop_column_if_exists("new_chat_messages", "platform_metadata") _drop_column_if_exists("new_chat_messages", "source") - _drop_index_if_exists("ix_new_chat_threads_external_chat_binding_id", "new_chat_threads") + _drop_index_if_exists( + "ix_new_chat_threads_external_chat_binding_id", "new_chat_threads" + ) _drop_index_if_exists("ix_new_chat_threads_source", "new_chat_threads") if _constraint_exists( - conn, "new_chat_threads", "fk_new_chat_threads_external_chat_external_chat_binding_id" + conn, + "new_chat_threads", + "fk_new_chat_threads_external_chat_external_chat_binding_id", ): op.drop_constraint( "fk_new_chat_threads_external_chat_external_chat_binding_id", @@ -583,8 +616,12 @@ def downgrade() -> None: _drop_index_if_exists( "ix_external_chat_inbound_binding_received_at", "external_chat_inbound_events" ) - _drop_index_if_exists("ix_external_chat_inbound_request_id", "external_chat_inbound_events") - _drop_index_if_exists("ix_external_chat_inbound_status_received_at", "external_chat_inbound_events") + _drop_index_if_exists( + "ix_external_chat_inbound_request_id", "external_chat_inbound_events" + ) + _drop_index_if_exists( + "ix_external_chat_inbound_status_received_at", "external_chat_inbound_events" + ) if _table_exists(conn, "external_chat_inbound_events"): op.drop_table("external_chat_inbound_events") @@ -606,9 +643,15 @@ def downgrade() -> None: if _table_exists(conn, "external_chat_bindings"): op.drop_table("external_chat_bindings") - _drop_index_if_exists("uq_external_chat_accounts_system_platform", "external_chat_accounts") - _drop_index_if_exists("uq_external_chat_accounts_owner_platform", "external_chat_accounts") - _drop_index_if_exists("uq_external_chat_accounts_webhook_secret", "external_chat_accounts") + _drop_index_if_exists( + "uq_external_chat_accounts_system_platform", "external_chat_accounts" + ) + _drop_index_if_exists( + "uq_external_chat_accounts_owner_platform", "external_chat_accounts" + ) + _drop_index_if_exists( + "uq_external_chat_accounts_webhook_secret", "external_chat_accounts" + ) if _table_exists(conn, "external_chat_accounts"): op.drop_table("external_chat_accounts") diff --git a/surfsense_backend/alembic/versions/152_add_document_files.py b/surfsense_backend/alembic/versions/152_add_document_files.py index e04c43c49..034399640 100644 --- a/surfsense_backend/alembic/versions/152_add_document_files.py +++ b/surfsense_backend/alembic/versions/152_add_document_files.py @@ -63,8 +63,7 @@ def upgrade() -> None: "ON document_files(search_space_id);" ) op.execute( - "CREATE INDEX IF NOT EXISTS ix_document_files_kind " - "ON document_files(kind);" + "CREATE INDEX IF NOT EXISTS ix_document_files_kind ON document_files(kind);" ) op.execute( "CREATE INDEX IF NOT EXISTS ix_document_files_created_by_id " diff --git a/surfsense_backend/alembic/versions/153_restore_automation_runs_to_zero_publication.py b/surfsense_backend/alembic/versions/153_restore_automation_runs_to_zero_publication.py index ebcf6e00c..320e75465 100644 --- a/surfsense_backend/alembic/versions/153_restore_automation_runs_to_zero_publication.py +++ b/surfsense_backend/alembic/versions/153_restore_automation_runs_to_zero_publication.py @@ -68,8 +68,12 @@ def _has_zero_version(conn, table: str) -> bool: def _set_table_ddl(*, with_automation_runs: bool, conn) -> str: - doc_cols = DOCUMENT_COLS + (['"_0_version"'] if _has_zero_version(conn, "documents") else []) - user_cols = USER_COLS + (['"_0_version"'] if _has_zero_version(conn, "user") else []) + doc_cols = DOCUMENT_COLS + ( + ['"_0_version"'] if _has_zero_version(conn, "documents") else [] + ) + user_cols = USER_COLS + ( + ['"_0_version"'] if _has_zero_version(conn, "user") else [] + ) tables = [ "notifications", f"documents ({', '.join(doc_cols)})", @@ -96,9 +100,17 @@ def _resync(*, with_automation_runs: bool, tag: str) -> None: tx = conn.begin_nested() if conn.in_transaction() else conn.begin() with tx: - conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'")) - conn.execute(sa.text(_set_table_ddl(with_automation_runs=with_automation_runs, conn=conn))) - conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'")) + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'") + ) + conn.execute( + sa.text( + _set_table_ddl(with_automation_runs=with_automation_runs, conn=conn) + ) + ) + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'") + ) def upgrade() -> None: diff --git a/surfsense_backend/alembic/versions/154_remove_document_summary_llm.py b/surfsense_backend/alembic/versions/154_remove_document_summary_llm.py index 6d0eb45cf..94f1bf4ce 100644 --- a/surfsense_backend/alembic/versions/154_remove_document_summary_llm.py +++ b/surfsense_backend/alembic/versions/154_remove_document_summary_llm.py @@ -67,8 +67,12 @@ def _has_zero_version(conn, table: str) -> bool: def _set_table_ddl(conn) -> str: - doc_cols = DOCUMENT_COLS + (['"_0_version"'] if _has_zero_version(conn, "documents") else []) - user_cols = USER_COLS + (['"_0_version"'] if _has_zero_version(conn, "user") else []) + doc_cols = DOCUMENT_COLS + ( + ['"_0_version"'] if _has_zero_version(conn, "documents") else [] + ) + user_cols = USER_COLS + ( + ['"_0_version"'] if _has_zero_version(conn, "user") else [] + ) tables = [ "notifications", f"documents ({', '.join(doc_cols)})", @@ -94,9 +98,13 @@ def _resync_zero_publication(tag: str) -> None: tx = conn.begin_nested() if conn.in_transaction() else conn.begin() with tx: - conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'")) + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'") + ) conn.execute(sa.text(_set_table_ddl(conn))) - conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'")) + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'") + ) def upgrade() -> None: @@ -117,7 +125,12 @@ def downgrade() -> None: if not _column_exists(conn, "searchspaces", "document_summary_llm_id"): op.add_column( "searchspaces", - sa.Column("document_summary_llm_id", sa.Integer(), nullable=True, server_default="0"), + sa.Column( + "document_summary_llm_id", + sa.Integer(), + nullable=True, + server_default="0", + ), ) if not _column_exists(conn, "search_source_connectors", "enable_summary"): diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py index 1aff733b2..9898d15f4 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py @@ -40,44 +40,149 @@ class ToolMetadata: # up in the UI tool picker. This list carries metadata only — wire the actual # implementation in the relevant builder/registry module. TOOL_CATALOG: list[ToolMetadata] = [ - ToolMetadata(name="generate_podcast", description="Generate an audio podcast from provided content"), - ToolMetadata(name="generate_video_presentation", description="Generate a video presentation with slides and narration from provided content"), - ToolMetadata(name="generate_report", description="Generate a structured report from provided content and export it"), - ToolMetadata(name="generate_resume", description="Generate a professional resume as a Typst document"), - ToolMetadata(name="generate_image", description="Generate images from text descriptions using AI image models"), - ToolMetadata(name="scrape_webpage", description="Scrape and extract the main content from a webpage"), - ToolMetadata(name="web_search", description="Search the web for real-time information using configured search engines"), - ToolMetadata(name="create_automation", description="Draft an automation from an NL intent; user approves the card; tool saves"), - ToolMetadata(name="update_memory", description="Save important long-term facts, preferences, and instructions to the (personal or team) memory"), - ToolMetadata(name="create_notion_page", description="Create a new page in the user's Notion workspace"), - ToolMetadata(name="update_notion_page", description="Append new content to an existing Notion page"), - ToolMetadata(name="delete_notion_page", description="Delete an existing Notion page"), - ToolMetadata(name="create_google_drive_file", description="Create a new Google Doc or Google Sheet in Google Drive"), - ToolMetadata(name="delete_google_drive_file", description="Move an indexed Google Drive file to trash"), - ToolMetadata(name="create_dropbox_file", description="Create a new file in Dropbox"), + ToolMetadata( + name="generate_podcast", + description="Generate an audio podcast from provided content", + ), + ToolMetadata( + name="generate_video_presentation", + description="Generate a video presentation with slides and narration from provided content", + ), + ToolMetadata( + name="generate_report", + description="Generate a structured report from provided content and export it", + ), + ToolMetadata( + name="generate_resume", + description="Generate a professional resume as a Typst document", + ), + ToolMetadata( + name="generate_image", + description="Generate images from text descriptions using AI image models", + ), + ToolMetadata( + name="scrape_webpage", + description="Scrape and extract the main content from a webpage", + ), + ToolMetadata( + name="web_search", + description="Search the web for real-time information using configured search engines", + ), + ToolMetadata( + name="create_automation", + description="Draft an automation from an NL intent; user approves the card; tool saves", + ), + ToolMetadata( + name="update_memory", + description="Save important long-term facts, preferences, and instructions to the (personal or team) memory", + ), + ToolMetadata( + name="create_notion_page", + description="Create a new page in the user's Notion workspace", + ), + ToolMetadata( + name="update_notion_page", + description="Append new content to an existing Notion page", + ), + ToolMetadata( + name="delete_notion_page", description="Delete an existing Notion page" + ), + ToolMetadata( + name="create_google_drive_file", + description="Create a new Google Doc or Google Sheet in Google Drive", + ), + ToolMetadata( + name="delete_google_drive_file", + description="Move an indexed Google Drive file to trash", + ), + ToolMetadata( + name="create_dropbox_file", description="Create a new file in Dropbox" + ), ToolMetadata(name="delete_dropbox_file", description="Delete a file from Dropbox"), - ToolMetadata(name="create_onedrive_file", description="Create a new file in Microsoft OneDrive"), - ToolMetadata(name="delete_onedrive_file", description="Move a OneDrive file to the recycle bin"), - ToolMetadata(name="search_calendar_events", description="Search Google Calendar events within a date range"), - ToolMetadata(name="create_calendar_event", description="Create a new event on Google Calendar"), - ToolMetadata(name="update_calendar_event", description="Update an existing indexed Google Calendar event"), - ToolMetadata(name="delete_calendar_event", description="Delete an existing indexed Google Calendar event"), - ToolMetadata(name="search_gmail", description="Search emails in Gmail using Gmail search syntax"), - ToolMetadata(name="read_gmail_email", description="Read the full content of a specific Gmail email"), - ToolMetadata(name="create_gmail_draft", description="Create a draft email in Gmail"), + ToolMetadata( + name="create_onedrive_file", + description="Create a new file in Microsoft OneDrive", + ), + ToolMetadata( + name="delete_onedrive_file", + description="Move a OneDrive file to the recycle bin", + ), + ToolMetadata( + name="search_calendar_events", + description="Search Google Calendar events within a date range", + ), + ToolMetadata( + name="create_calendar_event", + description="Create a new event on Google Calendar", + ), + ToolMetadata( + name="update_calendar_event", + description="Update an existing indexed Google Calendar event", + ), + ToolMetadata( + name="delete_calendar_event", + description="Delete an existing indexed Google Calendar event", + ), + ToolMetadata( + name="search_gmail", + description="Search emails in Gmail using Gmail search syntax", + ), + ToolMetadata( + name="read_gmail_email", + description="Read the full content of a specific Gmail email", + ), + ToolMetadata( + name="create_gmail_draft", description="Create a draft email in Gmail" + ), ToolMetadata(name="send_gmail_email", description="Send an email via Gmail"), - ToolMetadata(name="trash_gmail_email", description="Move an indexed email to trash in Gmail"), - ToolMetadata(name="update_gmail_draft", description="Update an existing Gmail draft"), - ToolMetadata(name="create_confluence_page", description="Create a new page in the user's Confluence space"), - ToolMetadata(name="update_confluence_page", description="Update an existing indexed Confluence page"), - ToolMetadata(name="delete_confluence_page", description="Delete an existing indexed Confluence page"), - ToolMetadata(name="list_discord_channels", description="List text channels in the connected Discord server"), - ToolMetadata(name="read_discord_messages", description="Read recent messages from a Discord text channel"), - ToolMetadata(name="send_discord_message", description="Send a message to a Discord text channel"), - ToolMetadata(name="list_teams_channels", description="List Microsoft Teams and their channels"), - ToolMetadata(name="read_teams_messages", description="Read recent messages from a Microsoft Teams channel"), - ToolMetadata(name="send_teams_message", description="Send a message to a Microsoft Teams channel"), - ToolMetadata(name="list_luma_events", description="List upcoming and recent Luma events"), - ToolMetadata(name="read_luma_event", description="Read detailed information about a specific Luma event"), + ToolMetadata( + name="trash_gmail_email", description="Move an indexed email to trash in Gmail" + ), + ToolMetadata( + name="update_gmail_draft", description="Update an existing Gmail draft" + ), + ToolMetadata( + name="create_confluence_page", + description="Create a new page in the user's Confluence space", + ), + ToolMetadata( + name="update_confluence_page", + description="Update an existing indexed Confluence page", + ), + ToolMetadata( + name="delete_confluence_page", + description="Delete an existing indexed Confluence page", + ), + ToolMetadata( + name="list_discord_channels", + description="List text channels in the connected Discord server", + ), + ToolMetadata( + name="read_discord_messages", + description="Read recent messages from a Discord text channel", + ), + ToolMetadata( + name="send_discord_message", + description="Send a message to a Discord text channel", + ), + ToolMetadata( + name="list_teams_channels", + description="List Microsoft Teams and their channels", + ), + ToolMetadata( + name="read_teams_messages", + description="Read recent messages from a Microsoft Teams channel", + ), + ToolMetadata( + name="send_teams_message", + description="Send a message to a Microsoft Teams channel", + ), + ToolMetadata( + name="list_luma_events", description="List upcoming and recent Luma events" + ), + ToolMetadata( + name="read_luma_event", + description="Read detailed information about a specific Luma event", + ), ToolMetadata(name="create_luma_event", description="Create a new event on Luma"), ] diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index a9f577c7f..75af17d11 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -308,12 +308,8 @@ def load_openrouter_integration_settings() -> dict | None: "anonymous_enabled_free instead. Both new flags have been " "seeded from the legacy value for back-compat." ) - settings.setdefault( - "anonymous_enabled_paid", settings["anonymous_enabled"] - ) - settings.setdefault( - "anonymous_enabled_free", settings["anonymous_enabled"] - ) + settings.setdefault("anonymous_enabled_paid", settings["anonymous_enabled"]) + settings.setdefault("anonymous_enabled_free", settings["anonymous_enabled"]) # Image generation + vision LLM emission are opt-in (issue L). # OpenRouter's catalogue contains hundreds of image / vision @@ -622,7 +618,9 @@ class Config: WHATSAPP_GRAPH_API_VERSION = os.getenv("WHATSAPP_GRAPH_API_VERSION", "v25.0") WHATSAPP_WEBHOOK_VERIFY_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFY_TOKEN") WHATSAPP_WEBHOOK_APP_SECRET = os.getenv("WHATSAPP_WEBHOOK_APP_SECRET") - WHATSAPP_BRIDGE_URL = os.getenv("WHATSAPP_BRIDGE_URL", "http://whatsapp-bridge:9929") + WHATSAPP_BRIDGE_URL = os.getenv( + "WHATSAPP_BRIDGE_URL", "http://whatsapp-bridge:9929" + ) GATEWAY_WHATSAPP_INTAKE_MODE = os.getenv( "GATEWAY_WHATSAPP_INTAKE_MODE", "disabled" ).lower() @@ -632,7 +630,9 @@ class Config: ) GATEWAY_SLACK_CLIENT_ID = os.getenv("SLACK_CLIENT_ID") GATEWAY_SLACK_CLIENT_SECRET = os.getenv("SLACK_CLIENT_SECRET") - GATEWAY_SLACK_ENABLED = os.getenv("GATEWAY_SLACK_ENABLED", "FALSE").upper() == "TRUE" + GATEWAY_SLACK_ENABLED = ( + os.getenv("GATEWAY_SLACK_ENABLED", "FALSE").upper() == "TRUE" + ) GATEWAY_SLACK_SIGNING_SECRET = os.getenv("GATEWAY_SLACK_SIGNING_SECRET") GATEWAY_SLACK_REDIRECT_URI = os.getenv("GATEWAY_SLACK_REDIRECT_URI") GATEWAY_DISCORD_ENABLED = ( diff --git a/surfsense_backend/app/connectors/webcrawler_connector.py b/surfsense_backend/app/connectors/webcrawler_connector.py index ff655c064..1d4ff1f58 100644 --- a/surfsense_backend/app/connectors/webcrawler_connector.py +++ b/surfsense_backend/app/connectors/webcrawler_connector.py @@ -105,14 +105,18 @@ class WebCrawlerConnector: logger.info(f"[webcrawler] Using Scrapling AsyncFetcher for: {url}") result = await self._crawl_with_async_fetcher(url) if result: - self._log_tier_outcome("scrapling-static", url, tier_start, "success") + self._log_tier_outcome( + "scrapling-static", url, tier_start, "success" + ) self._log_total(url, "scrapling-static", total_start) return result, None errors.append("Scrapling static: empty extraction") self._log_tier_outcome("scrapling-static", url, tier_start, "empty") except Exception as exc: errors.append(f"Scrapling static: {exc!s}") - self._log_tier_outcome("scrapling-static", url, tier_start, "error", exc) + self._log_tier_outcome( + "scrapling-static", url, tier_start, "error", exc + ) # --- 3. Scrapling DynamicFetcher (full browser) --- tier_start = time.perf_counter() @@ -120,7 +124,9 @@ class WebCrawlerConnector: logger.info(f"[webcrawler] Using Scrapling DynamicFetcher for: {url}") result = await self._crawl_with_dynamic(url) if result: - self._log_tier_outcome("scrapling-dynamic", url, tier_start, "success") + self._log_tier_outcome( + "scrapling-dynamic", url, tier_start, "success" + ) self._log_total(url, "scrapling-dynamic", total_start) return result, None errors.append("Scrapling dynamic: empty extraction") @@ -135,7 +141,9 @@ class WebCrawlerConnector: ) except Exception as exc: errors.append(f"Scrapling dynamic: {exc!s}") - self._log_tier_outcome("scrapling-dynamic", url, tier_start, "error", exc) + self._log_tier_outcome( + "scrapling-dynamic", url, tier_start, "error", exc + ) # --- 4. Scrapling StealthyFetcher (anti-bot, last resort) --- tier_start = time.perf_counter() @@ -143,7 +151,9 @@ class WebCrawlerConnector: logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}") result = await self._crawl_with_stealthy(url) if result: - self._log_tier_outcome("scrapling-stealthy", url, tier_start, "success") + self._log_tier_outcome( + "scrapling-stealthy", url, tier_start, "success" + ) self._log_total(url, "scrapling-stealthy", total_start) return result, None errors.append("Scrapling stealthy: empty extraction") diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 5b232b55c..6117caecb 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -714,7 +714,9 @@ class NewChatThread(BaseModel, TimestampMixin): # Surface metadata for first-party SurfSense and external chat threads. # Zero publishes all chat-message sources; the UI can decide which surfaces to render. - source = Column(Text, nullable=False, default="surfsense", server_default="surfsense") + source = Column( + Text, nullable=False, default="surfsense", server_default="surfsense" + ) external_chat_binding_id = Column( BigInteger, ForeignKey("external_chat_bindings.id", ondelete="SET NULL"), @@ -802,7 +804,9 @@ class NewChatMessage(BaseModel, TimestampMixin): # Mirrors the parent thread source for publication-level filtering. # This denormalization avoids join-dependent logical replication rules. - source = Column(Text, nullable=False, default="surfsense", server_default="surfsense") + source = Column( + Text, nullable=False, default="surfsense", server_default="surfsense" + ) platform_metadata = Column(JSONB, nullable=True) # Relationships @@ -848,11 +852,15 @@ class ExternalChatAccount(Base, TimestampMixin): owner_search_space_id = Column( Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=True ) - is_system_account = Column(Boolean, nullable=False, default=False, server_default="false") + is_system_account = Column( + Boolean, nullable=False, default=False, server_default="false" + ) encrypted_credentials = Column(Text, nullable=True) bot_username = Column(String(255), nullable=True) webhook_secret = Column(String(64), nullable=True) - cursor_state = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")) + cursor_state = Column( + JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb") + ) health_status = Column( SQLAlchemyEnum( ExternalChatHealthStatus, @@ -875,7 +883,9 @@ class ExternalChatAccount(Base, TimestampMixin): ) owner = relationship("User", foreign_keys=[owner_user_id]) - owner_search_space = relationship("SearchSpace", foreign_keys=[owner_search_space_id]) + owner_search_space = relationship( + "SearchSpace", foreign_keys=[owner_search_space_id] + ) bindings = relationship( "ExternalChatBinding", back_populates="account", @@ -980,7 +990,9 @@ class ExternalChatBinding(Base, TimestampMixin): external_thread_id = Column(Text, nullable=True) external_display_name = Column(Text, nullable=True) external_username = Column(Text, nullable=True) - external_metadata = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")) + external_metadata = Column( + JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb") + ) new_chat_thread_id = Column( Integer, ForeignKey("new_chat_threads.id", ondelete="SET NULL"), @@ -1030,7 +1042,9 @@ class ExternalChatBinding(Base, TimestampMixin): postgresql_where=text("state = 'pending'"), ), Index("ix_external_chat_bindings_user_state", "user_id", "state"), - Index("ix_external_chat_bindings_search_space_state", "search_space_id", "state"), + Index( + "ix_external_chat_bindings_search_space_state", "search_space_id", "state" + ), ) diff --git a/surfsense_backend/app/file_storage/backends/azure.py b/surfsense_backend/app/file_storage/backends/azure.py index ec59525e3..b82a47f0c 100644 --- a/surfsense_backend/app/file_storage/backends/azure.py +++ b/surfsense_backend/app/file_storage/backends/azure.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib from collections.abc import AsyncIterator from app.file_storage.backends.base import StorageBackend @@ -43,10 +44,8 @@ class AzureBlobBackend(StorageBackend): async with self._service() as service: blob = service.get_blob_client(self._container, key) - try: + with contextlib.suppress(ResourceNotFoundError): await blob.delete_blob() - except ResourceNotFoundError: - pass async def exists(self, key: str) -> bool: async with self._service() as service: diff --git a/surfsense_backend/app/file_storage/backends/local.py b/surfsense_backend/app/file_storage/backends/local.py index c55bd901a..68bb8facd 100644 --- a/surfsense_backend/app/file_storage/backends/local.py +++ b/surfsense_backend/app/file_storage/backends/local.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import contextlib from collections.abc import AsyncIterator from pathlib import Path @@ -53,10 +54,8 @@ class LocalFileBackend(StorageBackend): path = self._path_for(key) def _unlink() -> None: - try: + with contextlib.suppress(FileNotFoundError): path.unlink() - except FileNotFoundError: - pass await asyncio.to_thread(_unlink) diff --git a/surfsense_backend/app/gateway/accounts.py b/surfsense_backend/app/gateway/accounts.py index 2d924e200..8cbf748b6 100644 --- a/surfsense_backend/app/gateway/accounts.py +++ b/surfsense_backend/app/gateway/accounts.py @@ -31,7 +31,9 @@ def slack_account_credentials(account: ExternalChatAccount) -> dict: """Decrypt Slack gateway credentials stored as encrypted JSON.""" if not account.encrypted_credentials: return {} - raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token(account.encrypted_credentials) + raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token( + account.encrypted_credentials + ) try: data = json.loads(raw) except json.JSONDecodeError: @@ -44,7 +46,9 @@ def discord_account_credentials(account: ExternalChatAccount) -> dict: """Decrypt Discord gateway credentials stored as encrypted JSON.""" if not account.encrypted_credentials: return {} - raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token(account.encrypted_credentials) + raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token( + account.encrypted_credentials + ) try: data = json.loads(raw) except json.JSONDecodeError: @@ -135,4 +139,3 @@ async def get_discord_account_by_guild( ) ) return result.scalars().first() - diff --git a/surfsense_backend/app/gateway/agent_invoke.py b/surfsense_backend/app/gateway/agent_invoke.py index dcbf9a954..8701ccc55 100644 --- a/surfsense_backend/app/gateway/agent_invoke.py +++ b/surfsense_backend/app/gateway/agent_invoke.py @@ -21,7 +21,9 @@ from app.tasks.chat.streaming.flows import stream_new_chat logger = logging.getLogger(__name__) -async def _events_from_sse(chunks: AsyncIterator[str]) -> AsyncIterator[GatewayStreamEvent]: +async def _events_from_sse( + chunks: AsyncIterator[str], +) -> AsyncIterator[GatewayStreamEvent]: saw_text = False async for chunk in chunks: for raw_line in chunk.splitlines(): @@ -98,4 +100,3 @@ async def call_agent_for_gateway( record_gateway_turn_latency(0, platform=platform_label) finally: release_thread_lock(thread.id) - diff --git a/surfsense_backend/app/gateway/auth_invariant.py b/surfsense_backend/app/gateway/auth_invariant.py index fba38f64e..e72023ce1 100644 --- a/surfsense_backend/app/gateway/auth_invariant.py +++ b/surfsense_backend/app/gateway/auth_invariant.py @@ -52,4 +52,3 @@ async def assert_authorization_invariant( await _fail(session, binding, f"rbac_{exc.status_code}") return user - diff --git a/surfsense_backend/app/gateway/base/__init__.py b/surfsense_backend/app/gateway/base/__init__.py index 962d068b6..f27e3e087 100644 --- a/surfsense_backend/app/gateway/base/__init__.py +++ b/surfsense_backend/app/gateway/base/__init__.py @@ -1,2 +1 @@ """Base gateway interfaces.""" - diff --git a/surfsense_backend/app/gateway/base/adapter.py b/surfsense_backend/app/gateway/base/adapter.py index caf351c05..dfe896b4a 100644 --- a/surfsense_backend/app/gateway/base/adapter.py +++ b/surfsense_backend/app/gateway/base/adapter.py @@ -62,9 +62,10 @@ class BasePlatformAdapter(ABC): async def validate_credentials(self) -> dict[str, Any]: """Validate configured credentials and return account metadata.""" - async def fetch_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]: + async def fetch_updates( + self, *, offset: int | None + ) -> AsyncIterator[dict[str, Any]]: """Yield provider updates for long-polling adapters.""" if False: yield {} # pragma: no cover raise NotImplementedError("This adapter does not support long-polling") - diff --git a/surfsense_backend/app/gateway/base/identity.py b/surfsense_backend/app/gateway/base/identity.py index 608ae41c1..e445bcbbb 100644 --- a/surfsense_backend/app/gateway/base/identity.py +++ b/surfsense_backend/app/gateway/base/identity.py @@ -16,4 +16,3 @@ def hash_external_id(value: str | int | None) -> str | None: if not normalized: return None return hashlib.sha256(normalized.encode("utf-8")).hexdigest() - diff --git a/surfsense_backend/app/gateway/base/translator.py b/surfsense_backend/app/gateway/base/translator.py index af72188e9..2476adb2c 100644 --- a/surfsense_backend/app/gateway/base/translator.py +++ b/surfsense_backend/app/gateway/base/translator.py @@ -25,4 +25,3 @@ class BaseStreamTranslator(ABC): @abstractmethod async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: """Consume agent stream events and emit platform messages.""" - diff --git a/surfsense_backend/app/gateway/bindings.py b/surfsense_backend/app/gateway/bindings.py index 971633571..821dd21ca 100644 --- a/surfsense_backend/app/gateway/bindings.py +++ b/surfsense_backend/app/gateway/bindings.py @@ -64,4 +64,3 @@ def resume_binding(binding: ExternalChatBinding) -> None: binding.state = ExternalChatBindingState.BOUND binding.suspended_at = None binding.suspended_reason = None - diff --git a/surfsense_backend/app/gateway/byo_long_poll.py b/surfsense_backend/app/gateway/byo_long_poll.py index 29a0ed48c..e3f3ec093 100644 --- a/surfsense_backend/app/gateway/byo_long_poll.py +++ b/surfsense_backend/app/gateway/byo_long_poll.py @@ -58,8 +58,10 @@ async def _whatsapp_baileys_supervisor() -> None: async with async_session_maker() as session: result = await session.execute( select(ExternalChatAccount).where( - ExternalChatAccount.platform == ExternalChatPlatform.WHATSAPP, - ExternalChatAccount.mode == ExternalChatAccountMode.SELF_HOST_BYO, + ExternalChatAccount.platform + == ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.mode + == ExternalChatAccountMode.SELF_HOST_BYO, ExternalChatAccount.is_system_account.is_(False), ExternalChatAccount.suspended_at.is_(None), ) @@ -128,7 +130,9 @@ async def start_byo_long_poll_supervisors() -> None: ) _tasks.add(task) task.add_done_callback(_tasks.discard) - logger.info("Started BYO Telegram long-poll supervisor account_id=%s", account.id) + logger.info( + "Started BYO Telegram long-poll supervisor account_id=%s", account.id + ) if config.GATEWAY_WHATSAPP_INTAKE_MODE == "baileys": task = asyncio.create_task( @@ -151,9 +155,12 @@ async def stop_byo_long_poll_supervisors() -> None: task.cancel() if tasks: try: - await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=10) + await asyncio.wait_for( + asyncio.gather(*tasks, return_exceptions=True), timeout=10 + ) except TimeoutError: - logger.warning("Timed out waiting for BYO Telegram long-poll supervisors to stop") + logger.warning( + "Timed out waiting for BYO Telegram long-poll supervisors to stop" + ) _tasks.clear() _shutdown_event = None - diff --git a/surfsense_backend/app/gateway/discord/intake.py b/surfsense_backend/app/gateway/discord/intake.py index 3fe76f0c4..fa0c26780 100644 --- a/surfsense_backend/app/gateway/discord/intake.py +++ b/surfsense_backend/app/gateway/discord/intake.py @@ -39,7 +39,9 @@ def _message_reference_payload(message: discord.Message) -> dict[str, Any] | Non } -def _serialize_message(message: discord.Message, *, bot_user_id: str | None) -> dict[str, Any]: +def _serialize_message( + message: discord.Message, *, bot_user_id: str | None +) -> dict[str, Any]: guild = message.guild channel = message.channel thread_id = str(channel.id) if isinstance(channel, discord.Thread) else None @@ -62,8 +64,7 @@ def _serialize_message(message: discord.Message, *, bot_user_id: str | None) -> "bot": message.author.bot, }, "mentions": [ - {"id": str(user.id), "username": user.name} - for user in message.mentions + {"id": str(user.id), "username": user.name} for user in message.mentions ], "message_reference": _message_reference_payload(message), "created_at": message.created_at.isoformat() @@ -73,7 +74,9 @@ def _serialize_message(message: discord.Message, *, bot_user_id: str | None) -> } -async def _persist_message(message: discord.Message, *, bot_user_id: str | None) -> None: +async def _persist_message( + message: discord.Message, *, bot_user_id: str | None +) -> None: if message.guild is None: return guild_id = str(message.guild.id) @@ -82,7 +85,9 @@ async def _persist_message(message: discord.Message, *, bot_user_id: str | None) async with async_session_maker() as session: account = await get_discord_account_by_guild(session, guild_id=guild_id) if account is None: - logger.info("Ignoring Discord message for uninstalled guild_id=%s", guild_id) + logger.info( + "Ignoring Discord message for uninstalled guild_id=%s", guild_id + ) return inbox_id = await persist_inbound_event( @@ -144,7 +149,9 @@ def _build_client() -> discord.Client: try: await _persist_message(message, bot_user_id=bot_user_id) except Exception: - logger.exception("Discord gateway failed to persist message_id=%s", message.id) + logger.exception( + "Discord gateway failed to persist message_id=%s", message.id + ) return client diff --git a/surfsense_backend/app/gateway/discord/translator.py b/surfsense_backend/app/gateway/discord/translator.py index 2bd843e3d..c09b012cf 100644 --- a/surfsense_backend/app/gateway/discord/translator.py +++ b/surfsense_backend/app/gateway/discord/translator.py @@ -41,7 +41,9 @@ class DiscordStreamTranslator(BaseStreamTranslator): async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: async for event in events: if event.type in {"text-delta", "text_delta", "text"}: - self._buffer += str(event.data.get("text") or event.data.get("delta") or "") + self._buffer += str( + event.data.get("text") or event.data.get("delta") or "" + ) elif event.type in {"data-interrupt-request", "interrupt"}: await self._handle_hitl_interrupt() return @@ -53,7 +55,9 @@ class DiscordStreamTranslator(BaseStreamTranslator): async def _flush_final(self) -> None: if not self._buffer: return - for chunk in split_text_message(self._buffer, max_chars=DISCORD_MAX_MESSAGE_CHARS): + for chunk in split_text_message( + self._buffer, max_chars=DISCORD_MAX_MESSAGE_CHARS + ): await self._send_text(chunk) async def _send_text(self, text: str) -> PlatformSendResult: diff --git a/surfsense_backend/app/gateway/hitl_filter.py b/surfsense_backend/app/gateway/hitl_filter.py index e3acc6d42..4f0422f44 100644 --- a/surfsense_backend/app/gateway/hitl_filter.py +++ b/surfsense_backend/app/gateway/hitl_filter.py @@ -32,4 +32,3 @@ def filter_hitl_tools( return None blocked = blocked_names or DEFAULT_HITL_TOOL_NAMES return [tool for tool in toolkit if (_tool_name(tool) or "") not in blocked] - diff --git a/surfsense_backend/app/gateway/inbox.py b/surfsense_backend/app/gateway/inbox.py index cd0e2f9b7..7dcade65e 100644 --- a/surfsense_backend/app/gateway/inbox.py +++ b/surfsense_backend/app/gateway/inbox.py @@ -51,4 +51,3 @@ async def persist_inbound_event( ) result = await session.execute(stmt) return result.scalar_one_or_none() - diff --git a/surfsense_backend/app/gateway/inbox_processor.py b/surfsense_backend/app/gateway/inbox_processor.py index 478c42a5e..7343b3a03 100644 --- a/surfsense_backend/app/gateway/inbox_processor.py +++ b/surfsense_backend/app/gateway/inbox_processor.py @@ -128,7 +128,9 @@ async def process_inbound_event( event.status = ExternalChatEventStatus.PROCESSED event.processed_at = datetime.now(UTC) await session.commit() - record_gateway_inbox_processed(platform=event.platform.value, status="processed") + record_gateway_inbox_processed( + platform=event.platform.value, status="processed" + ) async def _mark_failed( @@ -173,7 +175,9 @@ async def _resolve_slack_thread_binding( parsed, ) -> ExternalChatBinding | None: user_peer_id = parsed.metadata.get("slack_user_peer_id") - thread_peer_id = parsed.metadata.get("slack_thread_peer_id") or parsed.external_peer_id + thread_peer_id = ( + parsed.metadata.get("slack_thread_peer_id") or parsed.external_peer_id + ) if not user_peer_id or not thread_peer_id: return None @@ -233,7 +237,9 @@ async def _resolve_discord_thread_binding( parsed, ) -> ExternalChatBinding | None: user_peer_id = parsed.metadata.get("discord_user_peer_id") - thread_peer_id = parsed.metadata.get("discord_thread_peer_id") or parsed.external_peer_id + thread_peer_id = ( + parsed.metadata.get("discord_thread_peer_id") or parsed.external_peer_id + ) if not user_peer_id or not thread_peer_id: return None @@ -357,7 +363,11 @@ async def _dispatch_inbound_event( return if binding is None: - if bundle.auto_bind_owner and account.owner_user_id and account.owner_search_space_id: + if ( + bundle.auto_bind_owner + and account.owner_user_id + and account.owner_search_space_id + ): binding = ExternalChatBinding( account_id=account.id, user_id=account.owner_user_id, @@ -385,7 +395,9 @@ async def _dispatch_inbound_event( event.external_chat_binding_id = binding.id if cmd == "/help": - handled = await bundle.commands.handle_help_command(adapter=adapter, event=parsed) + handled = await bundle.commands.handle_help_command( + adapter=adapter, event=parsed + ) if handled: event.status = ExternalChatEventStatus.PROCESSED await session.commit() diff --git a/surfsense_backend/app/gateway/inbox_worker.py b/surfsense_backend/app/gateway/inbox_worker.py index 8f35e7e6a..2170b9b1d 100644 --- a/surfsense_backend/app/gateway/inbox_worker.py +++ b/surfsense_backend/app/gateway/inbox_worker.py @@ -55,4 +55,3 @@ async def stop_gateway_inbox_worker() -> None: with suppress(TimeoutError, asyncio.CancelledError): await asyncio.wait_for(_task, timeout=10) _task = None - diff --git a/surfsense_backend/app/gateway/pairing.py b/surfsense_backend/app/gateway/pairing.py index 7818bed12..bafb6df05 100644 --- a/surfsense_backend/app/gateway/pairing.py +++ b/surfsense_backend/app/gateway/pairing.py @@ -8,7 +8,7 @@ from datetime import UTC, datetime, timedelta from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.db import ExternalChatBindingState, ExternalChatBinding +from app.db import ExternalChatBinding, ExternalChatBindingState PAIRING_CODE_TTL = timedelta(minutes=10) @@ -51,4 +51,3 @@ async def redeem_pairing_code( binding.external_username = external_username binding.external_metadata = external_metadata or {} return binding - diff --git a/surfsense_backend/app/gateway/ratelimit.py b/surfsense_backend/app/gateway/ratelimit.py index fbcbd16b8..1afc839d5 100644 --- a/surfsense_backend/app/gateway/ratelimit.py +++ b/surfsense_backend/app/gateway/ratelimit.py @@ -133,4 +133,3 @@ async def wait_for_token( if wait_ms > 0: await asyncio.sleep(wait_ms / 1000) return wait_ms - diff --git a/surfsense_backend/app/gateway/registry.py b/surfsense_backend/app/gateway/registry.py index 3aa9e607a..fb2aeefc0 100644 --- a/surfsense_backend/app/gateway/registry.py +++ b/surfsense_backend/app/gateway/registry.py @@ -186,4 +186,6 @@ def resolve_platform_bundle(account: ExternalChatAccount) -> PlatformBundle: auto_bind_owner=False, ) - raise RuntimeError(f"unsupported_gateway_platform:{account.platform.value}:{account.mode.value}") + raise RuntimeError( + f"unsupported_gateway_platform:{account.platform.value}:{account.mode.value}" + ) diff --git a/surfsense_backend/app/gateway/runner.py b/surfsense_backend/app/gateway/runner.py index 83afc2353..64e0bb2e2 100644 --- a/surfsense_backend/app/gateway/runner.py +++ b/surfsense_backend/app/gateway/runner.py @@ -8,7 +8,12 @@ import uuid from sqlalchemy import text -from app.db import ExternalChatPlatform, ExternalChatAccount, async_session_maker, engine +from app.db import ( + ExternalChatAccount, + ExternalChatPlatform, + async_session_maker, + engine, +) from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key from app.gateway.telegram.adapter import TelegramAdapter from app.observability.metrics import record_gateway_byo_longpoll_running_delta @@ -39,7 +44,9 @@ async def _run_telegram_account(account_id: int, token: str) -> None: account = await session.get(ExternalChatAccount, account_id) offset = None if account is not None: - offset = int((account.cursor_state or {}).get("last_update_id", 0)) + 1 + offset = ( + int((account.cursor_state or {}).get("last_update_id", 0)) + 1 + ) async for update in adapter.fetch_updates(offset=offset): request_id = f"gateway_{uuid.uuid4().hex[:16]}" @@ -58,8 +65,11 @@ async def _run_telegram_account(account_id: int, token: str) -> None: ) await session.commit() if inbox_id is not None: - logger.debug("Persisted Telegram polling update inbox_id=%s", inbox_id) + logger.debug( + "Persisted Telegram polling update inbox_id=%s", inbox_id + ) finally: record_gateway_byo_longpoll_running_delta(-1, account_id=account_id) - await conn.execute(text("SELECT pg_advisory_unlock(:key)"), {"key": lock_key}) - + await conn.execute( + text("SELECT pg_advisory_unlock(:key)"), {"key": lock_key} + ) diff --git a/surfsense_backend/app/gateway/slack/adapter.py b/surfsense_backend/app/gateway/slack/adapter.py index e49ca6b9c..9890261bd 100644 --- a/surfsense_backend/app/gateway/slack/adapter.py +++ b/surfsense_backend/app/gateway/slack/adapter.py @@ -38,7 +38,9 @@ class SlackAdapter(BasePlatformAdapter): slack_user_id = str(event.get("user") or "") message_ts = str(event.get("ts") or "") thread_ts = str(event.get("thread_ts") or message_ts) - bot_user_id = self.bot_user_id or str(raw_payload.get("authorizations", [{}])[0].get("user_id") or "") + bot_user_id = self.bot_user_id or str( + raw_payload.get("authorizations", [{}])[0].get("user_id") or "" + ) if not channel_id or not slack_user_id or not message_ts: return ParsedInboundEvent( diff --git a/surfsense_backend/app/gateway/slack/client.py b/surfsense_backend/app/gateway/slack/client.py index 37ccda3bd..8f2f16cec 100644 --- a/surfsense_backend/app/gateway/slack/client.py +++ b/surfsense_backend/app/gateway/slack/client.py @@ -15,7 +15,9 @@ class SlackGatewayClient: def __init__(self, bot_token: str) -> None: self.bot_token = bot_token - async def api_call(self, method: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: + async def api_call( + self, method: str, payload: dict[str, Any] | None = None + ) -> dict[str, Any]: async with httpx.AsyncClient(timeout=20.0) as client: response = await client.post( f"{SLACK_API}/{method}", @@ -55,7 +57,9 @@ class SlackGatewayClient: ts: str, text: str, ) -> PlatformSendResult: - data = await self.api_call("chat.update", {"channel": channel, "ts": ts, "text": text}) + data = await self.api_call( + "chat.update", {"channel": channel, "ts": ts, "text": text} + ) return PlatformSendResult( external_message_id=str(data.get("ts") or ts), raw_response=data, diff --git a/surfsense_backend/app/gateway/slack/translator.py b/surfsense_backend/app/gateway/slack/translator.py index 658b0cac7..a591474e5 100644 --- a/surfsense_backend/app/gateway/slack/translator.py +++ b/surfsense_backend/app/gateway/slack/translator.py @@ -41,7 +41,9 @@ class SlackStreamTranslator(BaseStreamTranslator): async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: async for event in events: if event.type in {"text-delta", "text_delta", "text"}: - self._buffer += str(event.data.get("text") or event.data.get("delta") or "") + self._buffer += str( + event.data.get("text") or event.data.get("delta") or "" + ) elif event.type in {"data-interrupt-request", "interrupt"}: await self._handle_hitl_interrupt() return @@ -53,7 +55,9 @@ class SlackStreamTranslator(BaseStreamTranslator): async def _flush_final(self) -> None: if not self._buffer: return - for chunk in split_text_message(self._buffer, max_chars=SLACK_MAX_MESSAGE_CHARS): + for chunk in split_text_message( + self._buffer, max_chars=SLACK_MAX_MESSAGE_CHARS + ): await self._send_text(chunk) async def _send_text(self, text: str) -> PlatformSendResult: diff --git a/surfsense_backend/app/gateway/telegram/__init__.py b/surfsense_backend/app/gateway/telegram/__init__.py index 45dc05414..a4e252642 100644 --- a/surfsense_backend/app/gateway/telegram/__init__.py +++ b/surfsense_backend/app/gateway/telegram/__init__.py @@ -1,2 +1 @@ """Telegram gateway adapter.""" - diff --git a/surfsense_backend/app/gateway/telegram/adapter.py b/surfsense_backend/app/gateway/telegram/adapter.py index 4f0001128..dc4266d42 100644 --- a/surfsense_backend/app/gateway/telegram/adapter.py +++ b/surfsense_backend/app/gateway/telegram/adapter.py @@ -51,9 +51,7 @@ class TelegramAdapter(BasePlatformAdapter): "channel": "channel", }.get(chat_type, "unknown") display_name = chat.get("title") or " ".join( - part - for part in (sender.get("first_name"), sender.get("last_name")) - if part + part for part in (sender.get("first_name"), sender.get("last_name")) if part ) return ParsedInboundEvent( @@ -62,14 +60,21 @@ class TelegramAdapter(BasePlatformAdapter): external_peer_id=str(chat["id"]) if chat.get("id") is not None else None, external_peer_kind=peer_kind, external_message_id=( - str(message["message_id"]) if message.get("message_id") is not None else None + str(message["message_id"]) + if message.get("message_id") is not None + else None ), - external_user_id=str(sender["id"]) if sender.get("id") is not None else None, + external_user_id=str(sender["id"]) + if sender.get("id") is not None + else None, text=message.get("text") or message.get("caption"), raw_payload=raw_payload, display_name=display_name or None, username=sender.get("username") or chat.get("username"), - metadata={"chat_type": chat_type, "update_id": raw_payload.get("update_id")}, + metadata={ + "chat_type": chat_type, + "update_id": raw_payload.get("update_id"), + }, ) async def send_message( @@ -108,7 +113,8 @@ class TelegramAdapter(BasePlatformAdapter): async def leave_chat(self, *, external_peer_id: str) -> None: await self.client.leave_chat(chat_id=external_peer_id) - async def fetch_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]: + async def fetch_updates( + self, *, offset: int | None + ) -> AsyncIterator[dict[str, Any]]: async for update in self.client.get_updates(offset=offset): yield update - diff --git a/surfsense_backend/app/gateway/telegram/client.py b/surfsense_backend/app/gateway/telegram/client.py index 6f36f0564..d3b054451 100644 --- a/surfsense_backend/app/gateway/telegram/client.py +++ b/surfsense_backend/app/gateway/telegram/client.py @@ -106,4 +106,3 @@ async def retry_plaintext_on_bad_markdown(call, *args, **kwargs) -> PlatformSend raise kwargs["parse_mode"] = None return await call(*args, **kwargs) - diff --git a/surfsense_backend/app/gateway/telegram/commands.py b/surfsense_backend/app/gateway/telegram/commands.py index 903330fd8..c9965fd90 100644 --- a/surfsense_backend/app/gateway/telegram/commands.py +++ b/surfsense_backend/app/gateway/telegram/commands.py @@ -54,7 +54,9 @@ async def handle_start_command( return True -async def handle_help_command(*, adapter: TelegramAdapter, event: ParsedInboundEvent) -> bool: +async def handle_help_command( + *, adapter: TelegramAdapter, event: ParsedInboundEvent +) -> bool: if not event.external_peer_id: return True await adapter.send_message(external_peer_id=event.external_peer_id, text=HELP_TEXT) @@ -114,4 +116,4 @@ class TelegramGatewayCommands(BaseGatewayCommands): adapter=adapter, event=event, dashboard_url=dashboard_url, - ) \ No newline at end of file + ) diff --git a/surfsense_backend/app/gateway/telegram/formatting.py b/surfsense_backend/app/gateway/telegram/formatting.py index a9bb73ed5..668a6c7ed 100644 --- a/surfsense_backend/app/gateway/telegram/formatting.py +++ b/surfsense_backend/app/gateway/telegram/formatting.py @@ -32,9 +32,13 @@ def _split_at_boundary(text: str, max_units: int) -> tuple[str, str]: end -= 1 candidate = text[:end] - boundary = max(candidate.rfind("\n\n"), candidate.rfind(". "), candidate.rfind("\n")) + boundary = max( + candidate.rfind("\n\n"), candidate.rfind(". "), candidate.rfind("\n") + ) if boundary > max(200, end // 2): - end = boundary + (2 if candidate[boundary : boundary + 2] in {"\n\n", ". "} else 1) + end = boundary + ( + 2 if candidate[boundary : boundary + 2] in {"\n\n", ". "} else 1 + ) return text[:end], text[end:] @@ -56,4 +60,3 @@ def chunk_message( chunks.append(chunk) return chunks return split_text_message(text, max_chars=max_units) - diff --git a/surfsense_backend/app/gateway/telegram/translator.py b/surfsense_backend/app/gateway/telegram/translator.py index 96903bea0..a1600b0e7 100644 --- a/surfsense_backend/app/gateway/telegram/translator.py +++ b/surfsense_backend/app/gateway/telegram/translator.py @@ -49,7 +49,9 @@ class TelegramStreamTranslator(BaseStreamTranslator): async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: async for event in events: if event.type in {"text-delta", "text_delta", "text"}: - self._buffer += str(event.data.get("text") or event.data.get("delta") or "") + self._buffer += str( + event.data.get("text") or event.data.get("delta") or "" + ) await self._maybe_flush() elif event.type in {"data-interrupt-request", "interrupt"}: await self._handle_hitl_interrupt() @@ -159,7 +161,9 @@ class TelegramStreamTranslator(BaseStreamTranslator): ) if chat_wait: record_gateway_rate_limit_hit(bucket="tg:chat") - global_wait = await wait_for_token("tg:global", capacity=25, refill_per_sec=25.0) + global_wait = await wait_for_token( + "tg:global", capacity=25, refill_per_sec=25.0 + ) if global_wait: record_gateway_rate_limit_hit(bucket="tg:global") @@ -168,4 +172,3 @@ class TelegramStreamTranslator(BaseStreamTranslator): await self._flush(final=False) await self._send_text(HITL_UNSUPPORTED_MESSAGE) record_gateway_hitl_aborted(platform="telegram") - diff --git a/surfsense_backend/app/gateway/thread_lock.py b/surfsense_backend/app/gateway/thread_lock.py index 82733bb69..208b2898d 100644 --- a/surfsense_backend/app/gateway/thread_lock.py +++ b/surfsense_backend/app/gateway/thread_lock.py @@ -36,5 +36,6 @@ def release_thread_lock(thread_id: int) -> None: try: _redis().delete(_lock_key(thread_id)) except redis.RedisError as exc: - logger.warning("Failed to release gateway thread lock for %s: %s", thread_id, exc) - + logger.warning( + "Failed to release gateway thread lock for %s: %s", thread_id, exc + ) diff --git a/surfsense_backend/app/gateway/whatsapp/adapter_baileys.py b/surfsense_backend/app/gateway/whatsapp/adapter_baileys.py index 99489e27b..330ef3bb9 100644 --- a/surfsense_backend/app/gateway/whatsapp/adapter_baileys.py +++ b/surfsense_backend/app/gateway/whatsapp/adapter_baileys.py @@ -36,7 +36,8 @@ class WhatsAppBaileysAdapter(BasePlatformAdapter): external_user_id=sender_id or None, text=str(body) if body is not None else None, raw_payload=raw_payload, - display_name=str(raw_payload.get("chatName") or sender_id or chat_id) or None, + display_name=str(raw_payload.get("chatName") or sender_id or chat_id) + or None, username=None, metadata={ "sender_id": sender_id, @@ -92,7 +93,9 @@ class WhatsAppBaileysAdapter(BasePlatformAdapter): response.raise_for_status() return response.json() - async def fetch_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]: + async def fetch_updates( + self, *, offset: int | None + ) -> AsyncIterator[dict[str, Any]]: async with httpx.AsyncClient(timeout=35) as client: response = await client.get(f"{self.bridge_url}/messages") response.raise_for_status() diff --git a/surfsense_backend/app/gateway/whatsapp/adapter_cloud.py b/surfsense_backend/app/gateway/whatsapp/adapter_cloud.py index f247db692..58d13e83e 100644 --- a/surfsense_backend/app/gateway/whatsapp/adapter_cloud.py +++ b/surfsense_backend/app/gateway/whatsapp/adapter_cloud.py @@ -54,7 +54,9 @@ class WhatsAppCloudAdapter(BasePlatformAdapter): username=None, metadata={ "phone_number_id": _metadata(raw_payload).get("phone_number_id"), - "display_phone_number": _metadata(raw_payload).get("display_phone_number"), + "display_phone_number": _metadata(raw_payload).get( + "display_phone_number" + ), "timestamp": message.get("timestamp"), "message_type": message.get("type"), }, @@ -96,7 +98,9 @@ def _changes(raw_payload: dict[str, Any]) -> list[dict[str, Any]]: for entry in raw_payload.get("entry") or []: if isinstance(entry, dict): changes.extend( - change for change in (entry.get("changes") or []) if isinstance(change, dict) + change + for change in (entry.get("changes") or []) + if isinstance(change, dict) ) return changes diff --git a/surfsense_backend/app/gateway/whatsapp/credentials.py b/surfsense_backend/app/gateway/whatsapp/credentials.py index fba79d470..ed725944a 100644 --- a/surfsense_backend/app/gateway/whatsapp/credentials.py +++ b/surfsense_backend/app/gateway/whatsapp/credentials.py @@ -18,8 +18,7 @@ class WhatsAppCredentials(TypedDict, total=False): def load_system_whatsapp_credentials() -> WhatsAppCredentials: if not ( - config.WHATSAPP_SHARED_BUSINESS_TOKEN - and config.WHATSAPP_SHARED_PHONE_NUMBER_ID + config.WHATSAPP_SHARED_BUSINESS_TOKEN and config.WHATSAPP_SHARED_PHONE_NUMBER_ID ): raise RuntimeError("whatsapp_system_credentials_not_configured") diff --git a/surfsense_backend/app/gateway/whatsapp/translator.py b/surfsense_backend/app/gateway/whatsapp/translator.py index deef8b452..4673a51ca 100644 --- a/surfsense_backend/app/gateway/whatsapp/translator.py +++ b/surfsense_backend/app/gateway/whatsapp/translator.py @@ -41,7 +41,9 @@ class WhatsAppCloudStreamTranslator(BaseStreamTranslator): if event.type in {"text-delta", "text_delta", "text"}: if not self._typing_sent: await self._send_typing_indicator() - self._buffer += str(event.data.get("text") or event.data.get("delta") or "") + self._buffer += str( + event.data.get("text") or event.data.get("delta") or "" + ) elif event.type in {"data-interrupt-request", "interrupt"}: await self._handle_hitl_interrupt() return diff --git a/surfsense_backend/app/gateway/whatsapp/translator_baileys.py b/surfsense_backend/app/gateway/whatsapp/translator_baileys.py index 8a4c8acfa..ab2afb9d7 100644 --- a/surfsense_backend/app/gateway/whatsapp/translator_baileys.py +++ b/surfsense_backend/app/gateway/whatsapp/translator_baileys.py @@ -42,7 +42,9 @@ class WhatsAppBaileysStreamTranslator(BaseStreamTranslator): await self._send_typing_indicator() async for event in events: if event.type in {"text-delta", "text_delta", "text"}: - self._buffer += str(event.data.get("text") or event.data.get("delta") or "") + self._buffer += str( + event.data.get("text") or event.data.get("delta") or "" + ) await self._maybe_flush() elif event.type in {"data-interrupt-request", "interrupt"}: await self._handle_hitl_interrupt() @@ -86,7 +88,9 @@ class WhatsAppBaileysStreamTranslator(BaseStreamTranslator): if not isinstance(self.adapter, WhatsAppBaileysAdapter): return try: - await self.adapter.send_typing_indicator(external_peer_id=self.external_peer_id) + await self.adapter.send_typing_indicator( + external_peer_id=self.external_peer_id + ) record_gateway_outbound(platform="whatsapp", kind="typing", status="sent") except Exception: logger.debug("WhatsApp Baileys typing indicator failed", exc_info=True) diff --git a/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py b/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py index 3d0124059..67a6778e0 100644 --- a/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py +++ b/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py @@ -202,7 +202,9 @@ class IndexingPipelineService: await self.session.commit() - async def index_batch(self, connector_docs: list[ConnectorDocument]) -> list[Document]: + async def index_batch( + self, connector_docs: list[ConnectorDocument] + ) -> list[Document]: """Convenience method: prepare_for_indexing then index each document. Indexers that need heartbeat callbacks or custom per-document logic @@ -347,7 +349,9 @@ class IndexingPipelineService: await self.session.rollback() return [] - async def index(self, document: Document, connector_doc: ConnectorDocument) -> Document: + async def index( + self, document: Document, connector_doc: ConnectorDocument + ) -> Document: """ Run deterministic content storage, embedding, and chunking for a document. """ diff --git a/surfsense_backend/app/notifications/__init__.py b/surfsense_backend/app/notifications/__init__.py index 6ffe45000..554872d85 100644 --- a/surfsense_backend/app/notifications/__init__.py +++ b/surfsense_backend/app/notifications/__init__.py @@ -9,7 +9,6 @@ from __future__ import annotations # Initialize app.db first to avoid a partial-init circular import when this # package is the entry point (e.g. Celery loading it before any ORM code). import app.db # noqa: F401 - from app.notifications.persistence import Notification from app.notifications.service import NotificationService diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 426346355..5cc029884 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -1,8 +1,9 @@ from fastapi import APIRouter, Depends from app.automations.api import router as automations_router -from app.gateway import require_gateway_enabled from app.file_storage.api import router as file_storage_router +from app.gateway import require_gateway_enabled +from app.notifications.api import router as notifications_router from .agent_action_log_route import router as agent_action_log_router from .agent_flags_route import router as agent_flags_router @@ -46,7 +47,6 @@ from .model_list_routes import router as model_list_router from .new_chat_routes import router as new_chat_router from .new_llm_config_routes import router as new_llm_config_router from .notes_routes import router as notes_router -from app.notifications.api import router as notifications_router from .notion_add_connector_route import router as notion_add_connector_router from .obsidian_plugin_routes import router as obsidian_plugin_router from .onedrive_add_connector_route import router as onedrive_add_connector_router @@ -76,8 +76,12 @@ router.include_router(documents_router) router.include_router(folders_router) _gateway_enabled_dep = [Depends(require_gateway_enabled)] router.include_router(gateway_router, dependencies=_gateway_enabled_dep) -router.include_router(gateway_whatsapp_webhook_router, dependencies=_gateway_enabled_dep) -router.include_router(gateway_whatsapp_baileys_router, dependencies=_gateway_enabled_dep) +router.include_router( + gateway_whatsapp_webhook_router, dependencies=_gateway_enabled_dep +) +router.include_router( + gateway_whatsapp_baileys_router, dependencies=_gateway_enabled_dep +) router.include_router(notes_router) router.include_router(new_chat_router) # Chat with assistant-ui persistence router.include_router(agent_revert_router) # POST /threads/{id}/revert/{action_id} diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py index 9c890b610..14f929567 100644 --- a/surfsense_backend/app/routes/gateway_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -119,21 +119,35 @@ def _discord_redirect_uri() -> str: return f"{base.rstrip('/')}/api/v1/gateway/discord/callback" -def _slack_frontend_redirect(space_id: int, *, success: bool = False, error: str | None = None) -> RedirectResponse: - qs = "slack_gateway=connected" if success else f"error={error or 'slack_gateway_failed'}" +def _slack_frontend_redirect( + space_id: int, *, success: bool = False, error: str | None = None +) -> RedirectResponse: + qs = ( + "slack_gateway=connected" + if success + else f"error={error or 'slack_gateway_failed'}" + ) return RedirectResponse( url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/user-settings?{qs}" ) -def _discord_frontend_redirect(space_id: int, *, success: bool = False, error: str | None = None) -> RedirectResponse: - qs = "discord_gateway=connected" if success else f"error={error or 'discord_gateway_failed'}" +def _discord_frontend_redirect( + space_id: int, *, success: bool = False, error: str | None = None +) -> RedirectResponse: + qs = ( + "discord_gateway=connected" + if success + else f"error={error or 'discord_gateway_failed'}" + ) return RedirectResponse( url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/user-settings?{qs}" ) -def verify_slack_signature(*, signing_secret: str, timestamp: str | None, signature: str | None, body: bytes) -> bool: +def verify_slack_signature( + *, signing_secret: str, timestamp: str | None, signature: str | None, body: bytes +) -> bool: if not signing_secret or not timestamp or not signature: return False try: @@ -239,7 +253,9 @@ async def install_slack_gateway( session: AsyncSession = Depends(get_async_session), ) -> dict[str, str]: if not _slack_gateway_enabled(): - raise HTTPException(status_code=500, detail="Slack gateway OAuth is not configured") + raise HTTPException( + status_code=500, detail="Slack gateway OAuth is not configured" + ) await check_search_space_access(session, user, search_space_id) state = _get_state_manager().generate_secure_state(search_space_id, user.id) auth_params = { @@ -269,11 +285,17 @@ async def slack_gateway_callback( state_data = None if error: - return _slack_frontend_redirect(space_id or 0, error="slack_gateway_oauth_denied") + return _slack_frontend_redirect( + space_id or 0, error="slack_gateway_oauth_denied" + ) if not code or state_data is None: - raise HTTPException(status_code=400, detail="Invalid Slack gateway OAuth callback") + raise HTTPException( + status_code=400, detail="Invalid Slack gateway OAuth callback" + ) if not _slack_gateway_enabled(): - raise HTTPException(status_code=500, detail="Slack gateway OAuth is not configured") + raise HTTPException( + status_code=500, detail="Slack gateway OAuth is not configured" + ) user_id = UUID(state_data["user_id"]) token_payload = { @@ -300,7 +322,9 @@ async def slack_gateway_callback( team = token_json.get("team") or {} team_id = team.get("id") if not bot_token or not team_id: - raise HTTPException(status_code=400, detail="Slack gateway OAuth returned incomplete data") + raise HTTPException( + status_code=400, detail="Slack gateway OAuth returned incomplete data" + ) bot_user_id = token_json.get("bot_user_id") app_id = token_json.get("app_id") @@ -388,7 +412,9 @@ async def install_discord_gateway( session: AsyncSession = Depends(get_async_session), ) -> dict[str, str]: if not _discord_gateway_enabled(): - raise HTTPException(status_code=500, detail="Discord gateway OAuth is not configured") + raise HTTPException( + status_code=500, detail="Discord gateway OAuth is not configured" + ) await check_search_space_access(session, user, search_space_id) state = _get_state_manager().generate_secure_state(search_space_id, user.id) auth_params = { @@ -420,11 +446,17 @@ async def discord_gateway_callback( state_data = None if error: - return _discord_frontend_redirect(space_id or 0, error="discord_gateway_oauth_denied") + return _discord_frontend_redirect( + space_id or 0, error="discord_gateway_oauth_denied" + ) if not code or state_data is None: - raise HTTPException(status_code=400, detail="Invalid Discord gateway OAuth callback") + raise HTTPException( + status_code=400, detail="Invalid Discord gateway OAuth callback" + ) if not _discord_gateway_enabled(): - raise HTTPException(status_code=500, detail="Discord gateway OAuth is not configured") + raise HTTPException( + status_code=500, detail="Discord gateway OAuth is not configured" + ) user_id = UUID(state_data["user_id"]) token_payload = { @@ -535,7 +567,10 @@ async def discord_gateway_callback( elif binding.user_id == user_id: binding.search_space_id = space_id binding.external_username = discord_username or binding.external_username - binding.external_metadata = {**(binding.external_metadata or {}), **metadata} + binding.external_metadata = { + **(binding.external_metadata or {}), + **metadata, + } await session.commit() return _discord_frontend_redirect(space_id, success=True) @@ -614,7 +649,9 @@ async def _resolve_webhook_account( if account is None or account.platform != ExternalChatPlatform.TELEGRAM: raise HTTPException(status_code=404, detail="Gateway account not found") expected_secret = account.webhook_secret or "" - if not expected_secret or not hmac.compare_digest(header_secret or "", expected_secret): + if not expected_secret or not hmac.compare_digest( + header_secret or "", expected_secret + ): raise HTTPException(status_code=403, detail="Invalid Telegram webhook secret") return account @@ -654,7 +691,9 @@ async def telegram_webhook( event_dedupe_key=telegram_event_dedupe_key(update_id), external_event_id=str(update_id), external_message_id=( - str(message["message_id"]) if message.get("message_id") is not None else None + str(message["message_id"]) + if message.get("message_id") is not None + else None ), event_kind=_classify_telegram_event(payload), raw_payload=payload, @@ -739,7 +778,10 @@ async def list_bindings( ) -> list[dict[str, Any]]: result = await session.execute( select(ExternalChatBinding, ExternalChatAccount) - .join(ExternalChatAccount, ExternalChatBinding.account_id == ExternalChatAccount.id) + .join( + ExternalChatAccount, + ExternalChatBinding.account_id == ExternalChatAccount.id, + ) .where(ExternalChatBinding.user_id == user.id) ) return [ @@ -777,13 +819,20 @@ async def list_connections( ] if platform is not None: filters.append(ExternalChatAccount.platform == platform) - if platform == ExternalChatPlatform.WHATSAPP and active_whatsapp_mode is not None: + if ( + platform == ExternalChatPlatform.WHATSAPP + and active_whatsapp_mode is not None + ): filters.append(ExternalChatAccount.mode == active_whatsapp_mode) else: if not _telegram_gateway_enabled(): - filters.append(ExternalChatAccount.platform != ExternalChatPlatform.TELEGRAM) + filters.append( + ExternalChatAccount.platform != ExternalChatPlatform.TELEGRAM + ) if active_whatsapp_mode is None: - filters.append(ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP) + filters.append( + ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP + ) else: filters.append( or_( @@ -794,7 +843,10 @@ async def list_connections( result = await session.execute( select(ExternalChatBinding, ExternalChatAccount) - .join(ExternalChatAccount, ExternalChatBinding.account_id == ExternalChatAccount.id) + .join( + ExternalChatAccount, + ExternalChatBinding.account_id == ExternalChatAccount.id, + ) .where(*filters) ) @@ -828,7 +880,9 @@ async def list_connections( baileys_account_ids.add(int(account.id)) route_type = "account" connection_id = account.id - search_space_id = account.owner_search_space_id or binding.search_space_id + search_space_id = ( + account.owner_search_space_id or binding.search_space_id + ) display_name = "WhatsApp Bridge" connections.append( @@ -853,9 +907,8 @@ async def list_connections( } ) - if ( - active_whatsapp_mode == ExternalChatAccountMode.SELF_HOST_BYO - and (platform is None or platform == ExternalChatPlatform.WHATSAPP) + if active_whatsapp_mode == ExternalChatAccountMode.SELF_HOST_BYO and ( + platform is None or platform == ExternalChatPlatform.WHATSAPP ): account_result = await session.execute( select(ExternalChatAccount).where( @@ -940,7 +993,9 @@ async def update_binding_search_space( ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED, }: - raise HTTPException(status_code=400, detail="Only active bindings can be routed") + raise HTTPException( + status_code=400, detail="Only active bindings can be routed" + ) account = await session.get(ExternalChatAccount, binding.account_id) if account is None or _is_inactive_whatsapp_account(account): raise HTTPException(status_code=404, detail="Binding not found") @@ -1062,4 +1117,3 @@ async def resume_external_chat_binding( binding.updated_at = datetime.now(UTC) await session.commit() return {"ok": True} - diff --git a/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py index 5ab669503..1fcf5c438 100644 --- a/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py +++ b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py @@ -33,7 +33,9 @@ class BaileysPairRequest(BaseModel): def _ensure_baileys_enabled() -> None: if config.GATEWAY_WHATSAPP_INTAKE_MODE != "baileys": - raise HTTPException(status_code=404, detail="WhatsApp Baileys gateway is disabled") + raise HTTPException( + status_code=404, detail="WhatsApp Baileys gateway is disabled" + ) if config.is_cloud(): raise HTTPException( status_code=403, diff --git a/surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py b/surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py index bb7b49712..39dc928df 100644 --- a/surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py @@ -79,7 +79,9 @@ async def whatsapp_webhook( def _verify_signature(raw_body: bytes, header_signature: str | None) -> None: if not config.WHATSAPP_WEBHOOK_APP_SECRET: - raise HTTPException(status_code=500, detail="WhatsApp app secret is not configured") + raise HTTPException( + status_code=500, detail="WhatsApp app secret is not configured" + ) received = (header_signature or "").removeprefix("sha256=") expected = hmac.new( config.WHATSAPP_WEBHOOK_APP_SECRET.encode(), @@ -87,7 +89,9 @@ def _verify_signature(raw_body: bytes, header_signature: str | None) -> None: hashlib.sha256, ).hexdigest() if not received or not hmac.compare_digest(received, expected): - raise HTTPException(status_code=403, detail="Invalid WhatsApp webhook signature") + raise HTTPException( + status_code=403, detail="Invalid WhatsApp webhook signature" + ) async def _process_payload(session: AsyncSession, payload: dict[str, Any]) -> None: @@ -114,7 +118,9 @@ async def _process_messages_change( change: dict[str, Any], value: dict[str, Any], ) -> None: - statuses = [status for status in value.get("statuses") or [] if isinstance(status, dict)] + statuses = [ + status for status in value.get("statuses") or [] if isinstance(status, dict) + ] for status in statuses: record_gateway_outbound( platform="whatsapp", diff --git a/surfsense_backend/app/routes/obsidian_plugin_routes.py b/surfsense_backend/app/routes/obsidian_plugin_routes.py index 512596550..bd54a4788 100644 --- a/surfsense_backend/app/routes/obsidian_plugin_routes.py +++ b/surfsense_backend/app/routes/obsidian_plugin_routes.py @@ -25,6 +25,7 @@ from app.db import ( User, get_async_session, ) +from app.notifications.service import NotificationService from app.schemas.obsidian_plugin import ( ALLOWED_ATTACHMENT_EXTENSIONS, ATTACHMENT_MIME_TYPES, @@ -43,7 +44,6 @@ from app.schemas.obsidian_plugin import ( SyncAckItem, SyncBatchRequest, ) -from app.notifications.service import NotificationService from app.services.obsidian_plugin_indexer import ( delete_note, get_manifest, diff --git a/surfsense_backend/app/services/chat_comments_service.py b/surfsense_backend/app/services/chat_comments_service.py index c9afb8a67..905482010 100644 --- a/surfsense_backend/app/services/chat_comments_service.py +++ b/surfsense_backend/app/services/chat_comments_service.py @@ -20,6 +20,7 @@ from app.db import ( User, has_permission, ) +from app.notifications.service import NotificationService from app.schemas.chat_comments import ( AuthorResponse, CommentBatchResponse, @@ -31,7 +32,6 @@ from app.schemas.chat_comments import ( MentionListResponse, MentionResponse, ) -from app.notifications.service import NotificationService from app.utils.chat_comments import parse_mentions, render_mentions from app.utils.rbac import check_permission, get_user_permissions diff --git a/surfsense_backend/app/services/confluence/kb_sync_service.py b/surfsense_backend/app/services/confluence/kb_sync_service.py index df07c3e81..7154637b4 100644 --- a/surfsense_backend/app/services/confluence/kb_sync_service.py +++ b/surfsense_backend/app/services/confluence/kb_sync_service.py @@ -64,9 +64,6 @@ class ConfluenceKBSyncService: if dup: content_hash = unique_hash - - - summary_content = f"Confluence Page: {page_title}\n\n{page_content}" summary_embedding = embed_text(summary_content) @@ -166,8 +163,6 @@ class ConfluenceKBSyncService: space_id = (document.document_metadata or {}).get("space_id", "") - - summary_content = f"Confluence Page: {page_title}\n\n{page_content}" summary_embedding = embed_text(summary_content) diff --git a/surfsense_backend/app/services/dropbox/kb_sync_service.py b/surfsense_backend/app/services/dropbox/kb_sync_service.py index b455e4fdd..a25cc054d 100644 --- a/surfsense_backend/app/services/dropbox/kb_sync_service.py +++ b/surfsense_backend/app/services/dropbox/kb_sync_service.py @@ -71,9 +71,6 @@ class DropboxKBSyncService: ) content_hash = unique_hash - - - summary_content = f"Dropbox File: {file_name}\n\n{indexable_content}" summary_embedding = embed_text(summary_content) diff --git a/surfsense_backend/app/services/gmail/kb_sync_service.py b/surfsense_backend/app/services/gmail/kb_sync_service.py index f2a8bed30..192570339 100644 --- a/surfsense_backend/app/services/gmail/kb_sync_service.py +++ b/surfsense_backend/app/services/gmail/kb_sync_service.py @@ -77,9 +77,6 @@ class GmailKBSyncService: ) content_hash = unique_hash - - - summary_content = f"Gmail Message: {subject}\n\n{indexable_content}" summary_embedding = await asyncio.to_thread(embed_text, summary_content) diff --git a/surfsense_backend/app/services/google_calendar/kb_sync_service.py b/surfsense_backend/app/services/google_calendar/kb_sync_service.py index 36fddc82a..495720a2d 100644 --- a/surfsense_backend/app/services/google_calendar/kb_sync_service.py +++ b/surfsense_backend/app/services/google_calendar/kb_sync_service.py @@ -89,9 +89,6 @@ class GoogleCalendarKBSyncService: ) content_hash = unique_hash - - - summary_content = ( f"Google Calendar Event: {event_summary}\n\n{indexable_content}" ) @@ -252,9 +249,6 @@ class GoogleCalendarKBSyncService: if not indexable_content: return {"status": "error", "message": "Event produced empty content"} - - - summary_content = ( f"Google Calendar Event: {event_summary}\n\n{indexable_content}" ) diff --git a/surfsense_backend/app/services/google_drive/kb_sync_service.py b/surfsense_backend/app/services/google_drive/kb_sync_service.py index 78c0e2491..30fbc14f2 100644 --- a/surfsense_backend/app/services/google_drive/kb_sync_service.py +++ b/surfsense_backend/app/services/google_drive/kb_sync_service.py @@ -73,12 +73,7 @@ class GoogleDriveKBSyncService: ) content_hash = unique_hash - - - - summary_content = ( - f"Google Drive File: {file_name}\n\n{indexable_content}" - ) + summary_content = f"Google Drive File: {file_name}\n\n{indexable_content}" summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(indexable_content) diff --git a/surfsense_backend/app/services/linear/kb_sync_service.py b/surfsense_backend/app/services/linear/kb_sync_service.py index 9ca7c99e5..3b8def6c3 100644 --- a/surfsense_backend/app/services/linear/kb_sync_service.py +++ b/surfsense_backend/app/services/linear/kb_sync_service.py @@ -83,9 +83,6 @@ class LinearKBSyncService: ) content_hash = unique_hash - - - summary_content = ( f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}" ) @@ -207,8 +204,6 @@ class LinearKBSyncService: comment_count = len(formatted_issue.get("comments", [])) formatted_issue.get("description", "") - - summary_content = ( f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}" ) diff --git a/surfsense_backend/app/services/notion/kb_sync_service.py b/surfsense_backend/app/services/notion/kb_sync_service.py index 826d01a15..ee85daf41 100644 --- a/surfsense_backend/app/services/notion/kb_sync_service.py +++ b/surfsense_backend/app/services/notion/kb_sync_service.py @@ -72,9 +72,6 @@ class NotionKBSyncService: ) content_hash = unique_hash - - - summary_content = f"Notion Page: {page_title}\n\n{markdown_content}" summary_embedding = embed_text(summary_content) @@ -225,7 +222,6 @@ class NotionKBSyncService: f"Final content length: {len(full_content)} chars, verified={content_verified}" ) - logger.debug("Generating summary and embeddings") summary_content = f"Notion Page: {document.document_metadata.get('page_title')}\n\n{full_content}" diff --git a/surfsense_backend/app/services/onedrive/kb_sync_service.py b/surfsense_backend/app/services/onedrive/kb_sync_service.py index 66a885b1c..2bfea6ef4 100644 --- a/surfsense_backend/app/services/onedrive/kb_sync_service.py +++ b/surfsense_backend/app/services/onedrive/kb_sync_service.py @@ -72,9 +72,6 @@ class OneDriveKBSyncService: ) content_hash = unique_hash - - - summary_content = f"OneDrive File: {file_name}\n\n{indexable_content}" summary_embedding = await asyncio.to_thread(embed_text, summary_content) diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index 211d9e5b3..d38014124 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -9,8 +9,8 @@ from uuid import UUID from app.celery_app import celery_app from app.config import config -from app.observability import metrics as ot_metrics from app.notifications.service import NotificationService +from app.observability import metrics as ot_metrics from app.services.task_logging_service import TaskLoggingService from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task from app.tasks.connector_indexers.local_folder_indexer import ( @@ -1335,7 +1335,7 @@ async def _index_local_folder_async( exclude_patterns=exclude_patterns, file_extensions=file_extensions, root_folder_id=root_folder_id, - target_file_paths=target_file_paths, + target_file_paths=target_file_paths, on_heartbeat_callback=_heartbeat_progress if (is_batch or is_full_scan) else None, @@ -1463,7 +1463,7 @@ async def _index_uploaded_folder_files_async( user_id=user_id, folder_name=folder_name, root_folder_id=root_folder_id, - file_mappings=file_mappings, + file_mappings=file_mappings, on_heartbeat_callback=_heartbeat_progress, use_vision_llm=use_vision_llm, processing_mode=processing_mode, diff --git a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py index 898d8c8af..7bd3d6788 100644 --- a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py @@ -46,7 +46,8 @@ def reconcile_inbox_task() -> None: result = await session.execute( update(ExternalChatInboundEvent) .where( - ExternalChatInboundEvent.status == ExternalChatEventStatus.PROCESSING, + ExternalChatInboundEvent.status + == ExternalChatEventStatus.PROCESSING, ExternalChatInboundEvent.received_at < stale_threshold, ) .values( @@ -163,4 +164,3 @@ async def enqueue_telegram_update(account_id: int, raw_update: dict) -> int | No ) await session.commit() return inbox_id - diff --git a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py index 5dbe4caec..1187edd98 100644 --- a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py @@ -260,7 +260,7 @@ async def index_confluence_pages( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - ) + ) with session.no_autoflush: duplicate_by_content = await check_duplicate_document_by_hash( diff --git a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py index e0053f614..97f01d68b 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py @@ -414,7 +414,7 @@ async def index_google_calendar_events( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - ) + ) with session.no_autoflush: duplicate = await check_duplicate_document_by_hash( diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index e20518ab0..68c43716b 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -552,7 +552,7 @@ async def _process_single_file( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - ) + ) pipeline = IndexingPipelineService(session) documents = await pipeline.prepare_for_indexing([doc]) diff --git a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py index 29b94a873..225e3618e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py @@ -444,7 +444,7 @@ async def index_google_gmail_messages( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - ) + ) with session.no_autoflush: duplicate = await check_duplicate_document_by_hash( diff --git a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py index 4ea781e6f..12749b82b 100644 --- a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py @@ -268,7 +268,7 @@ async def index_linear_issues( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - ) + ) with session.no_autoflush: duplicate = await check_duplicate_document_by_hash( diff --git a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py index 0354fce2e..1cd92dcf8 100644 --- a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py @@ -568,7 +568,7 @@ async def index_local_folder( folder_path=folder_path, folder_name=folder_name, target_file_path=target_file_paths[0], - root_folder_id=root_folder_id, + root_folder_id=root_folder_id, task_logger=task_logger, log_entry=log_entry, ) @@ -580,7 +580,7 @@ async def index_local_folder( folder_path=folder_path, folder_name=folder_name, target_file_paths=target_file_paths, - root_folder_id=root_folder_id, + root_folder_id=root_folder_id, on_progress_callback=on_heartbeat_callback, ) if err: @@ -766,7 +766,7 @@ async def index_local_folder( folder_name=folder_name, search_space_id=search_space_id, user_id=user_id, - ) + ) connector_docs.append(doc) file_meta_map[unique_identifier] = { "relative_path": relative_path, @@ -983,7 +983,7 @@ async def _index_batch_files( folder_path=folder_path, folder_name=folder_name, target_file_path=file_path, - root_folder_id=root_folder_id, + root_folder_id=root_folder_id, task_logger=task_logger, log_entry=log_entry, ) @@ -1111,7 +1111,7 @@ async def _index_single_file( folder_name=folder_name, search_space_id=search_space_id, user_id=user_id, - ) + ) if root_folder_id: connector_doc.folder_id = await _resolve_folder_for_file( @@ -1396,7 +1396,7 @@ async def index_uploaded_files( folder_name=folder_name, search_space_id=search_space_id, user_id=user_id, - ) + ) connector_doc.folder_id = await _resolve_folder_for_file( session, diff --git a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py index 662bb6b96..9bcba5a37 100644 --- a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py @@ -440,9 +440,7 @@ async def index_luma_events( summary_content = ( f"Luma Event: {item['event_name']}\n\n{item['event_markdown']}" ) - summary_embedding = await asyncio.to_thread( - embed_text, summary_content - ) + summary_embedding = await asyncio.to_thread(embed_text, summary_content) chunks = await create_document_chunks(item["event_markdown"]) diff --git a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py index 59589b7c7..1ca9ca4ba 100644 --- a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py @@ -308,7 +308,7 @@ async def index_notion_pages( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - ) + ) with session.no_autoflush: duplicate = await check_duplicate_document_by_hash( diff --git a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py index 8538f28d2..d81de67c0 100644 --- a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py @@ -318,9 +318,7 @@ async def index_crawled_urls( continue # Format content as structured document for summary generation - structured_document = crawler.format_to_structured_document( - crawl_result - ) + crawler.format_to_structured_document(crawl_result) # Generate content hash using a version WITHOUT metadata structured_document_for_hash = crawler.format_to_structured_document( @@ -332,8 +330,8 @@ async def index_crawled_urls( # Extract useful metadata title = metadata.get("title", url) - description = metadata.get("description", "") - language = metadata.get("language", "") + metadata.get("description", "") + metadata.get("language", "") # Update title immediately for better UX document.title = title diff --git a/surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py b/surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py index 17e96de4f..a005a9e72 100644 --- a/surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py +++ b/surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py @@ -34,9 +34,9 @@ class AnonymousProxiesProvider(ProxyProvider): "l": Config.RESIDENTIAL_PROXY_LOCATION, "t": Config.RESIDENTIAL_PROXY_TYPE, } - return base64.b64encode( - json.dumps(password_dict).encode("utf-8") - ).decode("utf-8") + return base64.b64encode(json.dumps(password_dict).encode("utf-8")).decode( + "utf-8" + ) def get_proxy_url(self) -> str | None: username = Config.RESIDENTIAL_PROXY_USERNAME diff --git a/surfsense_backend/app/zero_publication.py b/surfsense_backend/app/zero_publication.py index c3e41ef9b..d2755d0a1 100644 --- a/surfsense_backend/app/zero_publication.py +++ b/surfsense_backend/app/zero_publication.py @@ -110,7 +110,9 @@ def _format_table_entry(conn: Connection, table: str) -> str: def build_set_table_sql(conn: Connection) -> str: """Build the canonical plain SET TABLE statement for Zero's event triggers.""" - table_list = ", ".join(_format_table_entry(conn, table) for table in ZERO_PUBLICATION) + table_list = ", ".join( + _format_table_entry(conn, table) for table in ZERO_PUBLICATION + ) return f"ALTER PUBLICATION {_quote_identifier(PUBLICATION_NAME)} SET TABLE {table_list}" @@ -175,7 +177,9 @@ def verify_publication(conn: Connection) -> list[str]: actual_columns = actual[table] actual_key = sorted(actual_columns) if actual_columns is not None else None - expected_key = sorted(expected_columns) if expected_columns is not None else None + expected_key = ( + sorted(expected_columns) if expected_columns is not None else None + ) if actual_key != expected_key: mismatches.append( f"{table}: expected columns {expected_columns or 'ALL'}, " @@ -196,6 +200,7 @@ async def _verify_cli() -> int: engine = create_async_engine(database_url) async with engine.connect() as async_conn: + def run_verify(sync_conn: Connection) -> list[str]: return verify_publication(sync_conn) @@ -215,7 +220,9 @@ async def _verify_cli() -> int: def main() -> int: parser = argparse.ArgumentParser(description="Manage SurfSense's Zero publication") - parser.add_argument("--verify", action="store_true", help="verify zero_publication shape") + parser.add_argument( + "--verify", action="store_true", help="verify zero_publication shape" + ) args = parser.parse_args() if args.verify: diff --git a/surfsense_backend/scripts/register_webhook.py b/surfsense_backend/scripts/register_webhook.py index 44ead9470..d1b570943 100644 --- a/surfsense_backend/scripts/register_webhook.py +++ b/surfsense_backend/scripts/register_webhook.py @@ -41,7 +41,9 @@ async def main() -> int: await session.commit() account_id = int(account.id) - webhook_url = f"{base_url.rstrip('/')}/api/v1/gateway/webhooks/telegram/{account_id}" + webhook_url = ( + f"{base_url.rstrip('/')}/api/v1/gateway/webhooks/telegram/{account_id}" + ) bot = Bot(token=token) ok = await bot.set_webhook( url=webhook_url, @@ -58,4 +60,3 @@ async def main() -> int: if __name__ == "__main__": raise SystemExit(asyncio.run(main())) - diff --git a/surfsense_backend/tests/e2e/fakes/mcp_runtime.py b/surfsense_backend/tests/e2e/fakes/mcp_runtime.py index 5e4ef403f..5cd465af1 100644 --- a/surfsense_backend/tests/e2e/fakes/mcp_runtime.py +++ b/surfsense_backend/tests/e2e/fakes/mcp_runtime.py @@ -140,7 +140,10 @@ def install(active_patches: list[Any]) -> None: "app.agents.chat.multi_agent_chat.shared.tools.mcp.tool.streamablehttp_client", _fake_streamablehttp_client, ), - ("app.agents.chat.multi_agent_chat.shared.tools.mcp.tool.ClientSession", _FakeClientSession), + ( + "app.agents.chat.multi_agent_chat.shared.tools.mcp.tool.ClientSession", + _FakeClientSession, + ), ] for target, replacement in targets: p = patch(target, replacement) diff --git a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_agent_turn.py b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_agent_turn.py index b30744177..d45570484 100644 --- a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_agent_turn.py +++ b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_agent_turn.py @@ -135,8 +135,6 @@ async def test_agent_checkpoint_round_trips_across_turns( {"messages": [HumanMessage(content="second turn")]}, config ) - texts = [ - m.content for m in second["messages"] if isinstance(m, HumanMessage) - ] + texts = [m.content for m in second["messages"] if isinstance(m, HumanMessage)] assert "remember apple" in texts, "turn 1 history not reloaded from checkpoint" assert len(second["messages"]) > len(first["messages"]) diff --git a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py index 4c624d80d..e013ef35b 100644 --- a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py +++ b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py @@ -45,9 +45,7 @@ def _build_desktop_fs_mw(root: Path): """Build the production filesystem middleware bound to a real local folder.""" selection = FilesystemSelection( mode=FilesystemMode.DESKTOP_LOCAL_FOLDER, - local_mounts=( - LocalFilesystemMount(mount_id=_MOUNT_ID, root_path=str(root)), - ), + local_mounts=(LocalFilesystemMount(mount_id=_MOUNT_ID, root_path=str(root)),), ) resolver = build_backend_resolver(selection) return build_filesystem_mw( @@ -157,7 +155,7 @@ async def test_write_then_ls_lists_file(tmp_path: Path): async def test_edit_file_rewrites_on_disk(tmp_path: Path): """edit_file applies a real string replacement to the on-disk file.""" - result = await _run( + await _run( tmp_path, [ ScriptedTurn( diff --git a/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py b/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py index b3bb241a3..311716052 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py @@ -7,9 +7,7 @@ from app.indexing_pipeline.adapters.file_upload_adapter import UploadDocumentAda pytestmark = pytest.mark.integration -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_sets_status_ready(db_session, db_search_space, db_user, mocker): """Document status is READY after successful indexing.""" adapter = UploadDocumentAdapter(db_session) @@ -29,9 +27,7 @@ async def test_sets_status_ready(db_session, db_search_space, db_user, mocker): assert DocumentStatus.is_state(document.status, DocumentStatus.READY) -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_content_is_source_markdown(db_session, db_search_space, db_user, mocker): """Document content is set to the extracted source markdown.""" adapter = UploadDocumentAdapter(db_session) @@ -51,9 +47,7 @@ async def test_content_is_source_markdown(db_session, db_search_space, db_user, assert document.content == "## Hello\n\nSome content." -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_chunks_written_to_db(db_session, db_search_space, db_user, mocker): """Chunks derived from the source markdown are persisted in the DB.""" adapter = UploadDocumentAdapter(db_session) @@ -98,9 +92,7 @@ async def test_raises_on_indexing_failure(db_session, db_search_space, db_user, # --------------------------------------------------------------------------- -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_reindex_updates_content(db_session, db_search_space, db_user, mocker): """Document content is updated to the new source markdown after reindexing.""" adapter = UploadDocumentAdapter(db_session) @@ -126,9 +118,7 @@ async def test_reindex_updates_content(db_session, db_search_space, db_user, moc assert document.content == "## Edited\n\nNew content after user edit." -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_reindex_updates_content_hash( db_session, db_search_space, db_user, mocker ): @@ -157,9 +147,7 @@ async def test_reindex_updates_content_hash( assert document.content_hash != original_hash -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_reindex_sets_status_ready(db_session, db_search_space, db_user, mocker): """Document status is READY after successful reindexing.""" adapter = UploadDocumentAdapter(db_session) @@ -222,9 +210,7 @@ async def test_reindex_replaces_chunks(db_session, db_search_space, db_user, moc assert chunks[0].content == "Updated chunk." -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_reindex_clears_reindexing_flag( db_session, db_search_space, db_user, mocker ): diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py index 95afee5ef..8e1ed3752 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py @@ -34,9 +34,7 @@ def _cal_doc( ) -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_calendar_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker ): @@ -65,9 +63,7 @@ async def test_calendar_pipeline_creates_ready_document( assert DocumentStatus.is_state(row.status, DocumentStatus.READY) -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_calendar_legacy_doc_migrated( db_session, db_search_space, db_connector, db_user, mocker ): diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py index 4e8b8a4a2..c7565f4ba 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py @@ -33,9 +33,7 @@ def _drive_doc( ) -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_drive_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker ): @@ -64,9 +62,7 @@ async def test_drive_pipeline_creates_ready_document( assert DocumentStatus.is_state(row.status, DocumentStatus.READY) -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_drive_legacy_doc_migrated( db_session, db_search_space, db_connector, db_user, mocker ): diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py index d2a8cefc5..9faa3db91 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py @@ -32,9 +32,7 @@ def _dropbox_doc( ) -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_dropbox_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker ): @@ -63,9 +61,7 @@ async def test_dropbox_pipeline_creates_ready_document( assert DocumentStatus.is_state(row.status, DocumentStatus.READY) -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_dropbox_duplicate_content_skipped( db_session, db_search_space, db_connector, db_user, mocker ): diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py index 5b2efa1aa..2026393c5 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py @@ -36,9 +36,7 @@ def _gmail_doc( ) -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_gmail_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker ): @@ -68,9 +66,7 @@ async def test_gmail_pipeline_creates_ready_document( assert row.source_markdown == doc.source_markdown -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_gmail_legacy_doc_migrated_then_reused( db_session, db_search_space, db_connector, db_user, mocker ): diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py b/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py index 59b7c8814..855676f61 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py @@ -9,9 +9,7 @@ from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineServ pytestmark = pytest.mark.integration -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_index_batch_creates_ready_documents( db_session, db_search_space, make_connector_document, mocker ): @@ -49,9 +47,7 @@ async def test_index_batch_creates_ready_documents( assert row.embedding is not None -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_index_batch_empty_returns_empty(db_session, mocker): """index_batch with empty input returns an empty list.""" service = IndexingPipelineService(session=db_session) diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py index 41ac6894b..e368ec256 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py @@ -32,9 +32,7 @@ def _onedrive_doc( ) -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_onedrive_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker ): @@ -63,9 +61,7 @@ async def test_onedrive_pipeline_creates_ready_document( assert DocumentStatus.is_state(row.status, DocumentStatus.READY) -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_onedrive_duplicate_content_skipped( db_session, db_search_space, db_connector, db_user, mocker ): diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py b/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py index d0b8c7fed..4b6662fc8 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py @@ -32,9 +32,7 @@ async def test_new_document_is_persisted_with_pending_status( assert reloaded.source_markdown == doc.source_markdown -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_unchanged_ready_document_is_skipped( db_session, db_search_space, @@ -55,9 +53,7 @@ async def test_unchanged_ready_document_is_skipped( assert results == [] -@pytest.mark.usefixtures( -"patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_title_only_change_updates_title_in_db( db_session, db_search_space, diff --git a/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py b/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py index eed5b286f..894f036f0 100644 --- a/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py +++ b/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py @@ -47,7 +47,9 @@ async def test_comment_reply_truncates_long_preview( db_session: AsyncSession, db_user: User, db_search_space: SearchSpace ): """A long comment preview is truncated in the reply message.""" - notification = await _notify(db_session, db_user, db_search_space, preview="y" * 150) + notification = await _notify( + db_session, db_user, db_search_space, preview="y" * 150 + ) assert notification.message == "y" * 100 + "..." diff --git a/surfsense_backend/tests/integration/notifications/test_inbox_api.py b/surfsense_backend/tests/integration/notifications/test_inbox_api.py index 461e5c857..524a0ba60 100644 --- a/surfsense_backend/tests/integration/notifications/test_inbox_api.py +++ b/surfsense_backend/tests/integration/notifications/test_inbox_api.py @@ -12,7 +12,7 @@ from datetime import UTC, datetime, timedelta import pytest from sqlalchemy.ext.asyncio import AsyncSession -from app.db import SearchSpace, User +from app.db import User from app.notifications.persistence import Notification pytestmark = pytest.mark.integration diff --git a/surfsense_backend/tests/integration/notifications/test_mention_handler.py b/surfsense_backend/tests/integration/notifications/test_mention_handler.py index dc25f7888..3254d737c 100644 --- a/surfsense_backend/tests/integration/notifications/test_mention_handler.py +++ b/surfsense_backend/tests/integration/notifications/test_mention_handler.py @@ -47,7 +47,9 @@ async def test_new_mention_truncates_long_preview( db_session: AsyncSession, db_user: User, db_search_space: SearchSpace ): """A long comment preview is truncated in the mention message.""" - notification = await _notify(db_session, db_user, db_search_space, preview="x" * 150) + notification = await _notify( + db_session, db_user, db_search_space, preview="x" * 150 + ) assert notification.message == "x" * 100 + "..." diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py b/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py index 3aae7cc75..9b3931549 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py @@ -137,7 +137,10 @@ class TestPluginLoaderIsolation: _FakeEntryPoint("crashing", crashing_factory), _FakeEntryPoint("ok", year_substituter_factory), ] - with patch("app.agents.chat.multi_agent_chat.main_agent.plugins.loader.entry_points", return_value=eps): + with patch( + "app.agents.chat.multi_agent_chat.main_agent.plugins.loader.entry_points", + return_value=eps, + ): result = load_plugin_middlewares( _ctx(), allowed_plugin_names={"crashing", "ok"} ) diff --git a/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py b/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py index 951c2d124..de4386abb 100644 --- a/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py +++ b/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py @@ -5,8 +5,7 @@ import asyncio import pytest import pytest_asyncio -from app.gateway import byo_long_poll -from app.gateway import runner +from app.gateway import byo_long_poll, runner class ScalarResult: @@ -48,7 +47,9 @@ async def test_start_byo_long_poll_noops_when_mode_is_webhook(monkeypatch): @pytest.mark.asyncio async def test_start_byo_long_poll_noops_when_no_byo_accounts(mocker, monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll") + monkeypatch.setattr( + byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll" + ) session = mocker.AsyncMock() session.execute.return_value = ScalarResult([]) monkeypatch.setattr( @@ -63,8 +64,12 @@ async def test_start_byo_long_poll_noops_when_no_byo_accounts(mocker, monkeypatc @pytest.mark.asyncio -async def test_start_byo_long_poll_spawns_one_supervisor_per_account(mocker, monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll") +async def test_start_byo_long_poll_spawns_one_supervisor_per_account( + mocker, monkeypatch +): + monkeypatch.setattr( + byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll" + ) accounts = [mocker.Mock(id=1), mocker.Mock(id=2)] session = mocker.AsyncMock() session.execute.return_value = ScalarResult(accounts) @@ -73,7 +78,9 @@ async def test_start_byo_long_poll_spawns_one_supervisor_per_account(mocker, mon "async_session_maker", lambda: SessionContext(session), ) - monkeypatch.setattr(byo_long_poll, "account_token", lambda account: f"token-{account.id}") + monkeypatch.setattr( + byo_long_poll, "account_token", lambda account: f"token-{account.id}" + ) async def forever(_account_id: int, _token: str) -> None: await asyncio.Event().wait() @@ -108,7 +115,9 @@ async def test_supervisor_retries_after_run_returns(mocker, monkeypatch): @pytest.mark.asyncio async def test_shutdown_cancels_running_supervisors(mocker, monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll") + monkeypatch.setattr( + byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll" + ) session = mocker.AsyncMock() session.execute.return_value = ScalarResult([mocker.Mock(id=1)]) monkeypatch.setattr( @@ -130,7 +139,9 @@ async def test_shutdown_cancels_running_supervisors(mocker, monkeypatch): @pytest.mark.asyncio -async def test_run_telegram_account_persists_for_fastapi_inbox_worker(mocker, monkeypatch): +async def test_run_telegram_account_persists_for_fastapi_inbox_worker( + mocker, monkeypatch +): class ConnectionContext: async def __aenter__(self): conn = mocker.AsyncMock() @@ -169,4 +180,3 @@ async def test_run_telegram_account_persists_for_fastapi_inbox_worker(mocker, mo second_session.commit.assert_awaited_once() persist.assert_awaited_once() assert persist.await_args.kwargs["request_id"].startswith("gateway_") - diff --git a/surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py b/surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py index 5fe46502f..8be8942cc 100644 --- a/surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py +++ b/surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py @@ -5,7 +5,9 @@ from app.tasks.celery_tasks import gateway_tasks def test_enqueue_received_sweep_is_noop_guard(mocker): apply_async = mocker.Mock() - mocker.patch.object(gateway_tasks.process_inbound_event_task, "apply_async", apply_async) + mocker.patch.object( + gateway_tasks.process_inbound_event_task, "apply_async", apply_async + ) info = mocker.patch.object(gateway_tasks.logger, "info") replayed = gateway_tasks.enqueue_received_sweep_task.run() @@ -13,4 +15,3 @@ def test_enqueue_received_sweep_is_noop_guard(mocker): apply_async.assert_not_called() assert replayed == 0 info.assert_called_once() - diff --git a/surfsense_backend/tests/unit/gateway/test_formatting.py b/surfsense_backend/tests/unit/gateway/test_formatting.py index 61c7ea20f..4d842e169 100644 --- a/surfsense_backend/tests/unit/gateway/test_formatting.py +++ b/surfsense_backend/tests/unit/gateway/test_formatting.py @@ -15,4 +15,3 @@ def test_chunk_message_preserves_content_and_limits_size(): assert "".join(chunks) == text assert len(chunks) > 1 assert all(len(chunk.encode("utf-16-le")) // 2 <= 4096 for chunk in chunks) - diff --git a/surfsense_backend/tests/unit/gateway/test_hitl_filter.py b/surfsense_backend/tests/unit/gateway/test_hitl_filter.py index 90f94b6ab..04766b986 100644 --- a/surfsense_backend/tests/unit/gateway/test_hitl_filter.py +++ b/surfsense_backend/tests/unit/gateway/test_hitl_filter.py @@ -12,4 +12,3 @@ def test_filter_hitl_tools_removes_known_approval_tools(): filtered = filter_hitl_tools(tools) assert [getattr(tool, "name", tool) for tool in filtered] == ["search", "summarize"] - diff --git a/surfsense_backend/tests/unit/gateway/test_inbox_worker.py b/surfsense_backend/tests/unit/gateway/test_inbox_worker.py index 8ecc4d86a..1e5b2a184 100644 --- a/surfsense_backend/tests/unit/gateway/test_inbox_worker.py +++ b/surfsense_backend/tests/unit/gateway/test_inbox_worker.py @@ -8,7 +8,9 @@ from app.gateway import inbox_worker @pytest.mark.asyncio -async def test_inbox_worker_claims_and_processes_in_fastapi_process(mocker, monkeypatch): +async def test_inbox_worker_claims_and_processes_in_fastapi_process( + mocker, monkeypatch +): claim = mocker.AsyncMock(return_value=7) process = mocker.AsyncMock(side_effect=asyncio.CancelledError) monkeypatch.setattr(inbox_worker, "claim_next_inbound_event", claim) @@ -42,4 +44,3 @@ async def test_start_stop_gateway_inbox_worker(mocker, monkeypatch): assert stopped.is_set() assert inbox_worker._task is None - diff --git a/surfsense_backend/tests/unit/gateway/test_pairing.py b/surfsense_backend/tests/unit/gateway/test_pairing.py index facf908cd..9f90fa259 100644 --- a/surfsense_backend/tests/unit/gateway/test_pairing.py +++ b/surfsense_backend/tests/unit/gateway/test_pairing.py @@ -38,4 +38,3 @@ async def test_redeem_pairing_code_binds_pending_row(mocker): assert binding.state == ExternalChatBindingState.BOUND assert binding.external_peer_id == "telegram:123" assert binding.pairing_code is None - diff --git a/surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py b/surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py index 484eacd1a..a929ff12a 100644 --- a/surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py +++ b/surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py @@ -9,5 +9,6 @@ def test_process_inbound_event_task_is_noop_guard(mocker): assert gateway_tasks.process_inbound_event_task.run(123) is None warning.assert_called_once() - assert "FastAPI owns external chat agent turn processing" in warning.call_args.args[0] - + assert ( + "FastAPI owns external chat agent turn processing" in warning.call_args.args[0] + ) diff --git a/surfsense_backend/tests/unit/gateway/test_webhook_routes.py b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py index b686ebcb8..aa8bd3a89 100644 --- a/surfsense_backend/tests/unit/gateway/test_webhook_routes.py +++ b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py @@ -23,7 +23,9 @@ def _enable_gateways(monkeypatch): monkeypatch.setattr(routes.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "webhook") monkeypatch.setattr(routes.config, "TELEGRAM_SHARED_BOT_TOKEN", "telegram-token") monkeypatch.setattr(routes.config, "TELEGRAM_SHARED_BOT_USERNAME", "surf_bot") - monkeypatch.setattr(routes.config, "TELEGRAM_WEBHOOK_SECRET", "telegram-webhook-secret") + monkeypatch.setattr( + routes.config, "TELEGRAM_WEBHOOK_SECRET", "telegram-webhook-secret" + ) monkeypatch.setattr(routes.config, "GATEWAY_SLACK_ENABLED", True) monkeypatch.setattr(routes.config, "GATEWAY_SLACK_CLIENT_ID", "slack-client") @@ -37,7 +39,9 @@ def _enable_gateways(monkeypatch): class RequestStub: - def __init__(self, payload=None, *, headers=None, json_exc: Exception | None = None): + def __init__( + self, payload=None, *, headers=None, json_exc: Exception | None = None + ): self.headers = headers or {} self._payload = payload self._json_exc = json_exc @@ -70,7 +74,9 @@ def _slack_account() -> ExternalChatAccount: ) -def _signed_slack_request(payload: dict, *, secret: str = "signing-secret") -> RequestStub: +def _signed_slack_request( + payload: dict, *, secret: str = "signing-secret" +) -> RequestStub: body = json.dumps(payload).encode() timestamp = str(int(time.time())) digest = hmac.new( @@ -195,7 +201,9 @@ async def test_telegram_webhook_persists_for_fastapi_inbox_worker(mocker, monkey async def test_telegram_webhook_commits_dedup_without_enqueue(mocker, monkeypatch): session = mocker.AsyncMock() session.get.return_value = _account() - monkeypatch.setattr(routes, "persist_inbound_event", mocker.AsyncMock(return_value=None)) + monkeypatch.setattr( + routes, "persist_inbound_event", mocker.AsyncMock(return_value=None) + ) request = RequestStub( {"update_id": 10, "message": {"message_id": 7}}, @@ -250,7 +258,11 @@ async def test_slack_webhook_url_verification(monkeypatch, mocker): async def test_slack_webhook_persists_event(monkeypatch, mocker): _enable_slack_gateway(monkeypatch) session = mocker.AsyncMock() - monkeypatch.setattr(routes, "get_slack_account_by_team", mocker.AsyncMock(return_value=_slack_account())) + monkeypatch.setattr( + routes, + "get_slack_account_by_team", + mocker.AsyncMock(return_value=_slack_account()), + ) persist = mocker.AsyncMock(return_value=100) monkeypatch.setattr(routes, "persist_inbound_event", persist) payload = { @@ -280,7 +292,11 @@ async def test_slack_webhook_persists_event(monkeypatch, mocker): async def test_slack_webhook_ignores_self_event(monkeypatch, mocker): _enable_slack_gateway(monkeypatch) session = mocker.AsyncMock() - monkeypatch.setattr(routes, "get_slack_account_by_team", mocker.AsyncMock(return_value=_slack_account())) + monkeypatch.setattr( + routes, + "get_slack_account_by_team", + mocker.AsyncMock(return_value=_slack_account()), + ) persist = mocker.AsyncMock(return_value=100) monkeypatch.setattr(routes, "persist_inbound_event", persist) request = _signed_slack_request( @@ -331,4 +347,3 @@ def test_discord_gateway_callback_does_not_create_search_source_connector(): callback_source = inspect.getsource(routes.discord_gateway_callback) assert "SearchSourceConnector" not in callback_source - diff --git a/surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py b/surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py index a4e23c39f..1e648a9c9 100644 --- a/surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py +++ b/surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py @@ -81,7 +81,9 @@ class TestCwdDefaults: class TestRelativePathResolution: def test_relative_path_resolves_against_cwd(self): assert ( - resolve_relative(_mw(), "notes.md", _runtime({"cwd": "/documents/projects"})) + resolve_relative( + _mw(), "notes.md", _runtime({"cwd": "/documents/projects"}) + ) == "/documents/projects/notes.md" ) @@ -281,7 +283,11 @@ class TestNormalizeLocalMountPath: _desktop_mw(backend), "/brand-new-note.md", _runtime( - {"file_operation_contract": {"suggested_path": "/root_b/notes/context.md"}} + { + "file_operation_contract": { + "suggested_path": "/root_b/notes/context.md" + } + } ), ) assert resolved == "/root_b/brand-new-note.md" diff --git a/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py b/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py index 7724a4852..e78db1e76 100644 --- a/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py +++ b/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py @@ -15,7 +15,9 @@ from unittest.mock import AsyncMock import numpy as np import pytest -from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence import middleware as kb_persistence +from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence import ( + middleware as kb_persistence, +) from app.db import Document diff --git a/surfsense_backend/tests/unit/middleware/test_kb_persistence_revisions.py b/surfsense_backend/tests/unit/middleware/test_kb_persistence_revisions.py index 500c6cc60..023213aaa 100644 --- a/surfsense_backend/tests/unit/middleware/test_kb_persistence_revisions.py +++ b/surfsense_backend/tests/unit/middleware/test_kb_persistence_revisions.py @@ -21,7 +21,9 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence import middleware as kb_persistence +from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence import ( + middleware as kb_persistence, +) pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/middleware/test_knowledge_search.py b/surfsense_backend/tests/unit/middleware/test_knowledge_search.py index 25de7308d..027738fba 100644 --- a/surfsense_backend/tests/unit/middleware/test_knowledge_search.py +++ b/surfsense_backend/tests/unit/middleware/test_knowledge_search.py @@ -261,7 +261,8 @@ class TestKnowledgePriorityMiddlewarePlanner: return [] monkeypatch.setattr( - ks, "search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) @@ -304,7 +305,8 @@ class TestKnowledgePriorityMiddlewarePlanner: return [] monkeypatch.setattr( - ks, "search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) @@ -333,7 +335,8 @@ class TestKnowledgePriorityMiddlewarePlanner: return [] monkeypatch.setattr( - ks, "search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) @@ -378,11 +381,13 @@ class TestKnowledgePriorityMiddlewarePlanner: return [] monkeypatch.setattr( - ks, "browse_recent_documents", + ks, + "browse_recent_documents", fake_browse_recent_documents, ) monkeypatch.setattr( - ks, "search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) @@ -425,11 +430,13 @@ class TestKnowledgePriorityMiddlewarePlanner: return [] monkeypatch.setattr( - ks, "browse_recent_documents", + ks, + "browse_recent_documents", fake_browse_recent_documents, ) monkeypatch.setattr( - ks, "search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) @@ -552,11 +559,13 @@ class TestKnowledgePriorityMentionDrain: return [] monkeypatch.setattr( - ks, "fetch_mentioned_documents", + ks, + "fetch_mentioned_documents", fake_fetch_mentioned_documents, ) monkeypatch.setattr( - ks, "search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) @@ -600,11 +609,13 @@ class TestKnowledgePriorityMentionDrain: return [] monkeypatch.setattr( - ks, "fetch_mentioned_documents", + ks, + "fetch_mentioned_documents", fake_fetch_mentioned_documents, ) monkeypatch.setattr( - ks, "search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) @@ -645,11 +656,13 @@ class TestKnowledgePriorityMentionDrain: return [] monkeypatch.setattr( - ks, "fetch_mentioned_documents", + ks, + "fetch_mentioned_documents", fake_fetch_mentioned_documents, ) monkeypatch.setattr( - ks, "search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) diff --git a/surfsense_backend/tests/unit/notifications/api/test_transform.py b/surfsense_backend/tests/unit/notifications/api/test_transform.py index ba12ab3cf..96624fe61 100644 --- a/surfsense_backend/tests/unit/notifications/api/test_transform.py +++ b/surfsense_backend/tests/unit/notifications/api/test_transform.py @@ -50,18 +50,18 @@ class TestParseBeforeDate: def _notification(**overrides) -> Notification: - defaults = dict( - id=1, - user_id=uuid.uuid4(), - search_space_id=3, - type="document_processing", - title="Title", - message="Message", - read=False, - notification_metadata={"k": "v"}, - created_at=datetime(2024, 1, 1, tzinfo=UTC), - updated_at=datetime(2024, 1, 2, tzinfo=UTC), - ) + defaults = { + "id": 1, + "user_id": uuid.uuid4(), + "search_space_id": 3, + "type": "document_processing", + "title": "Title", + "message": "Message", + "read": False, + "notification_metadata": {"k": "v"}, + "created_at": datetime(2024, 1, 1, tzinfo=UTC), + "updated_at": datetime(2024, 1, 2, tzinfo=UTC), + } defaults.update(overrides) return Notification(**defaults) diff --git a/surfsense_backend/tests/unit/notifications/service/messages/test_page_limit.py b/surfsense_backend/tests/unit/notifications/service/messages/test_page_limit.py index 9b2ac9638..606e985f2 100644 --- a/surfsense_backend/tests/unit/notifications/service/messages/test_page_limit.py +++ b/surfsense_backend/tests/unit/notifications/service/messages/test_page_limit.py @@ -16,7 +16,9 @@ def test_operation_id_encodes_search_space(): def test_summary_title_and_message(): """The summary states the document and the used/limit page counts.""" - title, message = msg.summary("short.pdf", pages_used=95, pages_limit=100, pages_to_add=10) + title, message = msg.summary( + "short.pdf", pages_used=95, pages_limit=100, pages_to_add=10 + ) assert title == "Page limit exceeded: short.pdf" assert message == ( "This document has ~10 page(s) but you've used 95/100 pages. " diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx index 4d6382a60..ab6168305 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx @@ -89,13 +89,7 @@ export function AutomationDefinitionSection({ definition }: AutomationDefinition ); } -function Field({ - label, - children, -}: { - label: React.ReactNode; - children: React.ReactNode; -}) { +function Field({ label, children }: { label: React.ReactNode; children: React.ReactNode }) { return (
{label}
diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx index 4a6537385..71730baeb 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx @@ -101,12 +101,21 @@ export function AutomationDetailHeader({ disabled={updating} className="relative justify-start rounded-md bg-muted px-3 hover:bg-accent" > - + {pauseLabel} {updating && ( - + )} )} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx index 2f4eea7b8..abe739dcc 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx @@ -26,9 +26,7 @@ export function AutomationTriggersSection({ Triggers -

- When this automation runs -

+

When this automation runs

 {triggers.length === 0 ? ( diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx index 15a285322..7505ef49b 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx @@ -77,8 +77,10 @@ function getStepDetails(step: PlanStep): { label: string; value: string }[] { if (step.when) details.push({ label: "Runs when", value: step.when }); if (step.output_as) details.push({ label: "Saves output as", value: step.output_as }); - if (step.max_retries != null) details.push({ label: "Max retries", value: String(step.max_retries) }); - if (step.timeout_seconds != null) details.push({ label: "Timeout", value: `${step.timeout_seconds}s` }); + if (step.max_retries != null) + details.push({ label: "Max retries", value: String(step.max_retries) }); + if (step.timeout_seconds != null) + details.push({ label: "Timeout", value: `${step.timeout_seconds}s` }); return details; } diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx index 6e6f84bd0..de156a09c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx @@ -23,13 +23,13 @@ import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; import { type Trigger, triggerUpdateRequest } from "@/contracts/types/automation.types"; import { describeCron } from "@/lib/automations/describe-cron"; -import { formatRelativeFutureDate } from "@/lib/format-date"; import { DEFAULT_SCHEDULE, fromCron, type ScheduleFrequency, toCron, } from "@/lib/automations/schedule-builder"; +import { formatRelativeFutureDate } from "@/lib/format-date"; import { TimezoneCombobox } from "../../components/builder/timezone-combobox"; import { DeleteTriggerDialog } from "./delete-trigger-dialog"; @@ -277,7 +277,10 @@ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: Tri
) : draft.frequency !== "custom" ? (
-
); diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx index 31dc6b56a..335c7f084 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx @@ -3,4 +3,3 @@ import { MessagingChannelsContent } from "../components/MessagingChannelsContent export default function Page() { return ; } - diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx index 08d20f2ab..78d48070a 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx @@ -35,9 +35,7 @@ export const MCPTrustedTools: FC = ({ connector }) => { return (
-

- Trusted Tools -

+

Trusted Tools

diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx index d61460d48..011eeec96 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx @@ -278,7 +278,10 @@ export const ConnectorEditView: FC = ({ {/* Vision toggle (Obsidian is plugin-push, non-indexable by design) */} {showsVisionToggle && ( - + )} {/* Date-range and periodic sync stay indexable-only */} @@ -310,11 +313,13 @@ export const ConnectorEditView: FC = ({ connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR"; const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive; const selectedFolders = - (connector.config?.selected_folders as Array<{ id: string; name: string }> | undefined) || - []; + (connector.config?.selected_folders as + | Array<{ id: string; name: string }> + | undefined) || []; const selectedFiles = - (connector.config?.selected_files as Array<{ id: string; name: string }> | undefined) || - []; + (connector.config?.selected_files as + | Array<{ id: string; name: string }> + | undefined) || []; const hasItemsSelected = selectedFolders.length > 0 || selectedFiles.length > 0; const isDisabled = requiresFolderSelection && !hasItemsSelected; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx index 218b3d329..097360e14 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx @@ -177,7 +177,10 @@ export const IndexingConfigurationView: FC = ({ {/* Vision toggle (Obsidian is plugin-push, non-indexable by design) */} {showsVisionToggle && ( - + )} {/* Date-range and periodic sync stay indexable-only */} diff --git a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts index aa9440a12..45c174d74 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts @@ -654,7 +654,7 @@ export const useConnectorDialog = () => { setConnectorConfig(connector.config || {}); setPeriodicEnabled(false); setFrequencyMinutes("1440"); - setEnableVisionLlm(connector.enable_vision_llm ?? false); + setEnableVisionLlm(connector.enable_vision_llm ?? false); setStartDate(undefined); setEndDate(undefined); @@ -808,7 +808,7 @@ export const useConnectorDialog = () => { await updateConnector({ id: indexingConfig.connectorId, data: { - enable_vision_llm: enableVisionLlm, + enable_vision_llm: enableVisionLlm, ...(periodicEnabled && { periodic_indexing_enabled: true, indexing_frequency_minutes: frequency, diff --git a/surfsense_web/components/new-chat/chat-example-prompts.tsx b/surfsense_web/components/new-chat/chat-example-prompts.tsx index ee2e86daf..1217d994e 100644 --- a/surfsense_web/components/new-chat/chat-example-prompts.tsx +++ b/surfsense_web/components/new-chat/chat-example-prompts.tsx @@ -2,9 +2,9 @@ import { FilePlus2, + type LucideIcon, Search, Settings2, - type LucideIcon, WandSparkles, Workflow, X, @@ -55,7 +55,31 @@ export function ChatExamplePrompts({ onSelect }: ChatExamplePromptsProps) { return (

- {activeCategory ? null : ( + {activeCategory ? ( +
+
+ + {activeCategory.label} +
+ +
    + {activeCategory.prompts.map((prompt) => ( +
  • + +
    • + ))} +
+ +
+ ) : (
{CHAT_EXAMPLE_CATEGORIES.map((category) => { @@ -76,24 +100,7 @@ export function ChatExamplePrompts({ onSelect }: ChatExamplePromptsProps) { })}
- {CHAT_EXAMPLE_CATEGORIES.map((category) => ( - - -
    - {category.prompts.map((prompt) => ( -
  • - -
    • - ))} -
- - - ))} - + )}
); -} \ No newline at end of file +} diff --git a/surfsense_web/components/settings/llm-role-manager.tsx b/surfsense_web/components/settings/llm-role-manager.tsx index 6d47a7417..c32e79a8e 100644 --- a/surfsense_web/components/settings/llm-role-manager.tsx +++ b/surfsense_web/components/settings/llm-role-manager.tsx @@ -138,7 +138,7 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { useEffect(() => { setAssignments({ agent_llm_id: preferences.agent_llm_id ?? null, - image_generation_config_id: preferences.image_generation_config_id ?? null, + image_generation_config_id: preferences.image_generation_config_id ?? null, vision_llm_config_id: preferences.vision_llm_config_id ?? null, }); }, [ diff --git a/surfsense_web/components/sources/DocumentUploadTab.tsx b/surfsense_web/components/sources/DocumentUploadTab.tsx index 93b613a0a..3f68f6d64 100644 --- a/surfsense_web/components/sources/DocumentUploadTab.tsx +++ b/surfsense_web/components/sources/DocumentUploadTab.tsx @@ -693,7 +693,6 @@ export function DocumentUploadTab({
)} -

Enable Vision LLM

diff --git a/surfsense_web/components/sources/FolderWatchDialog.tsx b/surfsense_web/components/sources/FolderWatchDialog.tsx index ff168b2df..8c5629276 100644 --- a/surfsense_web/components/sources/FolderWatchDialog.tsx +++ b/surfsense_web/components/sources/FolderWatchDialog.tsx @@ -119,19 +119,13 @@ export function FolderWatchDialog({ setSubmitting(false); setProgress(null); } - }, [ - selectedFolder, - searchSpaceId, - supportedExtensions, - onOpenChange, - onSuccess, - ]); + }, [selectedFolder, searchSpaceId, supportedExtensions, onOpenChange, onSuccess]); const handleOpenChange = useCallback( (nextOpen: boolean) => { if (!nextOpen && !submitting) { setSelectedFolder(null); - setProgress(null); + setProgress(null); } onOpenChange(nextOpen); }, @@ -200,7 +194,6 @@ export function FolderWatchDialog({ {selectedFolder && ( <> - {progressLabel && (

{progressLabel}

diff --git a/surfsense_web/content/docs/messaging-channels/meta.json b/surfsense_web/content/docs/messaging-channels/meta.json index 00647bdb0..594fd95b9 100644 --- a/surfsense_web/content/docs/messaging-channels/meta.json +++ b/surfsense_web/content/docs/messaging-channels/meta.json @@ -1,13 +1,6 @@ { "title": "Messaging Channels", "icon": "MessageCircle", - "pages": [ - "telegram", - "whatsapp", - "slack", - "discord", - "docker", - "troubleshooting" - ], + "pages": ["telegram", "whatsapp", "slack", "discord", "docker", "troubleshooting"], "defaultOpen": false } diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts index bc2d609a6..5b50db0c1 100644 --- a/surfsense_web/lib/apis/documents-api.service.ts +++ b/surfsense_web/lib/apis/documents-api.service.ts @@ -3,6 +3,7 @@ import { createDocumentRequest, createDocumentResponse, type DeleteDocumentRequest, + type DocumentFileRead, deleteDocumentRequest, deleteDocumentResponse, documentTitleRead, @@ -16,6 +17,7 @@ import { getDocumentByChunkResponse, getDocumentChunksRequest, getDocumentChunksResponse, + getDocumentFilesResponse, getDocumentRequest, getDocumentResponse, getDocumentsRequest, @@ -30,8 +32,6 @@ import { searchDocumentsResponse, searchDocumentTitlesRequest, searchDocumentTitlesResponse, - type DocumentFileRead, - getDocumentFilesResponse, type UpdateDocumentRequest, type UploadDocumentRequest, updateDocumentRequest, diff --git a/surfsense_web/lib/folder-sync-upload.ts b/surfsense_web/lib/folder-sync-upload.ts index 14109b332..334d9550d 100644 --- a/surfsense_web/lib/folder-sync-upload.ts +++ b/surfsense_web/lib/folder-sync-upload.ts @@ -61,7 +61,7 @@ async function uploadBatchesWithConcurrency( folderName: string; searchSpaceId: number; rootFolderId: number | null; - processingMode?: "basic" | "premium"; + processingMode?: "basic" | "premium"; signal?: AbortSignal; onBatchComplete?: (filesInBatch: number) => void; } @@ -189,7 +189,7 @@ export async function uploadFolderScan(params: FolderSyncParams): Promise { uploaded += count; diff --git a/surfsense_web/zero/queries/automations.ts b/surfsense_web/zero/queries/automations.ts index 27b274acc..79772eb1f 100644 --- a/surfsense_web/zero/queries/automations.ts +++ b/surfsense_web/zero/queries/automations.ts @@ -6,9 +6,7 @@ import { zql } from "../schema/index"; // authorizes via `automation_id -> search_space`. No search_space_id on the // table by design. export const automationRunQueries = { - byAutomation: defineQuery( - z.object({ automationId: z.number() }), - ({ args: { automationId } }) => - zql.automation_runs.where("automationId", automationId).orderBy("createdAt", "desc") + byAutomation: defineQuery(z.object({ automationId: z.number() }), ({ args: { automationId } }) => + zql.automation_runs.where("automationId", automationId).orderBy("createdAt", "desc") ), };