From 69abf0d9165e3b89e3956a03023533e66eb0cdd0 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:28:34 +0530 Subject: [PATCH 01/63] feat: add python-telegram-bot dependency to project --- surfsense_backend/pyproject.toml | 1 + surfsense_backend/uv.lock | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 71c53caae..1e9dc4e23 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -87,6 +87,7 @@ dependencies = [ "opentelemetry-instrumentation-httpx>=0.61b0", "opentelemetry-instrumentation-celery>=0.61b0", "opentelemetry-instrumentation-logging>=0.61b0", + "python-telegram-bot>=22.7", ] [dependency-groups] diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index b902363dc..a5d712d6d 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -7030,6 +7030,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788, upload-time = "2024-08-07T17:33:28.192Z" }, ] +[[package]] +name = "python-telegram-bot" +version = "22.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpcore", marker = "python_full_version >= '3.14'" }, + { name = "httpx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/25/2258161b1069e66d6c39c0a602dbe57461d4767dc0012539970ea40bc9d6/python_telegram_bot-22.7.tar.gz", hash = "sha256:784b59ea3852fe4616ad63b4a0264c755637f5d725e87755ecdee28300febf61", size = 1516454, upload-time = "2026-03-16T09:36:03.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/f7/0e2f89dd62f45d46d4ea0d8aec5893ce5b37389638db010c117f46f11450/python_telegram_bot-22.7-py3-none-any.whl", hash = "sha256:d72eed532cf763758cd9331b57a6d790aff0bb4d37d8f4e92149436fe21c6475", size = 745365, upload-time = "2026-03-16T09:36:01.498Z" }, +] + [[package]] name = "pytz" version = "2026.1.post1" @@ -8187,6 +8200,7 @@ dependencies = [ { name = "pypandoc-binary" }, { name = "pypdf" }, { name = "python-ffmpeg" }, + { name = "python-telegram-bot" }, { name = "redis" }, { name = "rerankers", extra = ["flashrank"] }, { name = "sentence-transformers" }, @@ -8283,6 +8297,7 @@ requires-dist = [ { name = "pypandoc-binary", specifier = ">=1.16.2" }, { name = "pypdf", specifier = ">=5.1.0" }, { name = "python-ffmpeg", specifier = ">=2.0.12" }, + { name = "python-telegram-bot", specifier = ">=22.7" }, { name = "redis", specifier = ">=5.2.1" }, { name = "rerankers", extras = ["flashrank"], specifier = ">=0.7.1" }, { name = "sentence-transformers", specifier = ">=3.4.1" }, From 81cf63ac96e5bad9fa5aa2991bfb67a7754651f7 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:34:46 +0530 Subject: [PATCH 02/63] feat(gateway): add messaging gateway persistence schema --- .../versions/144_add_gateway_tables.py | 623 ++++++++++++++++++ surfsense_backend/app/db.py | 380 +++++++++++ surfsense_web/zero/schema/chat.ts | 2 + 3 files changed, 1005 insertions(+) create mode 100644 surfsense_backend/alembic/versions/144_add_gateway_tables.py diff --git a/surfsense_backend/alembic/versions/144_add_gateway_tables.py b/surfsense_backend/alembic/versions/144_add_gateway_tables.py new file mode 100644 index 000000000..011333d69 --- /dev/null +++ b/surfsense_backend/alembic/versions/144_add_gateway_tables.py @@ -0,0 +1,623 @@ +"""add gateway tables for Telegram messaging gateway + +Revision ID: 144 +Revises: 143 +Create Date: 2026-05-27 + +Adds the lean v6 gateway schema: + +* gateway_platform_accounts +* gateway_conversation_bindings +* gateway_inbound_events + +The gateway stores Telegram-originated conversations in the existing chat +tables but keeps them out of UI replication. This migration adds ``source`` to +``new_chat_messages`` as a denormalized Zero publication boundary and publishes +only ``source = 'web'`` rows. Gateway control-plane tables are served through +REST in v1, so they are intentionally not added to ``zero_publication``. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +revision: str = "144" +down_revision: str | None = "143" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +PUBLICATION_NAME = "zero_publication" + +DOCUMENT_COLS = [ + "id", + "title", + "document_type", + "search_space_id", + "folder_id", + "created_by_id", + "status", + "created_at", + "updated_at", +] + +USER_COLS = [ + "id", + "pages_limit", + "pages_used", + "premium_credit_micros_limit", + "premium_credit_micros_used", +] + +def _has_zero_version(conn, table: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = :tbl AND column_name = '_0_version'" + ), + {"tbl": table}, + ).fetchone() + is not None + ) + + +def _cols(columns: list[str]) -> str: + return ", ".join(columns) + + +def _table_exists(conn, table: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.tables " + "WHERE table_schema = current_schema() AND table_name = :tbl" + ), + {"tbl": table}, + ).fetchone() + is not None + ) + + +def _column_exists(conn, table: str, column: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_schema = current_schema() " + "AND table_name = :tbl AND column_name = :col" + ), + {"tbl": table, "col": column}, + ).fetchone() + is not None + ) + + +def _index_exists(conn, index_name: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM pg_indexes " + "WHERE schemaname = current_schema() AND indexname = :name" + ), + {"name": index_name}, + ).fetchone() + is not None + ) + + +def _constraint_exists(conn, table: str, constraint_name: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.table_constraints " + "WHERE table_schema = current_schema() " + "AND table_name = :tbl AND constraint_name = :name" + ), + {"tbl": table, "name": constraint_name}, + ).fetchone() + is not None + ) + + +def _drop_index_if_exists(index_name: str, table_name: str) -> None: + if _index_exists(op.get_bind(), index_name): + op.drop_index(index_name, table_name=table_name) + + +def _drop_column_if_exists(table_name: str, column_name: str) -> None: + if _column_exists(op.get_bind(), table_name, column_name): + op.drop_column(table_name, column_name) + + +def _build_set_table_ddl( + *, documents_has_zero_ver: bool, user_has_zero_ver: bool +) -> str: + doc_cols = DOCUMENT_COLS + (['"_0_version"'] if documents_has_zero_ver else []) + user_cols = USER_COLS + (['"_0_version"'] if user_has_zero_ver else []) + + return ( + f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + f"notifications, " + f"documents ({_cols(doc_cols)}), " + f"folders, " + f"search_source_connectors, " + f"new_chat_messages WHERE (source = 'web'), " + f"chat_comments, " + f"chat_session_state, " + f'"user" ({_cols(user_cols)})' + ) + + +def _create_enum(name: str, values: tuple[str, ...]) -> postgresql.ENUM: + enum = postgresql.ENUM(*values, name=name) + enum.create(op.get_bind(), checkfirst=True) + return postgresql.ENUM(*values, name=name, create_type=False) + + +def upgrade() -> None: + conn = op.get_bind() + gateway_platform_enum = _create_enum( + "gateway_platform", ("telegram", "whatsapp", "signal") + ) + gateway_account_mode_enum = _create_enum( + "gateway_account_mode", ("cloud_shared", "self_host_byo") + ) + gateway_health_status_enum = _create_enum( + "gateway_health_status", ("unknown", "ok", "failing") + ) + gateway_binding_state_enum = _create_enum( + "gateway_binding_state", ("pending", "bound", "revoked", "suspended") + ) + gateway_peer_kind_enum = _create_enum( + "gateway_peer_kind", ("direct", "group", "channel", "unknown") + ) + gateway_session_scope_enum = _create_enum( + "gateway_session_scope", + ("per_binding", "per_user_search_space", "ephemeral"), + ) + gateway_dm_policy_enum = _create_enum("gateway_dm_policy", ("enabled", "disabled")) + gateway_group_policy_enum = _create_enum( + "gateway_group_policy", ("disabled", "allowlist", "mention_required") + ) + gateway_event_kind_enum = _create_enum( + "gateway_event_kind", ("message", "edited_message", "callback_query", "other") + ) + gateway_event_status_enum = _create_enum( + "gateway_event_status", + ("received", "processing", "processed", "ignored", "failed"), + ) + + if not _table_exists(conn, "gateway_platform_accounts"): + op.create_table( + "gateway_platform_accounts", + sa.Column("id", sa.BigInteger(), primary_key=True), + sa.Column("platform", gateway_platform_enum, nullable=False), + sa.Column("mode", gateway_account_mode_enum, nullable=False), + sa.Column("owner_user_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("owner_search_space_id", sa.Integer(), nullable=True), + sa.Column("is_system_account", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("encrypted_credentials", sa.Text(), nullable=True), + sa.Column( + "account_metadata", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default=sa.text("'{}'::jsonb"), + ), + sa.Column( + "cursor_state", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default=sa.text("'{}'::jsonb"), + ), + sa.Column( + "health_status", + gateway_health_status_enum, + nullable=False, + server_default="unknown", + ), + sa.Column("last_health_check_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("suspended_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("suspended_reason", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.CheckConstraint( + "(is_system_account = true AND owner_user_id IS NULL) OR " + "(is_system_account = false AND owner_user_id IS NOT NULL)", + name="ck_gateway_accounts_owner_shape", + ), + sa.ForeignKeyConstraint(["owner_user_id"], ["user.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["owner_search_space_id"], ["searchspaces.id"], ondelete="CASCADE" + ), + ) + op.create_index( + "uq_gateway_accounts_owner_platform", + "gateway_platform_accounts", + ["owner_user_id", "platform"], + unique=True, + postgresql_where=sa.text("is_system_account = false"), + if_not_exists=True, + ) + op.create_index( + "uq_gateway_accounts_system_platform", + "gateway_platform_accounts", + ["platform"], + unique=True, + postgresql_where=sa.text("is_system_account = true"), + if_not_exists=True, + ) + + if not _table_exists(conn, "gateway_conversation_bindings"): + op.create_table( + "gateway_conversation_bindings", + sa.Column("id", sa.BigInteger(), primary_key=True), + sa.Column("account_id", sa.BigInteger(), nullable=False), + sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("search_space_id", sa.Integer(), nullable=False), + sa.Column( + "state", + gateway_binding_state_enum, + nullable=False, + server_default="pending", + ), + sa.Column("pairing_code", sa.Text(), nullable=True), + sa.Column("pairing_code_expires_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("external_peer_id", sa.Text(), nullable=True), + sa.Column( + "external_peer_kind", + gateway_peer_kind_enum, + nullable=False, + server_default="unknown", + ), + sa.Column("external_thread_id", sa.Text(), nullable=True), + sa.Column("external_display_name", sa.Text(), nullable=True), + sa.Column("external_username", sa.Text(), nullable=True), + sa.Column("external_pii_hashes", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column( + "external_metadata", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default=sa.text("'{}'::jsonb"), + ), + sa.Column("active_thread_id", sa.Integer(), nullable=True), + sa.Column( + "session_scope", + gateway_session_scope_enum, + nullable=False, + server_default="per_binding", + ), + sa.Column( + "dm_policy", + gateway_dm_policy_enum, + nullable=False, + server_default="enabled", + ), + sa.Column( + "group_policy", + gateway_group_policy_enum, + nullable=False, + server_default="disabled", + ), + sa.Column("revoked_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("suspended_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("suspended_reason", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.ForeignKeyConstraint( + ["account_id"], ["gateway_platform_accounts.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["search_space_id"], ["searchspaces.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["active_thread_id"], ["new_chat_threads.id"], ondelete="SET NULL" + ), + ) + op.create_index( + "uq_gateway_bindings_account_peer_active", + "gateway_conversation_bindings", + ["account_id", "external_peer_id"], + unique=True, + postgresql_where=sa.text( + "state IN ('bound', 'suspended') AND external_peer_id IS NOT NULL" + ), + if_not_exists=True, + ) + op.create_index( + "uq_gateway_bindings_pairing_code_pending", + "gateway_conversation_bindings", + ["pairing_code"], + unique=True, + postgresql_where=sa.text("state = 'pending'"), + if_not_exists=True, + ) + op.create_index( + "ix_gateway_bindings_user_state", + "gateway_conversation_bindings", + ["user_id", "state"], + if_not_exists=True, + ) + op.create_index( + "ix_gateway_bindings_search_space_state", + "gateway_conversation_bindings", + ["search_space_id", "state"], + if_not_exists=True, + ) + + if not _table_exists(conn, "gateway_inbound_events"): + op.create_table( + "gateway_inbound_events", + sa.Column("id", sa.BigInteger(), primary_key=True), + sa.Column("account_id", sa.BigInteger(), nullable=False), + sa.Column("binding_id", sa.BigInteger(), nullable=True), + sa.Column("platform", gateway_platform_enum, nullable=False), + sa.Column("event_dedupe_key", sa.Text(), nullable=False), + sa.Column("external_event_id", sa.Text(), nullable=True), + sa.Column("external_message_id", sa.Text(), nullable=True), + sa.Column("event_kind", gateway_event_kind_enum, nullable=False), + sa.Column( + "raw_payload", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + ), + sa.Column( + "processing_metadata", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default=sa.text("'{}'::jsonb"), + ), + sa.Column( + "status", + gateway_event_status_enum, + nullable=False, + server_default="received", + ), + sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column( + "received_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.Column("processed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.ForeignKeyConstraint( + ["account_id"], ["gateway_platform_accounts.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["binding_id"], ["gateway_conversation_bindings.id"], ondelete="SET NULL" + ), + sa.UniqueConstraint( + "account_id", + "event_dedupe_key", + name="uq_gateway_inbound_account_dedupe_key", + ), + ) + op.create_index( + "ix_gateway_inbound_status_received_at", + "gateway_inbound_events", + ["status", "received_at"], + if_not_exists=True, + ) + op.create_index( + "ix_gateway_inbound_binding_received_at", + "gateway_inbound_events", + ["binding_id", "received_at"], + if_not_exists=True, + ) + + if not _column_exists(conn, "new_chat_threads", "source"): + op.add_column( + "new_chat_threads", + sa.Column("source", sa.Text(), nullable=False, server_default="web"), + ) + op.alter_column("new_chat_threads", "source", type_=sa.Text()) + if not _column_exists(conn, "new_chat_threads", "binding_id"): + op.add_column( + "new_chat_threads", + sa.Column("binding_id", sa.BigInteger(), nullable=True), + ) + if not _constraint_exists( + conn, "new_chat_threads", "fk_new_chat_threads_gateway_binding_id" + ): + op.create_foreign_key( + "fk_new_chat_threads_gateway_binding_id", + "new_chat_threads", + "gateway_conversation_bindings", + ["binding_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_index("ix_new_chat_threads_source", "new_chat_threads", ["source"], if_not_exists=True) + op.create_index( + "ix_new_chat_threads_binding_id", + "new_chat_threads", + ["binding_id"], + if_not_exists=True, + ) + + if not _column_exists(conn, "new_chat_messages", "source"): + op.add_column( + "new_chat_messages", + sa.Column("source", sa.Text(), nullable=False, server_default="web"), + ) + op.alter_column("new_chat_messages", "source", type_=sa.Text()) + if not _column_exists(conn, "new_chat_messages", "platform_metadata"): + op.add_column( + "new_chat_messages", + sa.Column("platform_metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + op.create_index( + "ix_new_chat_messages_source", + "new_chat_messages", + ["source"], + if_not_exists=True, + ) + op.create_index( + "uq_new_chat_messages_inbound_platform", + "new_chat_messages", + [ + "thread_id", + sa.text("(platform_metadata->>'platform')"), + sa.text("(platform_metadata->>'external_message_id')"), + ], + unique=True, + postgresql_where=sa.text( + "platform_metadata IS NOT NULL " + "AND platform_metadata->>'direction' = 'inbound'" + ), + if_not_exists=True, + ) + op.execute("ALTER TABLE new_chat_messages REPLICA IDENTITY FULL") + + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if exists: + documents_has_zero_ver = _has_zero_version(conn, "documents") + user_has_zero_ver = _has_zero_version(conn, "user") + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-144-gateway'") + ) + conn.execute( + sa.text( + _build_set_table_ddl( + documents_has_zero_ver=documents_has_zero_ver, + user_has_zero_ver=user_has_zero_ver, + ) + ) + ) + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-144-gateway'") + ) + + +def downgrade() -> None: + conn = op.get_bind() + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if exists: + documents_has_zero_ver = _has_zero_version(conn, "documents") + user_has_zero_ver = _has_zero_version(conn, "user") + # Restore the publication shape from migration 143. + doc_cols = DOCUMENT_COLS + (['"_0_version"'] if documents_has_zero_ver else []) + user_cols = USER_COLS + (['"_0_version"'] if user_has_zero_ver else []) + ddl = ( + f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + f"notifications, " + f"documents ({_cols(doc_cols)}), " + f"folders, " + f"search_source_connectors, " + f"new_chat_messages, " + f"chat_comments, " + f"chat_session_state, " + f'"user" ({_cols(user_cols)})' + ) + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-144-downgrade'") + ) + conn.execute(sa.text(ddl)) + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-144-downgrade'") + ) + + if _column_exists(conn, "new_chat_messages", "source"): + op.execute("ALTER TABLE new_chat_messages REPLICA IDENTITY DEFAULT") + _drop_index_if_exists("uq_new_chat_messages_inbound_platform", "new_chat_messages") + _drop_index_if_exists("ix_new_chat_messages_source", "new_chat_messages") + _drop_column_if_exists("new_chat_messages", "platform_metadata") + _drop_column_if_exists("new_chat_messages", "source") + + _drop_index_if_exists("ix_new_chat_threads_binding_id", "new_chat_threads") + _drop_index_if_exists("ix_new_chat_threads_source", "new_chat_threads") + if _constraint_exists( + conn, "new_chat_threads", "fk_new_chat_threads_gateway_binding_id" + ): + op.drop_constraint( + "fk_new_chat_threads_gateway_binding_id", + "new_chat_threads", + type_="foreignkey", + ) + _drop_column_if_exists("new_chat_threads", "binding_id") + _drop_column_if_exists("new_chat_threads", "source") + + _drop_index_if_exists( + "ix_gateway_inbound_binding_received_at", "gateway_inbound_events" + ) + _drop_index_if_exists("ix_gateway_inbound_status_received_at", "gateway_inbound_events") + if _table_exists(conn, "gateway_inbound_events"): + op.drop_table("gateway_inbound_events") + + _drop_index_if_exists( + "ix_gateway_bindings_search_space_state", + "gateway_conversation_bindings", + ) + _drop_index_if_exists( + "ix_gateway_bindings_user_state", "gateway_conversation_bindings" + ) + _drop_index_if_exists( + "uq_gateway_bindings_pairing_code_pending", + "gateway_conversation_bindings", + ) + _drop_index_if_exists( + "uq_gateway_bindings_account_peer_active", + "gateway_conversation_bindings", + ) + if _table_exists(conn, "gateway_conversation_bindings"): + op.drop_table("gateway_conversation_bindings") + + _drop_index_if_exists("uq_gateway_accounts_system_platform", "gateway_platform_accounts") + _drop_index_if_exists("uq_gateway_accounts_owner_platform", "gateway_platform_accounts") + if _table_exists(conn, "gateway_platform_accounts"): + op.drop_table("gateway_platform_accounts") + + for enum_name in ( + "gateway_event_status", + "gateway_event_kind", + "gateway_group_policy", + "gateway_dm_policy", + "gateway_session_scope", + "gateway_peer_kind", + "gateway_binding_state", + "gateway_health_status", + "gateway_account_mode", + "gateway_platform", + ): + postgresql.ENUM(name=enum_name).drop(conn, checkfirst=True) diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 9fc27fb1f..82b641ca6 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -14,6 +14,7 @@ from sqlalchemy import ( TIMESTAMP, BigInteger, Boolean, + CheckConstraint, Column, Enum as SQLAlchemyEnum, ForeignKey, @@ -573,6 +574,73 @@ class ChatVisibility(StrEnum): # PUBLIC = "PUBLIC" # Reserved for future implementation +class GatewayPlatform(StrEnum): + TELEGRAM = "telegram" + WHATSAPP = "whatsapp" + SIGNAL = "signal" + + +class GatewayAccountMode(StrEnum): + CLOUD_SHARED = "cloud_shared" + SELF_HOST_BYO = "self_host_byo" + + +class GatewayHealthStatus(StrEnum): + UNKNOWN = "unknown" + OK = "ok" + FAILING = "failing" + + +class GatewayBindingState(StrEnum): + PENDING = "pending" + BOUND = "bound" + REVOKED = "revoked" + SUSPENDED = "suspended" + + +class GatewayPeerKind(StrEnum): + DIRECT = "direct" + GROUP = "group" + CHANNEL = "channel" + UNKNOWN = "unknown" + + +class GatewaySessionScope(StrEnum): + PER_BINDING = "per_binding" + PER_USER_SEARCH_SPACE = "per_user_search_space" + EPHEMERAL = "ephemeral" + + +class GatewayDmPolicy(StrEnum): + ENABLED = "enabled" + DISABLED = "disabled" + + +class GatewayGroupPolicy(StrEnum): + DISABLED = "disabled" + ALLOWLIST = "allowlist" + MENTION_REQUIRED = "mention_required" + + +class GatewayEventKind(StrEnum): + MESSAGE = "message" + EDITED_MESSAGE = "edited_message" + CALLBACK_QUERY = "callback_query" + OTHER = "other" + + +class GatewayEventStatus(StrEnum): + RECEIVED = "received" + PROCESSING = "processing" + PROCESSED = "processed" + IGNORED = "ignored" + FAILED = "failed" + + +def _enum_values(enum_cls): + return [item.value for item in enum_cls] + + class NewChatThread(BaseModel, TimestampMixin): """ Thread model for the new chat feature using assistant-ui. @@ -645,6 +713,16 @@ class NewChatThread(BaseModel, TimestampMixin): # agent_llm_id changes). Unindexed: all reads are by primary key. pinned_llm_config_id = Column(Integer, nullable=True) + # Gateway-originated threads are persisted for the agent, but the UI Zero + # publication only exposes ``source='web'`` rows. + source = Column(Text, nullable=False, default="web", server_default="web") + binding_id = Column( + BigInteger, + ForeignKey("gateway_conversation_bindings.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + # Relationships search_space = relationship("SearchSpace", back_populates="new_chat_threads") created_by = relationship("User", back_populates="new_chat_threads") @@ -665,6 +743,11 @@ class NewChatThread(BaseModel, TimestampMixin): back_populates="thread", cascade="all, delete-orphan", ) + gateway_binding = relationship( + "GatewayConversationBinding", + foreign_keys=[binding_id], + back_populates="threads", + ) class NewChatMessage(BaseModel, TimestampMixin): @@ -718,6 +801,11 @@ class NewChatMessage(BaseModel, TimestampMixin): # a message back to the LangGraph checkpoint that produced its turn. turn_id = Column(String(64), nullable=True, index=True) + # Mirrors the parent thread source for publication-level filtering. + # This denormalization avoids join-dependent logical replication rules. + source = Column(Text, nullable=False, default="web", server_default="web") + platform_metadata = Column(JSONB, nullable=True) + # Relationships thread = relationship("NewChatThread", back_populates="messages") author = relationship("User") @@ -734,6 +822,298 @@ class NewChatMessage(BaseModel, TimestampMixin): ) +class GatewayPlatformAccount(Base, TimestampMixin): + __tablename__ = "gateway_platform_accounts" + __allow_unmapped__ = True + + id = Column(BigInteger, primary_key=True, index=True) + platform = Column( + SQLAlchemyEnum( + GatewayPlatform, + name="gateway_platform", + values_callable=_enum_values, + ), + nullable=False, + ) + mode = Column( + SQLAlchemyEnum( + GatewayAccountMode, + name="gateway_account_mode", + values_callable=_enum_values, + ), + nullable=False, + ) + owner_user_id = Column( + UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=True + ) + owner_search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=True + ) + is_system_account = Column(Boolean, nullable=False, default=False, server_default="false") + encrypted_credentials = Column(Text, nullable=True) + account_metadata = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")) + cursor_state = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")) + health_status = Column( + SQLAlchemyEnum( + GatewayHealthStatus, + name="gateway_health_status", + values_callable=_enum_values, + ), + nullable=False, + default=GatewayHealthStatus.UNKNOWN, + server_default=GatewayHealthStatus.UNKNOWN.value, + ) + last_health_check_at = Column(TIMESTAMP(timezone=True), nullable=True) + suspended_at = Column(TIMESTAMP(timezone=True), nullable=True) + suspended_reason = Column(Text, nullable=True) + updated_at = Column( + TIMESTAMP(timezone=True), + nullable=False, + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + server_default=text("(now() AT TIME ZONE 'utc')"), + ) + + owner = relationship("User", foreign_keys=[owner_user_id]) + owner_search_space = relationship("SearchSpace", foreign_keys=[owner_search_space_id]) + bindings = relationship( + "GatewayConversationBinding", + back_populates="account", + cascade="all, delete-orphan", + ) + inbound_events = relationship( + "GatewayInboundEvent", + back_populates="account", + cascade="all, delete-orphan", + ) + + __table_args__ = ( + CheckConstraint( + "(is_system_account = true AND owner_user_id IS NULL) OR " + "(is_system_account = false AND owner_user_id IS NOT NULL)", + name="ck_gateway_accounts_owner_shape", + ), + Index( + "uq_gateway_accounts_owner_platform", + "owner_user_id", + "platform", + unique=True, + postgresql_where=text("is_system_account = false"), + ), + Index( + "uq_gateway_accounts_system_platform", + "platform", + unique=True, + postgresql_where=text("is_system_account = true"), + ), + ) + + +class GatewayConversationBinding(Base, TimestampMixin): + __tablename__ = "gateway_conversation_bindings" + __allow_unmapped__ = True + + id = Column(BigInteger, primary_key=True, index=True) + account_id = Column( + BigInteger, + ForeignKey("gateway_platform_accounts.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + user_id = Column( + UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False + ) + search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False + ) + state = Column( + SQLAlchemyEnum( + GatewayBindingState, + name="gateway_binding_state", + values_callable=_enum_values, + ), + nullable=False, + default=GatewayBindingState.PENDING, + server_default=GatewayBindingState.PENDING.value, + ) + pairing_code = Column(Text, nullable=True) + pairing_code_expires_at = Column(TIMESTAMP(timezone=True), nullable=True) + external_peer_id = Column(Text, nullable=True) + external_peer_kind = Column( + SQLAlchemyEnum( + GatewayPeerKind, + name="gateway_peer_kind", + values_callable=_enum_values, + ), + nullable=False, + default=GatewayPeerKind.UNKNOWN, + server_default=GatewayPeerKind.UNKNOWN.value, + ) + external_thread_id = Column(Text, nullable=True) + external_display_name = Column(Text, nullable=True) + external_username = Column(Text, nullable=True) + external_pii_hashes = Column(JSONB, nullable=True) + external_metadata = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")) + active_thread_id = Column( + Integer, + ForeignKey("new_chat_threads.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + session_scope = Column( + SQLAlchemyEnum( + GatewaySessionScope, + name="gateway_session_scope", + values_callable=_enum_values, + ), + nullable=False, + default=GatewaySessionScope.PER_BINDING, + server_default=GatewaySessionScope.PER_BINDING.value, + ) + dm_policy = Column( + SQLAlchemyEnum( + GatewayDmPolicy, + name="gateway_dm_policy", + values_callable=_enum_values, + ), + nullable=False, + default=GatewayDmPolicy.ENABLED, + server_default=GatewayDmPolicy.ENABLED.value, + ) + group_policy = Column( + SQLAlchemyEnum( + GatewayGroupPolicy, + name="gateway_group_policy", + values_callable=_enum_values, + ), + nullable=False, + default=GatewayGroupPolicy.DISABLED, + server_default=GatewayGroupPolicy.DISABLED.value, + ) + revoked_at = Column(TIMESTAMP(timezone=True), nullable=True) + suspended_at = Column(TIMESTAMP(timezone=True), nullable=True) + suspended_reason = Column(Text, nullable=True) + updated_at = Column( + TIMESTAMP(timezone=True), + nullable=False, + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + server_default=text("(now() AT TIME ZONE 'utc')"), + ) + + account = relationship("GatewayPlatformAccount", back_populates="bindings") + user = relationship("User", foreign_keys=[user_id]) + search_space = relationship("SearchSpace", foreign_keys=[search_space_id]) + active_thread = relationship("NewChatThread", foreign_keys=[active_thread_id]) + threads = relationship( + "NewChatThread", + back_populates="gateway_binding", + foreign_keys="NewChatThread.binding_id", + ) + inbound_events = relationship( + "GatewayInboundEvent", + back_populates="binding", + foreign_keys="GatewayInboundEvent.binding_id", + ) + + __table_args__ = ( + Index( + "uq_gateway_bindings_account_peer_active", + "account_id", + "external_peer_id", + unique=True, + postgresql_where=text( + "state IN ('bound', 'suspended') AND external_peer_id IS NOT NULL" + ), + ), + Index( + "uq_gateway_bindings_pairing_code_pending", + "pairing_code", + unique=True, + postgresql_where=text("state = 'pending'"), + ), + Index("ix_gateway_bindings_user_state", "user_id", "state"), + Index("ix_gateway_bindings_search_space_state", "search_space_id", "state"), + ) + + +class GatewayInboundEvent(Base, TimestampMixin): + __tablename__ = "gateway_inbound_events" + __allow_unmapped__ = True + + id = Column(BigInteger, primary_key=True, index=True) + account_id = Column( + BigInteger, + ForeignKey("gateway_platform_accounts.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + binding_id = Column( + BigInteger, + ForeignKey("gateway_conversation_bindings.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + platform = Column( + SQLAlchemyEnum( + GatewayPlatform, + name="gateway_platform", + values_callable=_enum_values, + ), + nullable=False, + ) + event_dedupe_key = Column(Text, nullable=False) + external_event_id = Column(Text, nullable=True) + external_message_id = Column(Text, nullable=True) + event_kind = Column( + SQLAlchemyEnum( + GatewayEventKind, + name="gateway_event_kind", + values_callable=_enum_values, + ), + nullable=False, + ) + raw_payload = Column(JSONB, nullable=True) + processing_metadata = Column( + JSONB, + nullable=False, + default=dict, + server_default=text("'{}'::jsonb"), + ) + status = Column( + SQLAlchemyEnum( + GatewayEventStatus, + name="gateway_event_status", + values_callable=_enum_values, + ), + nullable=False, + default=GatewayEventStatus.RECEIVED, + server_default=GatewayEventStatus.RECEIVED.value, + ) + attempt_count = Column(Integer, nullable=False, default=0, server_default="0") + last_error = Column(Text, nullable=True) + received_at = Column( + TIMESTAMP(timezone=True), + nullable=False, + default=lambda: datetime.now(UTC), + server_default=text("(now() AT TIME ZONE 'utc')"), + ) + processed_at = Column(TIMESTAMP(timezone=True), nullable=True) + + account = relationship("GatewayPlatformAccount", back_populates="inbound_events") + binding = relationship("GatewayConversationBinding", back_populates="inbound_events") + + __table_args__ = ( + UniqueConstraint( + "account_id", + "event_dedupe_key", + name="uq_gateway_inbound_account_dedupe_key", + ), + Index("ix_gateway_inbound_status_received_at", "status", "received_at"), + Index("ix_gateway_inbound_binding_received_at", "binding_id", "received_at"), + ) + + class TokenUsage(BaseModel, TimestampMixin): """ Tracks LLM token consumption per assistant turn. diff --git a/surfsense_web/zero/schema/chat.ts b/surfsense_web/zero/schema/chat.ts index fb3d7651e..8da41ee45 100644 --- a/surfsense_web/zero/schema/chat.ts +++ b/surfsense_web/zero/schema/chat.ts @@ -8,6 +8,8 @@ export const newChatMessageTable = table("new_chat_messages") threadId: number().from("thread_id"), authorId: string().optional().from("author_id"), createdAt: number().from("created_at"), + source: string(), + platformMetadata: json().optional().from("platform_metadata"), // Per-turn correlation id sourced from ``configurable.turn_id`` // at streaming time. Required by the inline Revert button's // (chat_turn_id, tool_name, position) fallback in tool-fallback.tsx From ae3ce914653ec2372f83e4d9ceb63bf148930fdc Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:37:26 +0530 Subject: [PATCH 03/63] feat(gateway): add configuration and metrics hooks --- surfsense_backend/.env.example | 7 + surfsense_backend/app/config/__init__.py | 6 + .../app/observability/metrics.py | 153 ++++++++++++++++++ 3 files changed, 166 insertions(+) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index b05369412..6fef9b20e 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -15,6 +15,13 @@ REDIS_APP_URL=redis://localhost:6379/0 # Optional: TTL in seconds for connector indexing lock key # CONNECTOR_INDEXING_LOCK_TTL_SECONDS=28800 +# Telegram Gateway +# TELEGRAM_WEBHOOK_SECRET must be 1-256 chars and contain only A-Z, a-z, 0-9, _ or - +TELEGRAM_SHARED_BOT_TOKEN= +TELEGRAM_SHARED_BOT_USERNAME= +TELEGRAM_WEBHOOK_SECRET= +GATEWAY_BASE_URL=http://localhost:8000 + # Platform Web Search (SearXNG) # Set this to enable built-in web search. Docker Compose sets it automatically. # Only uncomment if running the backend outside Docker (e.g. uvicorn on host). diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 5643c048b..a7739d6c4 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -541,6 +541,12 @@ class Config: # Backend URL to override the http to https in the OAuth redirect URI BACKEND_URL = os.getenv("BACKEND_URL") + # Messaging gateway (Telegram v1) + TELEGRAM_SHARED_BOT_TOKEN = os.getenv("TELEGRAM_SHARED_BOT_TOKEN") + TELEGRAM_SHARED_BOT_USERNAME = os.getenv("TELEGRAM_SHARED_BOT_USERNAME") + TELEGRAM_WEBHOOK_SECRET = os.getenv("TELEGRAM_WEBHOOK_SECRET") + GATEWAY_BASE_URL = os.getenv("GATEWAY_BASE_URL", BACKEND_URL) + # Stripe checkout for pay-as-you-go page packs STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY") STRIPE_WEBHOOK_SECRET = os.getenv("STRIPE_WEBHOOK_SECRET") diff --git a/surfsense_backend/app/observability/metrics.py b/surfsense_backend/app/observability/metrics.py index 798a6e2f7..8098ac307 100644 --- a/surfsense_backend/app/observability/metrics.py +++ b/surfsense_backend/app/observability/metrics.py @@ -314,6 +314,103 @@ def _celery_queue_latency(): ) +@lru_cache(maxsize=1) +def _gateway_redis_fallback(): + return _get_meter().create_counter( + "surfsense.gateway.redis.fallback", + description="Count of gateway Redis fallback uses.", + ) + + +@lru_cache(maxsize=1) +def _gateway_thread_lock_contention(): + return _get_meter().create_counter( + "surfsense.gateway.thread_lock.contention", + description="Count of gateway per-thread lock contention events.", + ) + + +@lru_cache(maxsize=1) +def _gateway_inbox_writes(): + return _get_meter().create_counter( + "surfsense.gateway.inbox.writes", + description="Count of gateway inbound event inbox writes.", + ) + + +@lru_cache(maxsize=1) +def _gateway_inbox_processed(): + return _get_meter().create_counter( + "surfsense.gateway.inbox.processed", + description="Count of gateway inbound event processing outcomes.", + ) + + +@lru_cache(maxsize=1) +def _gateway_inbound_reconciled(): + return _get_meter().create_counter( + "surfsense.gateway.inbound.reconciled", + description="Count of gateway inbox events re-enqueued by reconciliation.", + ) + + +@lru_cache(maxsize=1) +def _gateway_outbound(): + return _get_meter().create_counter( + "surfsense.gateway.outbound", + description="Count of gateway outbound platform operations.", + ) + + +@lru_cache(maxsize=1) +def _gateway_turn_latency(): + return _get_meter().create_histogram( + "surfsense.gateway.turn.latency", + unit="ms", + description="Latency of gateway-routed agent turns.", + ) + + +@lru_cache(maxsize=1) +def _gateway_rate_limit_hits(): + return _get_meter().create_counter( + "surfsense.gateway.rate_limit.hits", + description="Count of gateway outbound rate limit waits.", + ) + + +@lru_cache(maxsize=1) +def _gateway_health_check_failures(): + return _get_meter().create_counter( + "surfsense.gateway.health_check.failures", + description="Count of gateway account health-check failures.", + ) + + +@lru_cache(maxsize=1) +def _gateway_auth_invariant_failures(): + return _get_meter().create_counter( + "surfsense.gateway.auth_invariant.failures", + description="Count of gateway authorization invariant failures.", + ) + + +@lru_cache(maxsize=1) +def _gateway_hitl_aborted(): + return _get_meter().create_counter( + "surfsense.gateway.hitl.aborted", + description="Count of gateway turns aborted because HITL is unsupported.", + ) + + +@lru_cache(maxsize=1) +def _gateway_active_bindings(): + return _get_meter().create_up_down_counter( + "surfsense.gateway.active_bindings", + description="Current change in active gateway bindings.", + ) + + def record_model_call_duration( duration_ms: float, *, model: str | None, provider: str | None ) -> None: @@ -569,6 +666,62 @@ def record_celery_queue_latency( ) +def record_gateway_redis_fallback() -> None: + _add(_gateway_redis_fallback(), 1, {}) + + +def record_gateway_thread_lock_contention() -> None: + _add(_gateway_thread_lock_contention(), 1, {}) + + +def record_gateway_inbox_write(*, platform: str, dedup_skipped: bool) -> None: + _add( + _gateway_inbox_writes(), + 1, + {"platform": platform, "dedup.skipped": bool(dedup_skipped)}, + ) + + +def record_gateway_inbox_processed(*, platform: str, status: str) -> None: + _add(_gateway_inbox_processed(), 1, {"platform": platform, "status": status}) + + +def record_gateway_inbound_reconciled(*, reason: str) -> None: + _add(_gateway_inbound_reconciled(), 1, {"reason": reason}) + + +def record_gateway_outbound(*, platform: str, kind: str, status: str) -> None: + _add( + _gateway_outbound(), + 1, + {"platform": platform, "kind": kind, "status": status}, + ) + + +def record_gateway_turn_latency(duration_ms: float, *, platform: str) -> None: + _record(_gateway_turn_latency(), duration_ms, {"platform": platform}) + + +def record_gateway_rate_limit_hit(*, bucket: str) -> None: + _add(_gateway_rate_limit_hits(), 1, {"bucket": bucket}) + + +def record_gateway_health_check_failure(*, platform: str) -> None: + _add(_gateway_health_check_failures(), 1, {"platform": platform}) + + +def record_gateway_auth_invariant_failure(*, cause: str) -> None: + _add(_gateway_auth_invariant_failures(), 1, {"cause": cause}) + + +def record_gateway_hitl_aborted(*, platform: str) -> None: + _add(_gateway_hitl_aborted(), 1, {"platform": platform}) + + +def record_gateway_active_bindings_delta(delta: int, *, platform: str) -> None: + _add(_gateway_active_bindings(), delta, {"platform": platform}) + + def _runtime_snapshot_value(key: str, transform: Any = None) -> list[Any]: from opentelemetry.metrics import Observation From c9b7d7b5722bb83530425ccd8c7abdafe07e83a6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:37:54 +0530 Subject: [PATCH 04/63] feat(gateway): add gateway domain primitives --- surfsense_backend/app/gateway/__init__.py | 2 + surfsense_backend/app/gateway/accounts.py | 54 ++++++++++++++ .../app/gateway/auth_invariant.py | 55 +++++++++++++++ .../app/gateway/base/__init__.py | 2 + surfsense_backend/app/gateway/base/adapter.py | 70 +++++++++++++++++++ .../app/gateway/base/identity.py | 19 +++++ .../app/gateway/base/translator.py | 28 ++++++++ surfsense_backend/app/gateway/bindings.py | 62 ++++++++++++++++ surfsense_backend/app/gateway/hitl_filter.py | 35 ++++++++++ surfsense_backend/app/gateway/inbox.py | 44 ++++++++++++ surfsense_backend/app/gateway/pairing.py | 54 ++++++++++++++ .../tests/unit/gateway/test_hitl_filter.py | 15 ++++ .../tests/unit/gateway/test_pairing.py | 41 +++++++++++ 13 files changed, 481 insertions(+) create mode 100644 surfsense_backend/app/gateway/__init__.py create mode 100644 surfsense_backend/app/gateway/accounts.py create mode 100644 surfsense_backend/app/gateway/auth_invariant.py create mode 100644 surfsense_backend/app/gateway/base/__init__.py create mode 100644 surfsense_backend/app/gateway/base/adapter.py create mode 100644 surfsense_backend/app/gateway/base/identity.py create mode 100644 surfsense_backend/app/gateway/base/translator.py create mode 100644 surfsense_backend/app/gateway/bindings.py create mode 100644 surfsense_backend/app/gateway/hitl_filter.py create mode 100644 surfsense_backend/app/gateway/inbox.py create mode 100644 surfsense_backend/app/gateway/pairing.py create mode 100644 surfsense_backend/tests/unit/gateway/test_hitl_filter.py create mode 100644 surfsense_backend/tests/unit/gateway/test_pairing.py diff --git a/surfsense_backend/app/gateway/__init__.py b/surfsense_backend/app/gateway/__init__.py new file mode 100644 index 000000000..5cf91505b --- /dev/null +++ b/surfsense_backend/app/gateway/__init__.py @@ -0,0 +1,2 @@ +"""Messaging gateway infrastructure for external chat channels.""" + diff --git a/surfsense_backend/app/gateway/accounts.py b/surfsense_backend/app/gateway/accounts.py new file mode 100644 index 000000000..727d616c1 --- /dev/null +++ b/surfsense_backend/app/gateway/accounts.py @@ -0,0 +1,54 @@ +"""Gateway account helpers.""" + +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + GatewayAccountMode, + GatewayHealthStatus, + GatewayPlatform, + GatewayPlatformAccount, +) +from app.utils.oauth_security import TokenEncryption + + +def account_token(account: GatewayPlatformAccount) -> str | None: + if account.is_system_account and account.platform == GatewayPlatform.TELEGRAM: + return config.TELEGRAM_SHARED_BOT_TOKEN + if not account.encrypted_credentials: + return None + return TokenEncryption(config.SECRET_KEY or "").decrypt_token( + account.encrypted_credentials + ) + + +async def get_or_create_system_telegram_account( + session: AsyncSession, +) -> GatewayPlatformAccount: + result = await session.execute( + select(GatewayPlatformAccount).where( + GatewayPlatformAccount.platform == GatewayPlatform.TELEGRAM, + GatewayPlatformAccount.is_system_account.is_(True), + ) + ) + account = result.scalars().first() + if account is not None: + return account + account = GatewayPlatformAccount( + platform=GatewayPlatform.TELEGRAM, + mode=GatewayAccountMode.CLOUD_SHARED, + is_system_account=True, + account_metadata={ + "bot_username": config.TELEGRAM_SHARED_BOT_USERNAME, + "webhook_secret": config.TELEGRAM_WEBHOOK_SECRET, + }, + cursor_state={}, + health_status=GatewayHealthStatus.UNKNOWN, + ) + session.add(account) + await session.flush() + return account + diff --git a/surfsense_backend/app/gateway/auth_invariant.py b/surfsense_backend/app/gateway/auth_invariant.py new file mode 100644 index 000000000..414c69c5c --- /dev/null +++ b/surfsense_backend/app/gateway/auth_invariant.py @@ -0,0 +1,55 @@ +"""Authorization invariants for gateway-routed turns.""" + +from __future__ import annotations + +from fastapi import HTTPException +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import GatewayConversationBinding, Permission, User +from app.gateway.bindings import suspend_binding +from app.observability.metrics import record_gateway_auth_invariant_failure +from app.utils.rbac import check_permission, check_search_space_access + + +class GatewaySuspendedError(RuntimeError): + def __init__(self, reason: str) -> None: + self.reason = reason + super().__init__(reason) + + +async def _fail( + session: AsyncSession, + binding: GatewayConversationBinding, + reason: str, +) -> None: + suspend_binding(binding, reason) + record_gateway_auth_invariant_failure(cause=reason) + await session.flush() + raise GatewaySuspendedError(reason) + + +async def assert_authorization_invariant( + session: AsyncSession, + binding: GatewayConversationBinding, +) -> User: + if binding.state != "bound": + await _fail(session, binding, "binding_not_bound") + + user = await session.get(User, binding.user_id) + if user is None: + await _fail(session, binding, "owner_missing") + + try: + await check_search_space_access(session, user, binding.search_space_id) + await check_permission( + session, + user, + binding.search_space_id, + Permission.CHATS_CREATE.value, + "Gateway owner no longer has permission to chat in this search space", + ) + except HTTPException as exc: + await _fail(session, binding, f"rbac_{exc.status_code}") + + return user + diff --git a/surfsense_backend/app/gateway/base/__init__.py b/surfsense_backend/app/gateway/base/__init__.py new file mode 100644 index 000000000..962d068b6 --- /dev/null +++ b/surfsense_backend/app/gateway/base/__init__.py @@ -0,0 +1,2 @@ +"""Base gateway interfaces.""" + diff --git a/surfsense_backend/app/gateway/base/adapter.py b/surfsense_backend/app/gateway/base/adapter.py new file mode 100644 index 000000000..caf351c05 --- /dev/null +++ b/surfsense_backend/app/gateway/base/adapter.py @@ -0,0 +1,70 @@ +"""Platform adapter interfaces for messaging gateways.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class ParsedInboundEvent: + platform: str + event_kind: str + external_peer_id: str | None + external_peer_kind: str + external_message_id: str | None + external_user_id: str | None + text: str | None + raw_payload: dict[str, Any] + display_name: str | None = None + username: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class PlatformSendResult: + external_message_id: str + raw_response: dict[str, Any] = field(default_factory=dict) + + +class BasePlatformAdapter(ABC): + platform: str + + @abstractmethod + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + """Parse a provider webhook/update into the gateway's normalized shape.""" + + @abstractmethod + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + """Send a new platform message.""" + + @abstractmethod + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + """Edit an existing platform message.""" + + @abstractmethod + async def validate_credentials(self) -> dict[str, Any]: + """Validate configured credentials and return account metadata.""" + + async def fetch_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]: + """Yield provider updates for long-polling adapters.""" + if False: + yield {} # pragma: no cover + raise NotImplementedError("This adapter does not support long-polling") + diff --git a/surfsense_backend/app/gateway/base/identity.py b/surfsense_backend/app/gateway/base/identity.py new file mode 100644 index 000000000..608ae41c1 --- /dev/null +++ b/surfsense_backend/app/gateway/base/identity.py @@ -0,0 +1,19 @@ +"""Gateway identity helpers.""" + +from __future__ import annotations + +import hashlib + + +def normalize_external_peer_id(value: str | int | None) -> str | None: + if value is None: + return None + return str(value).strip() + + +def hash_external_id(value: str | int | None) -> str | None: + normalized = normalize_external_peer_id(value) + if not normalized: + return None + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + diff --git a/surfsense_backend/app/gateway/base/translator.py b/surfsense_backend/app/gateway/base/translator.py new file mode 100644 index 000000000..af72188e9 --- /dev/null +++ b/surfsense_backend/app/gateway/base/translator.py @@ -0,0 +1,28 @@ +"""Base stream translator for platform-specific outbound UX.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class GatewayStreamEvent: + """Small provider-neutral event shape consumed by translators. + + The existing chat stack emits Vercel/assistant-ui events. Gateway code + normalizes the subset it needs into this shape before handing it to the + platform translator. + """ + + type: str + data: dict[str, Any] = field(default_factory=dict) + + +class BaseStreamTranslator(ABC): + @abstractmethod + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + """Consume agent stream events and emit platform messages.""" + diff --git a/surfsense_backend/app/gateway/bindings.py b/surfsense_backend/app/gateway/bindings.py new file mode 100644 index 000000000..6f2b641f7 --- /dev/null +++ b/surfsense_backend/app/gateway/bindings.py @@ -0,0 +1,62 @@ +"""Gateway binding helpers.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import ( + ChatVisibility, + GatewayBindingState, + GatewayConversationBinding, + NewChatThread, +) + + +async def get_or_create_thread_for_binding( + session: AsyncSession, + binding: GatewayConversationBinding, +) -> NewChatThread: + if binding.active_thread_id is not None: + result = await session.execute( + select(NewChatThread).where(NewChatThread.id == binding.active_thread_id) + ) + thread = result.scalars().first() + if thread is not None and not thread.archived: + return thread + + thread = NewChatThread( + title="Telegram chat", + search_space_id=binding.search_space_id, + created_by_id=binding.user_id, + visibility=ChatVisibility.PRIVATE, + source="telegram", + binding_id=binding.id, + ) + session.add(thread) + await session.flush() + binding.active_thread_id = thread.id + return thread + + +def suspend_binding(binding: GatewayConversationBinding, reason: str) -> None: + now = datetime.now(UTC) + binding.state = GatewayBindingState.SUSPENDED + binding.suspended_at = now + binding.suspended_reason = reason + + +def revoke_binding(binding: GatewayConversationBinding) -> None: + now = datetime.now(UTC) + binding.state = GatewayBindingState.REVOKED + binding.revoked_at = now + binding.active_thread_id = None + + +def resume_binding(binding: GatewayConversationBinding) -> None: + binding.state = GatewayBindingState.BOUND + binding.suspended_at = None + binding.suspended_reason = None + diff --git a/surfsense_backend/app/gateway/hitl_filter.py b/surfsense_backend/app/gateway/hitl_filter.py new file mode 100644 index 000000000..e3acc6d42 --- /dev/null +++ b/surfsense_backend/app/gateway/hitl_filter.py @@ -0,0 +1,35 @@ +"""Filter approval-required tools from gateway agent invocations.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +DEFAULT_HITL_TOOL_NAMES = { + "delete_document", + "delete_folder", + "delete_note", + "delete_report", + "delete_connector", + "send_email", + "share_chat", +} + + +def _tool_name(tool: Any) -> str | None: + if isinstance(tool, str): + return tool + return getattr(tool, "name", None) or getattr(tool, "__name__", None) + + +def filter_hitl_tools( + toolkit: Iterable[Any] | None, + *, + blocked_names: set[str] | None = None, +) -> list[Any] | None: + """Return a toolkit with known approval-required tools removed.""" + if toolkit is None: + return None + blocked = blocked_names or DEFAULT_HITL_TOOL_NAMES + return [tool for tool in toolkit if (_tool_name(tool) or "") not in blocked] + diff --git a/surfsense_backend/app/gateway/inbox.py b/surfsense_backend/app/gateway/inbox.py new file mode 100644 index 000000000..c98ee5977 --- /dev/null +++ b/surfsense_backend/app/gateway/inbox.py @@ -0,0 +1,44 @@ +"""Durable gateway inbox helpers.""" + +from __future__ import annotations + +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import GatewayInboundEvent, GatewayPlatform + + +def telegram_event_dedupe_key(update_id: int | str) -> str: + return f"update:{update_id}" + + +async def persist_inbound_event( + session: AsyncSession, + *, + account_id: int, + platform: GatewayPlatform, + event_dedupe_key: str, + event_kind: str, + raw_payload: dict, + external_event_id: str | None = None, + external_message_id: str | None = None, +) -> int | None: + stmt = ( + insert(GatewayInboundEvent) + .values( + account_id=account_id, + platform=platform, + event_dedupe_key=event_dedupe_key, + external_event_id=external_event_id, + external_message_id=external_message_id, + event_kind=event_kind, + raw_payload=raw_payload, + ) + .on_conflict_do_nothing( + index_elements=["account_id", "event_dedupe_key"], + ) + .returning(GatewayInboundEvent.id) + ) + result = await session.execute(stmt) + return result.scalar_one_or_none() + diff --git a/surfsense_backend/app/gateway/pairing.py b/surfsense_backend/app/gateway/pairing.py new file mode 100644 index 000000000..55232022e --- /dev/null +++ b/surfsense_backend/app/gateway/pairing.py @@ -0,0 +1,54 @@ +"""Pairing code lifecycle for gateway bindings.""" + +from __future__ import annotations + +import secrets +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import GatewayBindingState, GatewayConversationBinding + +PAIRING_CODE_TTL = timedelta(minutes=10) + + +def generate_pairing_code() -> str: + return secrets.token_urlsafe(6) + + +def pairing_expires_at() -> datetime: + return datetime.now(UTC) + PAIRING_CODE_TTL + + +async def redeem_pairing_code( + session: AsyncSession, + *, + code: str, + external_peer_id: str, + external_peer_kind: str, + external_display_name: str | None, + external_username: str | None, + external_metadata: dict | None = None, +) -> GatewayConversationBinding | None: + result = await session.execute( + select(GatewayConversationBinding).where( + GatewayConversationBinding.pairing_code == code, + GatewayConversationBinding.state == GatewayBindingState.PENDING, + GatewayConversationBinding.pairing_code_expires_at > datetime.now(UTC), + ) + ) + binding = result.scalars().first() + if binding is None: + return None + + binding.state = GatewayBindingState.BOUND + binding.pairing_code = None + binding.pairing_code_expires_at = None + binding.external_peer_id = external_peer_id + binding.external_peer_kind = external_peer_kind + binding.external_display_name = external_display_name + binding.external_username = external_username + binding.external_metadata = external_metadata or {} + return binding + diff --git a/surfsense_backend/tests/unit/gateway/test_hitl_filter.py b/surfsense_backend/tests/unit/gateway/test_hitl_filter.py new file mode 100644 index 000000000..90f94b6ab --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_hitl_filter.py @@ -0,0 +1,15 @@ +from app.gateway.hitl_filter import filter_hitl_tools + + +class Tool: + def __init__(self, name: str) -> None: + self.name = name + + +def test_filter_hitl_tools_removes_known_approval_tools(): + tools = [Tool("delete_document"), Tool("search"), "send_email", "summarize"] + + filtered = filter_hitl_tools(tools) + + assert [getattr(tool, "name", tool) for tool in filtered] == ["search", "summarize"] + diff --git a/surfsense_backend/tests/unit/gateway/test_pairing.py b/surfsense_backend/tests/unit/gateway/test_pairing.py new file mode 100644 index 000000000..c50bd6b7c --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_pairing.py @@ -0,0 +1,41 @@ +from datetime import UTC, datetime, timedelta + +import pytest + +from app.db import GatewayBindingState +from app.gateway.pairing import generate_pairing_code, redeem_pairing_code + + +def test_generate_pairing_code_is_short_display_token(): + code = generate_pairing_code() + + assert len(code) >= 8 + assert "\n" not in code + + +@pytest.mark.asyncio +async def test_redeem_pairing_code_binds_pending_row(mocker): + binding = mocker.Mock() + binding.state = GatewayBindingState.PENDING + binding.pairing_code_expires_at = datetime.now(UTC) + timedelta(minutes=1) + scalars = mocker.Mock() + scalars.first.return_value = binding + result = mocker.Mock() + result.scalars.return_value = scalars + session = mocker.AsyncMock() + session.execute.return_value = result + + redeemed = await redeem_pairing_code( + session, + code="abc", + external_peer_id="telegram:123", + external_peer_kind="direct", + external_display_name="Anish", + external_username="anish", + ) + + assert redeemed is binding + assert binding.state == GatewayBindingState.BOUND + assert binding.external_peer_id == "telegram:123" + assert binding.pairing_code is None + From 59e64753484e4cd4929c88b2fa588c572f30f53f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:38:09 +0530 Subject: [PATCH 05/63] feat(gateway): add Telegram adapter and formatting --- .../app/gateway/telegram/__init__.py | 2 + .../app/gateway/telegram/adapter.py | 114 ++++++++++++++++++ .../app/gateway/telegram/client.py | 109 +++++++++++++++++ .../app/gateway/telegram/formatting.py | 55 +++++++++ .../tests/unit/gateway/test_formatting.py | 18 +++ 5 files changed, 298 insertions(+) create mode 100644 surfsense_backend/app/gateway/telegram/__init__.py create mode 100644 surfsense_backend/app/gateway/telegram/adapter.py create mode 100644 surfsense_backend/app/gateway/telegram/client.py create mode 100644 surfsense_backend/app/gateway/telegram/formatting.py create mode 100644 surfsense_backend/tests/unit/gateway/test_formatting.py diff --git a/surfsense_backend/app/gateway/telegram/__init__.py b/surfsense_backend/app/gateway/telegram/__init__.py new file mode 100644 index 000000000..45dc05414 --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/__init__.py @@ -0,0 +1,2 @@ +"""Telegram gateway adapter.""" + diff --git a/surfsense_backend/app/gateway/telegram/adapter.py b/surfsense_backend/app/gateway/telegram/adapter.py new file mode 100644 index 000000000..4f0001128 --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/adapter.py @@ -0,0 +1,114 @@ +"""Telegram platform adapter.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from app.gateway.base.adapter import ( + BasePlatformAdapter, + ParsedInboundEvent, + PlatformSendResult, +) +from app.gateway.telegram.client import TelegramClient + + +class TelegramAdapter(BasePlatformAdapter): + platform = "telegram" + + def __init__(self, token: str) -> None: + self.client = TelegramClient(token) + + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + event_kind = "other" + message = raw_payload.get("message") + if message is not None: + event_kind = "message" + else: + message = raw_payload.get("edited_message") + if message is not None: + event_kind = "edited_message" + + if message is None: + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_kind, + external_peer_id=None, + external_peer_kind="unknown", + external_message_id=None, + external_user_id=None, + text=None, + raw_payload=raw_payload, + ) + + chat = message.get("chat") or {} + sender = message.get("from") or {} + chat_type = str(chat.get("type") or "unknown") + peer_kind = { + "private": "direct", + "group": "group", + "supergroup": "group", + "channel": "channel", + }.get(chat_type, "unknown") + display_name = chat.get("title") or " ".join( + part + for part in (sender.get("first_name"), sender.get("last_name")) + if part + ) + + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_kind, + external_peer_id=str(chat["id"]) if chat.get("id") is not None else None, + external_peer_kind=peer_kind, + external_message_id=( + str(message["message_id"]) if message.get("message_id") is not None else None + ), + external_user_id=str(sender["id"]) if sender.get("id") is not None else None, + text=message.get("text") or message.get("caption"), + raw_payload=raw_payload, + display_name=display_name or None, + username=sender.get("username") or chat.get("username"), + metadata={"chat_type": chat_type, "update_id": raw_payload.get("update_id")}, + ) + + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + return await self.client.send_message( + chat_id=external_peer_id, + text=text, + parse_mode=parse_mode, + reply_to_message_id=reply_to_message_id, + ) + + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + return await self.client.edit_message( + chat_id=external_peer_id, + message_id=external_message_id, + text=text, + parse_mode=parse_mode, + ) + + async def validate_credentials(self) -> dict[str, Any]: + return await self.client.validate() + + async def leave_chat(self, *, external_peer_id: str) -> None: + await self.client.leave_chat(chat_id=external_peer_id) + + async def fetch_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]: + async for update in self.client.get_updates(offset=offset): + yield update + diff --git a/surfsense_backend/app/gateway/telegram/client.py b/surfsense_backend/app/gateway/telegram/client.py new file mode 100644 index 000000000..6f36f0564 --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/client.py @@ -0,0 +1,109 @@ +"""Thin async Telegram Bot API client.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from datetime import timedelta +from typing import Any + +from telegram import Bot +from telegram.error import BadRequest, RetryAfter + +from app.gateway.base.adapter import PlatformSendResult + + +def retry_after_seconds(value: int | timedelta) -> float: + if isinstance(value, timedelta): + return value.total_seconds() + return float(value) + + +class TelegramClient: + def __init__(self, token: str) -> None: + self.token = token + self.bot = Bot(token=token) + + async def send_message( + self, + *, + chat_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + kwargs: dict[str, Any] = {} + if parse_mode: + kwargs["parse_mode"] = parse_mode + if reply_to_message_id: + kwargs["reply_to_message_id"] = int(reply_to_message_id) + try: + msg = await self.bot.send_message(chat_id=chat_id, text=text, **kwargs) + except RetryAfter as exc: + await asyncio.sleep(retry_after_seconds(exc.retry_after)) + msg = await self.bot.send_message(chat_id=chat_id, text=text, **kwargs) + return PlatformSendResult( + external_message_id=str(msg.message_id), + raw_response=msg.to_dict(), + ) + + async def edit_message( + self, + *, + chat_id: str, + message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + kwargs: dict[str, Any] = {} + if parse_mode: + kwargs["parse_mode"] = parse_mode + try: + msg = await self.bot.edit_message_text( + chat_id=chat_id, + message_id=int(message_id), + text=text, + **kwargs, + ) + except RetryAfter as exc: + await asyncio.sleep(retry_after_seconds(exc.retry_after)) + msg = await self.bot.edit_message_text( + chat_id=chat_id, + message_id=int(message_id), + text=text, + **kwargs, + ) + return PlatformSendResult( + external_message_id=str(msg.message_id), + raw_response=msg.to_dict(), + ) + + async def validate(self) -> dict[str, Any]: + me = await self.bot.get_me() + return me.to_dict() + + async def leave_chat(self, *, chat_id: str) -> None: + await self.bot.leave_chat(chat_id=chat_id) + + async def get_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]: + next_offset = offset + while True: + updates = await self.bot.get_updates( + offset=next_offset, + timeout=30, + allowed_updates=["message", "edited_message"], + ) + for update in updates: + next_offset = update.update_id + 1 + yield update.to_dict() + + +async def retry_plaintext_on_bad_markdown(call, *args, **kwargs) -> PlatformSendResult: + try: + return await call(*args, **kwargs) + except BadRequest as exc: + if "can't parse entities" not in str(exc).lower(): + raise + kwargs["parse_mode"] = None + return await call(*args, **kwargs) + diff --git a/surfsense_backend/app/gateway/telegram/formatting.py b/surfsense_backend/app/gateway/telegram/formatting.py new file mode 100644 index 000000000..ecc7064bd --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/formatting.py @@ -0,0 +1,55 @@ +"""Telegram formatting helpers.""" + +from __future__ import annotations + +import re + +MARKDOWN_V2_RESERVED = r"_*[]()~`>#+-=|{}.!" +MAX_TELEGRAM_MESSAGE_UNITS = 4096 + +_RESERVED_RE = re.compile(r"([_\*\[\]\(\)~`>#+\-=|{}\.!])") + + +def escape_markdown_v2(text: str) -> str: + """Escape all Telegram MarkdownV2 reserved characters.""" + return _RESERVED_RE.sub(r"\\\1", text) + + +def _utf16_len(text: str) -> int: + return len(text.encode("utf-16-le")) // 2 + + +def _split_at_boundary(text: str, max_units: int) -> tuple[str, str]: + if _utf16_len(text) <= max_units: + return text, "" + + # Build a hard upper bound by code point, then walk back to natural + # boundaries. Telegram's limit is UTF-16 code units, so verify candidates. + end = min(len(text), max_units) + while end > 0 and _utf16_len(text[:end]) > max_units: + end -= 1 + + candidate = text[:end] + boundary = max(candidate.rfind("\n\n"), candidate.rfind(". "), candidate.rfind("\n")) + if boundary > max(200, end // 2): + end = boundary + (2 if candidate[boundary : boundary + 2] in {"\n\n", ". "} else 1) + + return text[:end], text[end:] + + +def chunk_message( + text: str, + *, + max_units: int = MAX_TELEGRAM_MESSAGE_UNITS, +) -> list[str]: + """Split a Telegram message at paragraph/sentence boundaries.""" + if not text: + return [""] + + chunks: list[str] = [] + remaining = text + while remaining: + chunk, remaining = _split_at_boundary(remaining, max_units) + chunks.append(chunk) + return chunks + diff --git a/surfsense_backend/tests/unit/gateway/test_formatting.py b/surfsense_backend/tests/unit/gateway/test_formatting.py new file mode 100644 index 000000000..61c7ea20f --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_formatting.py @@ -0,0 +1,18 @@ +from app.gateway.telegram.formatting import chunk_message, escape_markdown_v2 + + +def test_escape_markdown_v2_reserved_chars(): + text = r"_*[]()~`>#+-=|{}.!" + + assert escape_markdown_v2(text) == r"\_\*\[\]\(\)\~\`\>\#\+\-\=\|\{\}\.\!" + + +def test_chunk_message_preserves_content_and_limits_size(): + text = "First paragraph.\n\n" + ("x" * 5000) + + chunks = chunk_message(text, max_units=4096) + + assert "".join(chunks) == text + assert len(chunks) > 1 + assert all(len(chunk.encode("utf-16-le")) // 2 <= 4096 for chunk in chunks) + From 967ec099c8c4f3669aaac9474b9e7feb52b74013 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:38:25 +0530 Subject: [PATCH 06/63] feat(gateway): add Telegram command and stream handling --- .../app/gateway/telegram/commands.py | 91 +++++++++++ .../app/gateway/telegram/translator.py | 150 ++++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100644 surfsense_backend/app/gateway/telegram/commands.py create mode 100644 surfsense_backend/app/gateway/telegram/translator.py diff --git a/surfsense_backend/app/gateway/telegram/commands.py b/surfsense_backend/app/gateway/telegram/commands.py new file mode 100644 index 000000000..bc4a64377 --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/commands.py @@ -0,0 +1,91 @@ +"""Telegram command handlers.""" + +from __future__ import annotations + +from app.gateway.base.adapter import ParsedInboundEvent +from app.gateway.pairing import redeem_pairing_code +from app.gateway.ratelimit import acquire_token +from app.gateway.telegram.adapter import TelegramAdapter + +HELP_TEXT = ( + "SurfSense Telegram commands:\n" + "/start - pair this chat\n" + "/new - start a fresh conversation\n" + "/help - show this help" +) + + +def command_name(text: str | None) -> str | None: + if not text or not text.startswith("/"): + return None + return text.split(maxsplit=1)[0].split("@", 1)[0].lower() + + +async def handle_start_command( + *, + session, + adapter: TelegramAdapter, + event: ParsedInboundEvent, +) -> bool: + text = event.text or "" + parts = text.split(maxsplit=1) + if len(parts) != 2 or not event.external_peer_id: + await adapter.send_message( + external_peer_id=event.external_peer_id or "", + text="Generate a pairing code in SurfSense Settings > Messaging Channels, then send /start CODE here.", + ) + return True + + binding = await redeem_pairing_code( + session, + code=parts[1].strip(), + external_peer_id=event.external_peer_id, + external_peer_kind=event.external_peer_kind, + external_display_name=event.display_name, + external_username=event.username, + external_metadata=event.metadata, + ) + if binding is None: + await adapter.send_message( + external_peer_id=event.external_peer_id, + text="That pairing code is invalid or expired. Generate a new code in SurfSense.", + ) + return True + + await adapter.send_message( + external_peer_id=event.external_peer_id, + text="SurfSense is connected. Send a message here to chat with your agent.", + ) + return True + + +async def handle_help_command(*, adapter: TelegramAdapter, event: ParsedInboundEvent) -> bool: + if not event.external_peer_id: + return True + await adapter.send_message(external_peer_id=event.external_peer_id, text=HELP_TEXT) + return True + + +async def send_unbound_onboarding( + *, + adapter: TelegramAdapter, + event: ParsedInboundEvent, + dashboard_url: str, +) -> None: + if not event.external_peer_id: + return + wait_ms = await acquire_token( + f"tg:onboarded:{event.external_peer_id}", + capacity=1, + refill_per_sec=1 / 3600, + ) + if wait_ms > 0: + return + await adapter.send_message( + external_peer_id=event.external_peer_id, + text=( + "Hi! To use SurfSense via Telegram, generate a pairing code at " + f"{dashboard_url} and send /start CODE here." + ), + ) + diff --git a/surfsense_backend/app/gateway/telegram/translator.py b/surfsense_backend/app/gateway/telegram/translator.py new file mode 100644 index 000000000..d98f208c4 --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/translator.py @@ -0,0 +1,150 @@ +"""Translate agent stream events into Telegram messages.""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator + +from telegram.constants import ParseMode + +from app.gateway.base.adapter import PlatformSendResult +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.ratelimit import wait_for_token +from app.gateway.telegram.adapter import TelegramAdapter +from app.gateway.telegram.client import retry_plaintext_on_bad_markdown +from app.gateway.telegram.formatting import chunk_message, escape_markdown_v2 +from app.observability.metrics import ( + record_gateway_hitl_aborted, + record_gateway_outbound, + record_gateway_rate_limit_hit, +) + +logger = logging.getLogger(__name__) + +HITL_UNSUPPORTED_MESSAGE = ( + "This action requires approval and is not yet supported from Telegram. " + "Try again with a different request." +) + + +class TelegramStreamTranslator(BaseStreamTranslator): + def __init__( + self, + *, + adapter: TelegramAdapter, + external_peer_id: str, + assistant_message_id: int | None = None, + debounce_seconds: float = 1.5, + ) -> None: + self.adapter = adapter + self.external_peer_id = external_peer_id + self.assistant_message_id = assistant_message_id + self.debounce_seconds = debounce_seconds + self._buffer = "" + self._last_flush_at = 0.0 + self._external_message_ids: list[str] = [] + self._plaintext_mode = False + + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + async for event in events: + if event.type in {"text-delta", "text_delta", "text"}: + self._buffer += str(event.data.get("text") or event.data.get("delta") or "") + await self._maybe_flush() + elif event.type in {"data-interrupt-request", "interrupt"}: + await self._handle_hitl_interrupt() + return + elif event.type in {"text-end", "finish", "done"}: + break + + await self._flush(final=True) + + async def _maybe_flush(self) -> None: + now = asyncio.get_running_loop().time() + if now - self._last_flush_at < self.debounce_seconds: + return + await self._flush(final=False) + self._last_flush_at = now + + async def _flush(self, *, final: bool) -> None: + if not self._buffer: + return + + chunks = chunk_message(self._buffer) + # During streaming, keep edits on the last chunk only. At final flush, + # send any additional chunks and mark the message as finalized by the + # persistence layer (wired through agent/task code). + if len(chunks) > 1: + for chunk in chunks[:-1]: + result = await self._send_text(chunk) + self._external_message_ids.append(result.external_message_id) + self._buffer = chunks[-1] + + text = self._format_text(self._buffer) + if self._external_message_ids: + await self._edit_text(self._external_message_ids[-1], text) + else: + result = await self._send_text(self._buffer) + self._external_message_ids.append(result.external_message_id) + + if final: + logger.debug( + "Telegram gateway finalized assistant message id=%s external_ids=%s", + self.assistant_message_id, + self._external_message_ids, + ) + + def _format_text(self, text: str) -> str: + return text if self._plaintext_mode else escape_markdown_v2(text) + + async def _send_text(self, text: str) -> PlatformSendResult: + await self._throttle() + parse_mode = None if self._plaintext_mode else ParseMode.MARKDOWN_V2 + try: + result = await retry_plaintext_on_bad_markdown( + self.adapter.send_message, + external_peer_id=self.external_peer_id, + text=self._format_text(text), + parse_mode=parse_mode, + ) + except Exception: + record_gateway_outbound(platform="telegram", kind="send", status="failed") + raise + record_gateway_outbound(platform="telegram", kind="send", status="sent") + return result + + async def _edit_text(self, message_id: str, text: str) -> PlatformSendResult: + await self._throttle() + parse_mode = None if self._plaintext_mode else ParseMode.MARKDOWN_V2 + try: + result = await retry_plaintext_on_bad_markdown( + self.adapter.edit_message, + external_peer_id=self.external_peer_id, + external_message_id=message_id, + text=text, + parse_mode=parse_mode, + ) + except Exception: + record_gateway_outbound(platform="telegram", kind="edit", status="failed") + raise + record_gateway_outbound(platform="telegram", kind="edit", status="edited") + return result + + async def _throttle(self) -> None: + chat_wait = await wait_for_token( + f"tg:chat:{self.external_peer_id}", + capacity=1, + refill_per_sec=1.0, + ) + if chat_wait: + record_gateway_rate_limit_hit(bucket="tg:chat") + global_wait = await wait_for_token("tg:global", capacity=25, refill_per_sec=25.0) + if global_wait: + record_gateway_rate_limit_hit(bucket="tg:global") + + async def _handle_hitl_interrupt(self) -> None: + if self._buffer: + await self._flush(final=False) + await self._send_text(HITL_UNSUPPORTED_MESSAGE) + record_gateway_hitl_aborted(platform="telegram") + From b8538655bb9745f9aed698acad18d03d7f39f170 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:38:52 +0530 Subject: [PATCH 07/63] feat(gateway): process inbound events through the agent --- surfsense_backend/app/gateway/agent_invoke.py | 80 ++++++ .../app/gateway/inbox_processor.py | 262 ++++++++++++++++++ surfsense_backend/app/gateway/ratelimit.py | 136 +++++++++ surfsense_backend/app/gateway/thread_lock.py | 40 +++ 4 files changed, 518 insertions(+) create mode 100644 surfsense_backend/app/gateway/agent_invoke.py create mode 100644 surfsense_backend/app/gateway/inbox_processor.py create mode 100644 surfsense_backend/app/gateway/ratelimit.py create mode 100644 surfsense_backend/app/gateway/thread_lock.py diff --git a/surfsense_backend/app/gateway/agent_invoke.py b/surfsense_backend/app/gateway/agent_invoke.py new file mode 100644 index 000000000..b0cccddaa --- /dev/null +++ b/surfsense_backend/app/gateway/agent_invoke.py @@ -0,0 +1,80 @@ +"""Invoke SurfSense chat agent for gateway channels.""" + +from __future__ import annotations + +import json +import logging +from collections.abc import AsyncIterator + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import GatewayConversationBinding +from app.gateway.auth_invariant import assert_authorization_invariant +from app.gateway.base.translator import GatewayStreamEvent +from app.gateway.bindings import get_or_create_thread_for_binding +from app.gateway.hitl_filter import DEFAULT_HITL_TOOL_NAMES +from app.gateway.telegram.translator import TelegramStreamTranslator +from app.gateway.thread_lock import acquire_thread_lock, release_thread_lock +from app.observability.metrics import record_gateway_turn_latency +from app.tasks.chat.stream_new_chat import stream_new_chat + +logger = logging.getLogger(__name__) + + +async def _events_from_sse(chunks: AsyncIterator[str]) -> AsyncIterator[GatewayStreamEvent]: + async for chunk in chunks: + for raw_line in chunk.splitlines(): + line = raw_line.strip() + if not line.startswith("data:"): + continue + payload = line.removeprefix("data:").strip() + if payload == "[DONE]": + yield GatewayStreamEvent(type="done") + continue + try: + data = json.loads(payload) + except json.JSONDecodeError: + continue + event_type = str(data.get("type") or "") + if event_type == "text-delta": + yield GatewayStreamEvent(type="text-delta", data={"delta": data.get("delta", "")}) + elif event_type == "text-end": + yield GatewayStreamEvent(type="text-end", data=data) + elif event_type == "finish": + yield GatewayStreamEvent(type="finish", data=data) + elif event_type == "data-interrupt-request": + yield GatewayStreamEvent(type="data-interrupt-request", data=data) + + +async def call_agent_for_gateway( + *, + session: AsyncSession, + binding: GatewayConversationBinding, + user_text: str, + translator: TelegramStreamTranslator, + request_id: str | None = None, +) -> None: + user = await assert_authorization_invariant(session, binding) + thread = await get_or_create_thread_for_binding(session, binding) + await session.commit() + + if not acquire_thread_lock(thread.id): + raise RuntimeError("gateway_thread_busy") + + try: + stream = stream_new_chat( + user_query=user_text, + search_space_id=binding.search_space_id, + chat_id=thread.id, + user_id=str(user.id), + needs_history_bootstrap=thread.needs_history_bootstrap, + thread_visibility=thread.visibility, + current_user_display_name=user.display_name or "A team member", + disabled_tools=sorted(DEFAULT_HITL_TOOL_NAMES), + request_id=request_id or "gateway", + ) + await translator.translate(_events_from_sse(stream)) + record_gateway_turn_latency(0, platform="telegram") + finally: + release_thread_lock(thread.id) + diff --git a/surfsense_backend/app/gateway/inbox_processor.py b/surfsense_backend/app/gateway/inbox_processor.py new file mode 100644 index 000000000..3e3f962b7 --- /dev/null +++ b/surfsense_backend/app/gateway/inbox_processor.py @@ -0,0 +1,262 @@ +"""Long-lived gateway inbox processing. + +This module owns the agent-turn execution path for messaging gateways. It is +intentionally independent of Celery so LangGraph, async Postgres, Redis, and +Telegram clients all run on one stable event loop in ``GatewayRunner``. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from datetime import UTC, datetime + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from app.config import config +from app.db import ( + GatewayBindingState, + GatewayConversationBinding, + GatewayEventStatus, + GatewayInboundEvent, + GatewayPeerKind, + GatewayPlatformAccount, + NewChatThread, + async_session_maker, +) +from app.gateway.accounts import account_token +from app.gateway.agent_invoke import call_agent_for_gateway +from app.gateway.bindings import get_or_create_thread_for_binding +from app.gateway.telegram.adapter import TelegramAdapter +from app.gateway.telegram.commands import ( + command_name, + handle_help_command, + handle_start_command, + send_unbound_onboarding, +) +from app.gateway.telegram.translator import TelegramStreamTranslator +from app.observability.metrics import record_gateway_inbox_processed + +logger = logging.getLogger(__name__) + +SessionMaker = async_sessionmaker[AsyncSession] | Callable[[], AsyncSession] + + +def _dashboard_url() -> str: + return config.NEXT_FRONTEND_URL or "/dashboard" + + +async def claim_next_inbound_event( + session_maker: SessionMaker = async_session_maker, +) -> int | None: + """Claim the oldest received inbox event for processing.""" + + async with session_maker() as session: + result = await session.execute( + select(GatewayInboundEvent) + .where(GatewayInboundEvent.status == GatewayEventStatus.RECEIVED) + .order_by(GatewayInboundEvent.received_at.asc()) + .with_for_update(skip_locked=True) + .limit(1) + ) + event = result.scalars().first() + if event is None: + return None + event.status = GatewayEventStatus.PROCESSING + event.attempt_count += 1 + await session.commit() + return int(event.id) + + +async def process_inbound_event( + inbox_id: int, + session_maker: SessionMaker = async_session_maker, +) -> None: + """Process one gateway inbox row and mark its terminal status.""" + + async with session_maker() as session: + result = await session.execute( + select(GatewayInboundEvent) + .where(GatewayInboundEvent.id == inbox_id) + .with_for_update(skip_locked=True) + ) + event = result.scalars().first() + if event is None or event.status in { + GatewayEventStatus.PROCESSED, + GatewayEventStatus.IGNORED, + }: + return + if event.status == GatewayEventStatus.RECEIVED: + event.status = GatewayEventStatus.PROCESSING + event.attempt_count += 1 + await session.commit() + + try: + await _dispatch_inbound_event(inbox_id, session_maker) + except RuntimeError as exc: + if str(exc) == "gateway_thread_busy": + async with session_maker() as session: + await session.execute( + update(GatewayInboundEvent) + .where(GatewayInboundEvent.id == inbox_id) + .values( + status=GatewayEventStatus.RECEIVED, + last_error="gateway_thread_busy", + ) + ) + await session.commit() + return + await _mark_failed(inbox_id, str(exc), session_maker) + raise + except Exception as exc: + await _mark_failed(inbox_id, str(exc), session_maker) + raise + + async with session_maker() as session: + event = await session.get(GatewayInboundEvent, inbox_id) + if event is not None and event.status == GatewayEventStatus.PROCESSING: + event.status = GatewayEventStatus.PROCESSED + event.processed_at = datetime.now(UTC) + await session.commit() + record_gateway_inbox_processed(platform=event.platform.value, status="processed") + + +async def _mark_failed( + inbox_id: int, + error: str, + session_maker: SessionMaker, +) -> None: + async with session_maker() as session: + await session.execute( + update(GatewayInboundEvent) + .where(GatewayInboundEvent.id == inbox_id) + .values(status=GatewayEventStatus.FAILED, last_error=error) + ) + await session.commit() + + +async def _dispatch_inbound_event( + inbox_id: int, + session_maker: SessionMaker, +) -> None: + async with session_maker() as session: + event = await session.get(GatewayInboundEvent, inbox_id) + if event is None: + return + account = await session.get(GatewayPlatformAccount, event.account_id) + if account is None: + event.status = GatewayEventStatus.IGNORED + event.last_error = "account_missing" + await session.commit() + return + + token = account_token(account) + if not token: + event.status = GatewayEventStatus.FAILED + event.last_error = "missing_telegram_token" + await session.commit() + return + + adapter = TelegramAdapter(token) + parsed = adapter.parse_inbound(event.raw_payload or {}) + if parsed.external_peer_id is None: + event.status = GatewayEventStatus.IGNORED + event.last_error = "missing_external_peer_id" + await session.commit() + return + + _update_account_cursor(account, parsed.metadata.get("update_id")) + + result = await session.execute( + select(GatewayConversationBinding).where( + GatewayConversationBinding.account_id == account.id, + GatewayConversationBinding.external_peer_id == parsed.external_peer_id, + GatewayConversationBinding.state.in_( + [GatewayBindingState.BOUND, GatewayBindingState.SUSPENDED] + ), + ) + ) + binding = result.scalars().first() + + if parsed.external_peer_kind != GatewayPeerKind.DIRECT.value: + await adapter.leave_chat(external_peer_id=parsed.external_peer_id) + event.status = GatewayEventStatus.IGNORED + event.last_error = "group_rejected" + await session.commit() + return + + cmd = command_name(parsed.text) + if cmd == "/start": + handled = await handle_start_command( + session=session, adapter=adapter, event=parsed + ) + await session.commit() + if handled: + return + + if binding is None: + await send_unbound_onboarding( + adapter=adapter, + event=parsed, + dashboard_url=_dashboard_url(), + ) + event.status = GatewayEventStatus.IGNORED + event.last_error = "unbound_chat" + await session.commit() + return + + event.binding_id = binding.id + + if cmd == "/help": + await handle_help_command(adapter=adapter, event=parsed) + event.status = GatewayEventStatus.PROCESSED + await session.commit() + return + if cmd == "/new": + binding.active_thread_id = None + await adapter.send_message( + external_peer_id=parsed.external_peer_id, + text="Started a new SurfSense conversation.", + ) + event.status = GatewayEventStatus.PROCESSED + await session.commit() + return + + if not parsed.text: + event.status = GatewayEventStatus.IGNORED + event.last_error = "empty_message" + await session.commit() + return + + thread = await get_or_create_thread_for_binding(session, binding) + await session.commit() + + translator = TelegramStreamTranslator( + adapter=adapter, + external_peer_id=parsed.external_peer_id, + ) + await call_agent_for_gateway( + session=session, + binding=binding, + user_text=parsed.text, + translator=translator, + request_id=f"gateway:{inbox_id}", + ) + + thread = await session.get(NewChatThread, thread.id) + if thread is not None: + thread.source = "telegram" + await session.commit() + + +def _update_account_cursor(account: GatewayPlatformAccount, update_id: object) -> None: + if update_id is None: + return + account.cursor_state = { + **(account.cursor_state or {}), + "last_update_id": max( + int((account.cursor_state or {}).get("last_update_id", 0)), + int(update_id), + ), + } diff --git a/surfsense_backend/app/gateway/ratelimit.py b/surfsense_backend/app/gateway/ratelimit.py new file mode 100644 index 000000000..fbcbd16b8 --- /dev/null +++ b/surfsense_backend/app/gateway/ratelimit.py @@ -0,0 +1,136 @@ +"""Redis token-bucket rate limiter for gateway outbound traffic.""" + +from __future__ import annotations + +import asyncio +import logging +import time +from dataclasses import dataclass + +import redis.asyncio as aioredis + +from app.config import config +from app.observability.metrics import record_gateway_redis_fallback + +logger = logging.getLogger(__name__) + +_TOKEN_BUCKET_LUA = """ +local capacity = tonumber(ARGV[1]) +local refill_rate = tonumber(ARGV[2]) +local now = tonumber(ARGV[3]) +local consume = tonumber(ARGV[4]) + +local bucket = redis.call('HMGET', KEYS[1], 'tokens', 'last_refill') +local tokens = tonumber(bucket[1]) or capacity +local last_refill = tonumber(bucket[2]) or now + +local elapsed = math.max(0, now - last_refill) +tokens = math.min(capacity, tokens + (elapsed * refill_rate)) + +if tokens >= consume then + tokens = tokens - consume + redis.call('HMSET', KEYS[1], 'tokens', tokens, 'last_refill', now) + redis.call('EXPIRE', KEYS[1], 3600) + return 0 +else + redis.call('HMSET', KEYS[1], 'tokens', tokens, 'last_refill', now) + redis.call('EXPIRE', KEYS[1], 3600) + local needed = consume - tokens + return math.ceil((needed / refill_rate) * 1000) +end +""" + +_redis_client: aioredis.Redis | None = None + + +@dataclass +class _MemoryBucket: + tokens: float + last_refill: float + + +_memory_buckets: dict[str, _MemoryBucket] = {} +_memory_lock = asyncio.Lock() + + +def _redis() -> aioredis.Redis: + global _redis_client + if _redis_client is None: + _redis_client = aioredis.from_url(config.REDIS_APP_URL, decode_responses=True) + return _redis_client + + +async def _memory_fallback_acquire( + scope: str, + capacity: int, + refill_per_sec: float, + consume: float, +) -> int: + now = time.time() + async with _memory_lock: + bucket = _memory_buckets.get(scope) + if bucket is None: + bucket = _MemoryBucket(tokens=float(capacity), last_refill=now) + _memory_buckets[scope] = bucket + + elapsed = max(0.0, now - bucket.last_refill) + bucket.tokens = min(float(capacity), bucket.tokens + elapsed * refill_per_sec) + bucket.last_refill = now + + if bucket.tokens >= consume: + bucket.tokens -= consume + return 0 + + needed = consume - bucket.tokens + return int((needed / refill_per_sec) * 1000) if refill_per_sec > 0 else 1000 + + +async def acquire_token( + scope: str, + *, + capacity: int, + refill_per_sec: float, + consume: float = 1.0, +) -> int: + """Return 0 if allowed, otherwise milliseconds to wait. + + Redis is the primary coordination mechanism. If Redis is unavailable, + fall back to per-process memory so the gateway degrades instead of failing + closed during a short Redis outage. + """ + + redis_key = f"gateway:bucket:{scope}" + try: + wait_ms = await _redis().eval( + _TOKEN_BUCKET_LUA, + 1, + redis_key, + capacity, + refill_per_sec, + time.time(), + consume, + ) + return int(wait_ms) + except (aioredis.RedisError, OSError) as exc: + logger.warning("Redis rate limiter unavailable; using memory fallback: %s", exc) + record_gateway_redis_fallback() + return await _memory_fallback_acquire(scope, capacity, refill_per_sec, consume) + + +async def wait_for_token( + scope: str, + *, + capacity: int, + refill_per_sec: float, + consume: float = 1.0, +) -> int: + wait_ms = await acquire_token( + scope, + capacity=capacity, + refill_per_sec=refill_per_sec, + consume=consume, + ) + if wait_ms > 0: + await asyncio.sleep(wait_ms / 1000) + return wait_ms + diff --git a/surfsense_backend/app/gateway/thread_lock.py b/surfsense_backend/app/gateway/thread_lock.py new file mode 100644 index 000000000..82733bb69 --- /dev/null +++ b/surfsense_backend/app/gateway/thread_lock.py @@ -0,0 +1,40 @@ +"""Redis-backed distributed locks for gateway conversation turns.""" + +from __future__ import annotations + +import logging + +import redis + +from app.config import config +from app.observability.metrics import record_gateway_thread_lock_contention + +logger = logging.getLogger(__name__) + +_redis_client: redis.Redis | None = None + + +def _redis() -> redis.Redis: + global _redis_client + if _redis_client is None: + _redis_client = redis.from_url(config.REDIS_APP_URL, decode_responses=True) + return _redis_client + + +def _lock_key(thread_id: int) -> str: + return f"gateway:thread_lock:{thread_id}" + + +def acquire_thread_lock(thread_id: int, ttl: int = 60) -> bool: + acquired = bool(_redis().set(_lock_key(thread_id), "1", nx=True, ex=ttl)) + if not acquired: + record_gateway_thread_lock_contention() + return acquired + + +def release_thread_lock(thread_id: int) -> None: + try: + _redis().delete(_lock_key(thread_id)) + except redis.RedisError as exc: + logger.warning("Failed to release gateway thread lock for %s: %s", thread_id, exc) + From bd86a72587911be1d956244803a6c8c017b4bd5e Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:39:05 +0530 Subject: [PATCH 08/63] feat(gateway): add long-lived gateway runner --- surfsense_backend/app/gateway/runner.py | 97 +++++++++++++++++++++++++ surfsense_backend/gateway_runner.py | 11 +++ 2 files changed, 108 insertions(+) create mode 100644 surfsense_backend/app/gateway/runner.py create mode 100644 surfsense_backend/gateway_runner.py diff --git a/surfsense_backend/app/gateway/runner.py b/surfsense_backend/app/gateway/runner.py new file mode 100644 index 000000000..1e56ff25d --- /dev/null +++ b/surfsense_backend/app/gateway/runner.py @@ -0,0 +1,97 @@ +"""Long-lived messaging gateway runner.""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging + +from sqlalchemy import select, text + +from app.db import GatewayPlatform, GatewayPlatformAccount, async_session_maker, engine +from app.gateway.accounts import account_token +from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key +from app.gateway.inbox_processor import claim_next_inbound_event, process_inbound_event +from app.gateway.telegram.adapter import TelegramAdapter + +logger = logging.getLogger(__name__) + + +def _lock_key(token: str) -> int: + digest = hashlib.sha256(f"gateway:telegram:{token}".encode()).digest() + return int.from_bytes(digest[:8], "big", signed=True) + + +class GatewayRunner: + async def run(self) -> None: + print("Gateway runner started. Waiting for inbound events...", flush=True) + tasks = [asyncio.create_task(self._process_inbox_forever())] + + async with async_session_maker() as session: + result = await session.execute( + select(GatewayPlatformAccount).where( + GatewayPlatformAccount.platform == GatewayPlatform.TELEGRAM, + GatewayPlatformAccount.is_system_account.is_(False), + GatewayPlatformAccount.suspended_at.is_(None), + ) + ) + accounts = list(result.scalars()) + + for account in accounts: + token = account_token(account) + if not token: + continue + logger.info("Starting Telegram long-poll loop for account_id=%s", account.id) + tasks.append(asyncio.create_task(self._run_telegram_account(account.id, token))) + + await asyncio.gather(*tasks) + + async def _process_inbox_forever(self) -> None: + logger.info("Gateway inbox processor started") + while True: + try: + inbox_id = await claim_next_inbound_event() + if inbox_id is None: + await asyncio.sleep(0.5) + continue + logger.info("Gateway processing inbox_id=%s", inbox_id) + await process_inbound_event(inbox_id) + logger.info("Gateway processed inbox_id=%s", inbox_id) + except Exception: + logger.exception("Gateway inbox processor failed one iteration") + await asyncio.sleep(1) + + async def _run_telegram_account(self, account_id: int, token: str) -> None: + async with engine.connect() as conn: + got_lock = await conn.scalar( + text("SELECT pg_try_advisory_lock(:key)"), + {"key": _lock_key(token)}, + ) + if not got_lock: + logger.warning("Another Telegram gateway runner is active; exiting") + return + + adapter = TelegramAdapter(token) + async with async_session_maker() as session: + account = await session.get(GatewayPlatformAccount, account_id) + offset = None + if account is not None: + offset = int((account.cursor_state or {}).get("last_update_id", 0)) + 1 + + async for update in adapter.fetch_updates(offset=offset): + async with async_session_maker() as session: + parsed = adapter.parse_inbound(update) + inbox_id = await persist_inbound_event( + session, + account_id=account_id, + platform=GatewayPlatform.TELEGRAM, + event_dedupe_key=telegram_event_dedupe_key(update["update_id"]), + external_event_id=str(update["update_id"]), + external_message_id=parsed.external_message_id, + event_kind=parsed.event_kind, + raw_payload=update, + ) + await session.commit() + if inbox_id is not None: + logger.debug("Persisted Telegram polling update inbox_id=%s", inbox_id) + diff --git a/surfsense_backend/gateway_runner.py b/surfsense_backend/gateway_runner.py new file mode 100644 index 000000000..72a1749a9 --- /dev/null +++ b/surfsense_backend/gateway_runner.py @@ -0,0 +1,11 @@ +"""Entrypoint for SERVICE_ROLE=gateway.""" + +from __future__ import annotations + +import asyncio + +from app.gateway.runner import GatewayRunner + +if __name__ == "__main__": + asyncio.run(GatewayRunner().run()) + From d32e8c6a90391f36f6e59f5dc5c3632a45101b6f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:39:24 +0530 Subject: [PATCH 09/63] feat(gateway): expose binding and webhook APIs --- surfsense_backend/app/routes/__init__.py | 2 + .../app/routes/gateway_webhook_routes.py | 232 ++++++++++++++++++ surfsense_backend/scripts/register_webhook.py | 52 ++++ 3 files changed, 286 insertions(+) create mode 100644 surfsense_backend/app/routes/gateway_webhook_routes.py create mode 100644 surfsense_backend/scripts/register_webhook.py diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index ec4d1650f..f46b6fc65 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -18,6 +18,7 @@ from .dropbox_add_connector_route import router as dropbox_add_connector_router from .editor_routes import router as editor_router from .export_routes import router as export_router from .folders_routes import router as folders_router +from .gateway_webhook_routes import router as gateway_router from .google_calendar_add_connector_route import ( router as google_calendar_add_connector_router, ) @@ -68,6 +69,7 @@ router.include_router(editor_router) router.include_router(export_router) router.include_router(documents_router) router.include_router(folders_router) +router.include_router(gateway_router) router.include_router(notes_router) router.include_router(new_chat_router) # Chat with assistant-ui persistence router.include_router(agent_revert_router) # POST /threads/{id}/revert/{action_id} diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py new file mode 100644 index 000000000..86b84f067 --- /dev/null +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -0,0 +1,232 @@ +"""Messaging gateway routes.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Request +from pydantic import BaseModel +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from starlette.responses import Response + +from app.config import config +from app.db import ( + GatewayBindingState, + GatewayConversationBinding, + GatewayPlatform, + GatewayPlatformAccount, + User, + get_async_session, +) +from app.gateway.accounts import get_or_create_system_telegram_account +from app.gateway.bindings import resume_binding, revoke_binding +from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key +from app.gateway.pairing import generate_pairing_code, pairing_expires_at +from app.observability.metrics import record_gateway_inbox_write +from app.rate_limiter import limiter +from app.users import current_active_user + +router = APIRouter(prefix="/gateway", tags=["gateway"]) + + +class StartBindingRequest(BaseModel): + platform: GatewayPlatform = GatewayPlatform.TELEGRAM + search_space_id: int + + +class StartBindingResponse(BaseModel): + binding_id: int + code: str + deep_link: str + expires_at: datetime + + +def _classify_telegram_event(payload: dict[str, Any]) -> str: + if "message" in payload: + return "message" + if "edited_message" in payload: + return "edited_message" + if "callback_query" in payload: + return "callback_query" + return "other" + + +def _telegram_message(payload: dict[str, Any]) -> dict[str, Any] | None: + return payload.get("message") or payload.get("edited_message") + + +async def _resolve_webhook_account( + session: AsyncSession, + *, + secret: str, + header_secret: str | None, +) -> GatewayPlatformAccount: + if config.TELEGRAM_WEBHOOK_SECRET and secret == config.TELEGRAM_WEBHOOK_SECRET: + if header_secret != config.TELEGRAM_WEBHOOK_SECRET: + raise HTTPException(status_code=403, detail="Invalid Telegram webhook secret") + return await get_or_create_system_telegram_account(session) + + result = await session.execute( + select(GatewayPlatformAccount).where( + GatewayPlatformAccount.platform == GatewayPlatform.TELEGRAM + ) + ) + for account in result.scalars(): + metadata = account.account_metadata or {} + webhook_secret = metadata.get("webhook_secret") + if webhook_secret and webhook_secret == secret: + if header_secret != webhook_secret: + raise HTTPException(status_code=403, detail="Invalid Telegram webhook secret") + return account + + raise HTTPException(status_code=404, detail="Gateway account not found") + + +@router.post("/webhooks/telegram/{secret}") +@limiter.limit("60/minute", key_func=lambda request: f"tg-webhook:{request.path_params['secret']}") +async def telegram_webhook( + request: Request, + secret: str, + session: AsyncSession = Depends(get_async_session), +) -> Response: + payload = await request.json() + account = await _resolve_webhook_account( + session, + secret=secret, + header_secret=request.headers.get("X-Telegram-Bot-Api-Secret-Token"), + ) + update_id = payload.get("update_id") + if update_id is None: + return Response(status_code=200) + + message = _telegram_message(payload) or {} + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=GatewayPlatform.TELEGRAM, + event_dedupe_key=telegram_event_dedupe_key(update_id), + external_event_id=str(update_id), + external_message_id=( + str(message["message_id"]) if message.get("message_id") is not None else None + ), + event_kind=_classify_telegram_event(payload), + raw_payload=payload, + ) + await session.commit() + record_gateway_inbox_write(platform="telegram", dedup_skipped=inbox_id is None) + return Response(status_code=200) + + +@router.post("/bindings/start", response_model=StartBindingResponse) +async def start_binding( + body: StartBindingRequest, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> StartBindingResponse: + if body.platform != GatewayPlatform.TELEGRAM: + raise HTTPException(status_code=400, detail="Only Telegram is supported in v1") + + account = await get_or_create_system_telegram_account(session) + code = generate_pairing_code() + expires_at = pairing_expires_at() + binding = GatewayConversationBinding( + account_id=account.id, + user_id=user.id, + search_space_id=body.search_space_id, + state=GatewayBindingState.PENDING, + pairing_code=code, + pairing_code_expires_at=expires_at, + ) + session.add(binding) + await session.commit() + await session.refresh(binding) + + username = account.account_metadata.get("bot_username") or config.TELEGRAM_SHARED_BOT_USERNAME + if not username: + raise HTTPException(status_code=500, detail="Telegram bot username is not configured") + return StartBindingResponse( + binding_id=binding.id, + code=code, + deep_link=f"https://t.me/{username}?start={code}", + expires_at=expires_at, + ) + + +@router.get("/bindings") +async def list_bindings( + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> list[dict[str, Any]]: + result = await session.execute( + select(GatewayConversationBinding).where( + GatewayConversationBinding.user_id == user.id + ) + ) + return [ + { + "id": binding.id, + "platform": "telegram", + "state": binding.state.value, + "search_space_id": binding.search_space_id, + "external_display_name": binding.external_display_name, + "external_username": binding.external_username, + "suspended_reason": binding.suspended_reason, + } + for binding in result.scalars() + ] + + +@router.get("/platforms") +async def list_platforms( + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> list[dict[str, Any]]: + result = await session.execute( + select(GatewayPlatformAccount).where( + (GatewayPlatformAccount.owner_user_id == user.id) + | (GatewayPlatformAccount.is_system_account.is_(True)) + ) + ) + return [ + { + "id": account.id, + "platform": account.platform.value, + "mode": account.mode.value, + "bot_username": (account.account_metadata or {}).get("bot_username"), + "health_status": account.health_status.value, + "last_health_check_at": account.last_health_check_at, + } + for account in result.scalars() + ] + + +@router.delete("/bindings/{binding_id}") +async def delete_binding( + binding_id: int, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, bool]: + binding = await session.get(GatewayConversationBinding, binding_id) + if binding is None or binding.user_id != user.id: + raise HTTPException(status_code=404, detail="Binding not found") + revoke_binding(binding) + await session.commit() + return {"ok": True} + + +@router.post("/bindings/{binding_id}/resume") +async def resume_gateway_binding( + binding_id: int, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, bool]: + binding = await session.get(GatewayConversationBinding, binding_id) + if binding is None or binding.user_id != user.id: + raise HTTPException(status_code=404, detail="Binding not found") + resume_binding(binding) + binding.updated_at = datetime.now(UTC) + await session.commit() + return {"ok": True} + diff --git a/surfsense_backend/scripts/register_webhook.py b/surfsense_backend/scripts/register_webhook.py new file mode 100644 index 000000000..2004ad118 --- /dev/null +++ b/surfsense_backend/scripts/register_webhook.py @@ -0,0 +1,52 @@ +"""Register the SurfSense Telegram webhook.""" + +from __future__ import annotations + +import asyncio +import os +import re +import sys + +from dotenv import load_dotenv +from telegram import Bot + +load_dotenv() + +WEBHOOK_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{1,256}$") + + +async def main() -> int: + token = os.getenv("TELEGRAM_SHARED_BOT_TOKEN") + secret = os.getenv("TELEGRAM_WEBHOOK_SECRET") + base_url = os.getenv("GATEWAY_BASE_URL") or os.getenv("BACKEND_URL") + if not token or not secret or not base_url: + print( + "Missing TELEGRAM_SHARED_BOT_TOKEN, TELEGRAM_WEBHOOK_SECRET, or GATEWAY_BASE_URL/BACKEND_URL", + file=sys.stderr, + ) + return 1 + if not WEBHOOK_SECRET_RE.fullmatch(secret): + print( + "TELEGRAM_WEBHOOK_SECRET must be 1-256 chars and contain only A-Z, a-z, 0-9, '_' or '-'", + file=sys.stderr, + ) + return 1 + + webhook_url = f"{base_url.rstrip('/')}/api/v1/gateway/webhooks/telegram/{secret}" + bot = Bot(token=token) + ok = await bot.set_webhook( + url=webhook_url, + secret_token=secret, + allowed_updates=["message", "edited_message"], + drop_pending_updates=True, + ) + if not ok: + print("Telegram rejected webhook registration", file=sys.stderr) + return 1 + print(f"Registered Telegram webhook: {webhook_url}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) + From a428f6c05f1481a5f0b39ba6e2c7a27d26811e92 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:40:47 +0530 Subject: [PATCH 10/63] feat(gateway): schedule gateway maintenance tasks --- surfsense_backend/app/celery_app.py | 20 +++ .../app/tasks/celery_tasks/gateway_tasks.py | 138 ++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py diff --git a/surfsense_backend/app/celery_app.py b/surfsense_backend/app/celery_app.py index 5b45baca1..2423133fb 100644 --- a/surfsense_backend/app/celery_app.py +++ b/surfsense_backend/app/celery_app.py @@ -188,6 +188,7 @@ celery_app = Celery( "app.tasks.celery_tasks.document_reindex_tasks", "app.tasks.celery_tasks.stale_notification_cleanup_task", "app.tasks.celery_tasks.stripe_reconciliation_task", + "app.tasks.celery_tasks.gateway_tasks", ], ) @@ -242,6 +243,10 @@ celery_app.conf.update( "index_obsidian_attachment": {"queue": CONNECTORS_QUEUE}, # Everything else (document processing, podcasts, reindexing, # schedule checker, cleanup) stays on the default fast queue. + "gateway.process_inbound_event": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, + "gateway.reconcile_inbox": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, + "gateway.health_check": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, + "gateway.retention_sweep": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, }, ) @@ -282,4 +287,19 @@ celery_app.conf.beat_schedule = { "expires": 60, }, }, + "gateway-reconcile-inbox": { + "task": "gateway.reconcile_inbox", + "schedule": crontab(minute="*"), + "options": {"expires": 60}, + }, + "gateway-health-check": { + "task": "gateway.health_check", + "schedule": crontab(minute="*/5"), + "options": {"expires": 120}, + }, + "gateway-retention-sweep": { + "task": "gateway.retention_sweep", + "schedule": crontab(hour="3", minute="17"), + "options": {"expires": 600}, + }, } diff --git a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py new file mode 100644 index 000000000..b8076b5a7 --- /dev/null +++ b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py @@ -0,0 +1,138 @@ +"""Celery tasks for messaging gateway intake and maintenance.""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select, update + +from app.celery_app import celery_app +from app.db import ( + GatewayEventStatus, + GatewayHealthStatus, + GatewayInboundEvent, + GatewayPlatform, + GatewayPlatformAccount, +) +from app.gateway.accounts import account_token +from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key +from app.gateway.telegram.adapter import TelegramAdapter +from app.observability.metrics import ( + record_gateway_health_check_failure, + record_gateway_inbound_reconciled, +) +from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task + +logger = logging.getLogger(__name__) + + +@celery_app.task( + bind=True, + name="gateway.process_inbound_event", + acks_late=True, + max_retries=5, + retry_backoff=True, +) +def process_inbound_event_task(self, inbox_id: int) -> None: + logger.warning( + "Ignoring Celery gateway.process_inbound_event for inbox_id=%s; " + "GatewayRunner owns agent turn processing.", + inbox_id, + ) + return None + + +@celery_app.task(name="gateway.reconcile_inbox") +def reconcile_inbox_task() -> None: + async def _run() -> None: + session_maker = get_celery_session_maker() + async with session_maker() as session: + stale_threshold = datetime.now(UTC) - timedelta(minutes=10) + result = await session.execute( + update(GatewayInboundEvent) + .where( + GatewayInboundEvent.status == GatewayEventStatus.PROCESSING, + GatewayInboundEvent.received_at < stale_threshold, + ) + .values( + status=GatewayEventStatus.RECEIVED, + last_error="stale processing reset for gateway runner", + ) + ) + for _ in range(result.rowcount or 0): + record_gateway_inbound_reconciled(reason="stale_processing_reset") + await session.commit() + + return run_async_celery_task(_run) + + +@celery_app.task(name="gateway.health_check") +def gateway_health_check_task() -> None: + async def _run() -> None: + session_maker = get_celery_session_maker() + async with session_maker() as session: + result = await session.execute(select(GatewayPlatformAccount)) + accounts = list(result.scalars()) + for account in accounts: + token = account_token(account) + if not token or account.platform != GatewayPlatform.TELEGRAM: + continue + try: + metadata = await TelegramAdapter(token).validate_credentials() + account.health_status = GatewayHealthStatus.OK + account.account_metadata = { + **(account.account_metadata or {}), + "bot_username": metadata.get("username"), + } + except Exception: + logger.warning("Gateway Telegram health check failed", exc_info=True) + account.health_status = GatewayHealthStatus.FAILING + record_gateway_health_check_failure(platform=account.platform.value) + account.last_health_check_at = datetime.now(UTC) + await session.commit() + + return run_async_celery_task(_run) + + +@celery_app.task(name="gateway.retention_sweep") +def gateway_retention_sweep_task() -> None: + async def _run() -> None: + session_maker = get_celery_session_maker() + async with session_maker() as session: + raw_cutoff = datetime.now(UTC) - timedelta(days=30) + delete_cutoff = datetime.now(UTC) - timedelta(days=365) + await session.execute( + update(GatewayInboundEvent) + .where(GatewayInboundEvent.received_at < raw_cutoff) + .values(raw_payload=None) + ) + result = await session.execute( + select(GatewayInboundEvent).where( + GatewayInboundEvent.received_at < delete_cutoff + ) + ) + for event in result.scalars(): + await session.delete(event) + await session.commit() + + return run_async_celery_task(_run) + + +async def enqueue_telegram_update(account_id: int, raw_update: dict) -> int | None: + session_maker = get_celery_session_maker() + async with session_maker() as session: + parsed = TelegramAdapter("placeholder").parse_inbound(raw_update) + inbox_id = await persist_inbound_event( + session, + account_id=account_id, + platform=GatewayPlatform.TELEGRAM, + event_dedupe_key=telegram_event_dedupe_key(raw_update["update_id"]), + external_event_id=str(raw_update["update_id"]), + external_message_id=parsed.external_message_id, + event_kind=parsed.event_kind, + raw_payload=raw_update, + ) + await session.commit() + return inbox_id + From 5f9d16530d61fb5e1d327c6fcd4d83f32065c21e Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 27 May 2026 23:41:18 +0530 Subject: [PATCH 11/63] feat(web): add messaging channels settings page --- .../components/MessagingChannelsContent.tsx | 174 ++++++++++++++++++ .../user-settings/layout-shell.tsx | 9 +- .../user-settings/messaging-channels/page.tsx | 6 + 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx create mode 100644 surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx new file mode 100644 index 000000000..0c35533c6 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx @@ -0,0 +1,174 @@ +"use client"; + +import { MessageCircle, RefreshCw, ShieldAlert } from "lucide-react"; +import { useParams } from "next/navigation"; +import { useCallback, useEffect, useState } from "react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { authenticatedFetch } from "@/lib/auth-utils"; +import { BACKEND_URL } from "@/lib/env-config"; + +type Binding = { + id: number; + state: string; + search_space_id: number; + external_display_name?: string | null; + external_username?: string | null; + suspended_reason?: string | null; +}; + +type Platform = { + id: number; + platform: string; + mode: string; + bot_username?: string | null; + health_status: string; + last_health_check_at?: string | null; +}; + +type Pairing = { + binding_id: number; + code: string; + deep_link: string; + expires_at: string; +}; + +export function MessagingChannelsContent() { + const params = useParams<{ search_space_id: string }>(); + const searchSpaceId = Number(params.search_space_id); + const [bindings, setBindings] = useState([]); + const [platforms, setPlatforms] = useState([]); + const [pairing, setPairing] = useState(null); + const [loading, setLoading] = useState(true); + + const refresh = useCallback(async () => { + setLoading(true); + const [bindingsRes, platformsRes] = await Promise.all([ + authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings`), + authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/platforms`), + ]); + setBindings(await bindingsRes.json()); + setPlatforms(await platformsRes.json()); + setLoading(false); + }, []); + + useEffect(() => { + void refresh(); + }, [refresh]); + + async function startPairing() { + const res = await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings/start`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ platform: "telegram", search_space_id: searchSpaceId }), + }); + setPairing(await res.json()); + await refresh(); + } + + async function revoke(id: number) { + await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings/${id}`, { + method: "DELETE", + }); + await refresh(); + } + + async function resume(id: number) { + await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings/${id}/resume`, { + method: "POST", + }); + await refresh(); + } + + const telegram = platforms.find((p) => p.platform === "telegram"); + const activeBindings = bindings.filter((binding) => binding.search_space_id === searchSpaceId); + + return ( +
+ + +
+ + + Telegram + + + {telegram?.health_status ?? "not configured"} + +
+

+ Pair a Telegram chat with this search space. Telegram conversations stay in Telegram and + are not mirrored in SurfSense chat history. +

+ + +
+ + +
+ + {pairing ? ( +
+

Pairing code

+

{pairing.code}

+ + Open Telegram pairing link + +

+ Expires at {new Date(pairing.expires_at).toLocaleString()}. SurfSense stores this + channel's messages for agent memory and operational debugging. +

+
+ ) : null} + + + + + + Active Chats + + + {activeBindings.length === 0 ? ( +

No Telegram chats paired yet.

+ ) : ( + activeBindings.map((binding) => ( +
+
+

+ {binding.external_display_name || + binding.external_username || + `Binding ${binding.id}`} +

+

{binding.state}

+ {binding.suspended_reason ? ( +

+ + {binding.suspended_reason} +

+ ) : null} +
+
+ {binding.state === "suspended" ? ( + + ) : null} + +
+
+ )) + )} + + +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/layout-shell.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/layout-shell.tsx index 037568db3..4aac4d2f6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/layout-shell.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/layout-shell.tsx @@ -5,6 +5,7 @@ import { Keyboard, KeyRound, Library, + MessageCircle, Monitor, ReceiptText, ShieldCheck, @@ -29,7 +30,8 @@ export type UserSettingsTab = | "agent-status" | "purchases" | "desktop" - | "hotkeys"; + | "hotkeys" + | "messaging-channels"; const DEFAULT_TAB: UserSettingsTab = "profile"; @@ -83,6 +85,11 @@ export function UserSettingsLayoutShell({ searchSpaceId, children }: UserSetting label: "Agent Status", icon: , }, + { + value: "messaging-channels" as const, + label: "Messaging Channels", + icon: , + }, { value: "purchases" as const, label: "Purchase History", diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx new file mode 100644 index 000000000..31dc6b56a --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx @@ -0,0 +1,6 @@ +import { MessagingChannelsContent } from "../components/MessagingChannelsContent"; + +export default function Page() { + return ; +} + From 708e3a9120b8114cac36a57bfe7f5319096b475e Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 28 May 2026 00:07:37 +0530 Subject: [PATCH 12/63] feat(gateway): enhance logging and event handling in agent and Telegram translator --- surfsense_backend/app/gateway/agent_invoke.py | 14 +++++++---- surfsense_backend/app/gateway/runner.py | 2 +- .../app/gateway/telegram/translator.py | 23 ++++++++++++++++++- surfsense_backend/gateway_runner.py | 5 ++++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/surfsense_backend/app/gateway/agent_invoke.py b/surfsense_backend/app/gateway/agent_invoke.py index b0cccddaa..4faf3711f 100644 --- a/surfsense_backend/app/gateway/agent_invoke.py +++ b/surfsense_backend/app/gateway/agent_invoke.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) async def _events_from_sse(chunks: AsyncIterator[str]) -> AsyncIterator[GatewayStreamEvent]: + saw_text = False async for chunk in chunks: for raw_line in chunk.splitlines(): line = raw_line.strip() @@ -29,6 +30,7 @@ async def _events_from_sse(chunks: AsyncIterator[str]) -> AsyncIterator[GatewayS continue payload = line.removeprefix("data:").strip() if payload == "[DONE]": + logger.info("Gateway SSE normalized: done") yield GatewayStreamEvent(type="done") continue try: @@ -37,12 +39,16 @@ async def _events_from_sse(chunks: AsyncIterator[str]) -> AsyncIterator[GatewayS continue event_type = str(data.get("type") or "") if event_type == "text-delta": - yield GatewayStreamEvent(type="text-delta", data={"delta": data.get("delta", "")}) - elif event_type == "text-end": - yield GatewayStreamEvent(type="text-end", data=data) - elif event_type == "finish": + delta = data.get("delta", "") + if delta and not saw_text: + logger.info("Gateway SSE normalized: text stream started") + saw_text = True + yield GatewayStreamEvent(type="text-delta", data={"delta": delta}) + elif event_type in {"finish", "done"}: + logger.info("Gateway SSE normalized: %s", event_type) yield GatewayStreamEvent(type="finish", data=data) elif event_type == "data-interrupt-request": + logger.info("Gateway SSE normalized: interrupt request") yield GatewayStreamEvent(type="data-interrupt-request", data=data) diff --git a/surfsense_backend/app/gateway/runner.py b/surfsense_backend/app/gateway/runner.py index 1e56ff25d..8ebd89253 100644 --- a/surfsense_backend/app/gateway/runner.py +++ b/surfsense_backend/app/gateway/runner.py @@ -24,7 +24,7 @@ def _lock_key(token: str) -> int: class GatewayRunner: async def run(self) -> None: - print("Gateway runner started. Waiting for inbound events...", flush=True) + logger.info("Gateway runner started. Waiting for inbound events.") tasks = [asyncio.create_task(self._process_inbox_forever())] async with async_session_maker() as session: diff --git a/surfsense_backend/app/gateway/telegram/translator.py b/surfsense_backend/app/gateway/telegram/translator.py index d98f208c4..96903bea0 100644 --- a/surfsense_backend/app/gateway/telegram/translator.py +++ b/surfsense_backend/app/gateway/telegram/translator.py @@ -54,7 +54,7 @@ class TelegramStreamTranslator(BaseStreamTranslator): elif event.type in {"data-interrupt-request", "interrupt"}: await self._handle_hitl_interrupt() return - elif event.type in {"text-end", "finish", "done"}: + elif event.type in {"finish", "done"}: break await self._flush(final=True) @@ -100,6 +100,11 @@ class TelegramStreamTranslator(BaseStreamTranslator): async def _send_text(self, text: str) -> PlatformSendResult: await self._throttle() parse_mode = None if self._plaintext_mode else ParseMode.MARKDOWN_V2 + logger.info( + "Telegram gateway sending message peer=%s chars=%d", + self.external_peer_id, + len(text), + ) try: result = await retry_plaintext_on_bad_markdown( self.adapter.send_message, @@ -110,12 +115,23 @@ class TelegramStreamTranslator(BaseStreamTranslator): except Exception: record_gateway_outbound(platform="telegram", kind="send", status="failed") raise + logger.info( + "Telegram gateway sent message peer=%s message_id=%s", + self.external_peer_id, + result.external_message_id, + ) record_gateway_outbound(platform="telegram", kind="send", status="sent") return result async def _edit_text(self, message_id: str, text: str) -> PlatformSendResult: await self._throttle() parse_mode = None if self._plaintext_mode else ParseMode.MARKDOWN_V2 + logger.info( + "Telegram gateway editing message peer=%s message_id=%s chars=%d", + self.external_peer_id, + message_id, + len(text), + ) try: result = await retry_plaintext_on_bad_markdown( self.adapter.edit_message, @@ -127,6 +143,11 @@ class TelegramStreamTranslator(BaseStreamTranslator): except Exception: record_gateway_outbound(platform="telegram", kind="edit", status="failed") raise + logger.info( + "Telegram gateway edited message peer=%s message_id=%s", + self.external_peer_id, + result.external_message_id, + ) record_gateway_outbound(platform="telegram", kind="edit", status="edited") return result diff --git a/surfsense_backend/gateway_runner.py b/surfsense_backend/gateway_runner.py index 72a1749a9..27077ef48 100644 --- a/surfsense_backend/gateway_runner.py +++ b/surfsense_backend/gateway_runner.py @@ -3,9 +3,14 @@ from __future__ import annotations import asyncio +import logging from app.gateway.runner import GatewayRunner if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s [%(name)s] %(message)s", + ) asyncio.run(GatewayRunner().run()) From f2d82234d4af21cd3e7e904a3eb5524a0bad1dbc Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 28 May 2026 00:25:12 +0530 Subject: [PATCH 13/63] fix(gateway): ensure proper closure of event streams in agent invocation --- surfsense_backend/app/gateway/agent_invoke.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/gateway/agent_invoke.py b/surfsense_backend/app/gateway/agent_invoke.py index 4faf3711f..b876d1977 100644 --- a/surfsense_backend/app/gateway/agent_invoke.py +++ b/surfsense_backend/app/gateway/agent_invoke.py @@ -79,7 +79,12 @@ async def call_agent_for_gateway( disabled_tools=sorted(DEFAULT_HITL_TOOL_NAMES), request_id=request_id or "gateway", ) - await translator.translate(_events_from_sse(stream)) + events = _events_from_sse(stream) + try: + await translator.translate(events) + finally: + await events.aclose() + await stream.aclose() record_gateway_turn_latency(0, platform="telegram") finally: release_thread_lock(thread.id) From a57b741d5ef30f49d373f843b40b92c9a25c0686 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 28 May 2026 04:37:27 +0530 Subject: [PATCH 14/63] refactor(gateway): rename persistence models to external chat --- .../versions/144_add_gateway_tables.py | 278 +++++++++--------- surfsense_backend/app/db.py | 211 ++++++------- surfsense_backend/app/gateway/accounts.py | 36 ++- .../app/gateway/auth_invariant.py | 10 +- surfsense_backend/app/gateway/bindings.py | 30 +- surfsense_backend/app/gateway/inbox.py | 10 +- surfsense_backend/app/gateway/pairing.py | 16 +- .../tests/unit/gateway/test_pairing.py | 6 +- 8 files changed, 274 insertions(+), 323 deletions(-) diff --git a/surfsense_backend/alembic/versions/144_add_gateway_tables.py b/surfsense_backend/alembic/versions/144_add_gateway_tables.py index 011333d69..35eb662c8 100644 --- a/surfsense_backend/alembic/versions/144_add_gateway_tables.py +++ b/surfsense_backend/alembic/versions/144_add_gateway_tables.py @@ -1,20 +1,21 @@ -"""add gateway tables for Telegram messaging gateway +"""add external chat surface tables Revision ID: 144 Revises: 143 Create Date: 2026-05-27 -Adds the lean v6 gateway schema: +Adds the lean external chat surface schema: -* gateway_platform_accounts -* gateway_conversation_bindings -* gateway_inbound_events +* external_chat_accounts +* external_chat_bindings +* external_chat_inbound_events -The gateway stores Telegram-originated conversations in the existing chat -tables but keeps them out of UI replication. This migration adds ``source`` to -``new_chat_messages`` as a denormalized Zero publication boundary and publishes -only ``source = 'web'`` rows. Gateway control-plane tables are served through -REST in v1, so they are intentionally not added to ``zero_publication``. +External chat surfaces store Telegram-originated conversations in the existing +chat tables. This migration adds ``source`` to ``new_chat_threads`` and +``new_chat_messages`` as UI metadata while publishing all chat-message sources +through Zero so a future SurfSense UI layer can render external chats. External +chat adapter tables are served through REST in v1, so they are intentionally not +added to ``zero_publication``. """ from __future__ import annotations @@ -146,7 +147,7 @@ def _build_set_table_ddl( f"documents ({_cols(doc_cols)}), " f"folders, " f"search_source_connectors, " - f"new_chat_messages WHERE (source = 'web'), " + f"new_chat_messages, " f"chat_comments, " f"chat_session_state, " f'"user" ({_cols(user_cols)})' @@ -161,53 +162,41 @@ def _create_enum(name: str, values: tuple[str, ...]) -> postgresql.ENUM: def upgrade() -> None: conn = op.get_bind() - gateway_platform_enum = _create_enum( - "gateway_platform", ("telegram", "whatsapp", "signal") + external_chat_platform_enum = _create_enum( + "external_chat_platform", ("telegram", "whatsapp", "signal") ) - gateway_account_mode_enum = _create_enum( - "gateway_account_mode", ("cloud_shared", "self_host_byo") + external_chat_account_mode_enum = _create_enum( + "external_chat_account_mode", ("cloud_shared", "self_host_byo") ) - gateway_health_status_enum = _create_enum( - "gateway_health_status", ("unknown", "ok", "failing") + external_chat_health_status_enum = _create_enum( + "external_chat_health_status", ("unknown", "ok", "failing") ) - gateway_binding_state_enum = _create_enum( - "gateway_binding_state", ("pending", "bound", "revoked", "suspended") + external_chat_binding_state_enum = _create_enum( + "external_chat_binding_state", ("pending", "bound", "revoked", "suspended") ) - gateway_peer_kind_enum = _create_enum( - "gateway_peer_kind", ("direct", "group", "channel", "unknown") + external_chat_peer_kind_enum = _create_enum( + "external_chat_peer_kind", ("direct", "group", "channel", "unknown") ) - gateway_session_scope_enum = _create_enum( - "gateway_session_scope", - ("per_binding", "per_user_search_space", "ephemeral"), + external_chat_event_kind_enum = _create_enum( + "external_chat_event_kind", ("message", "edited_message", "callback_query", "other") ) - gateway_dm_policy_enum = _create_enum("gateway_dm_policy", ("enabled", "disabled")) - gateway_group_policy_enum = _create_enum( - "gateway_group_policy", ("disabled", "allowlist", "mention_required") - ) - gateway_event_kind_enum = _create_enum( - "gateway_event_kind", ("message", "edited_message", "callback_query", "other") - ) - gateway_event_status_enum = _create_enum( - "gateway_event_status", + external_chat_event_status_enum = _create_enum( + "external_chat_event_status", ("received", "processing", "processed", "ignored", "failed"), ) - if not _table_exists(conn, "gateway_platform_accounts"): + if not _table_exists(conn, "external_chat_accounts"): op.create_table( - "gateway_platform_accounts", + "external_chat_accounts", sa.Column("id", sa.BigInteger(), primary_key=True), - sa.Column("platform", gateway_platform_enum, nullable=False), - sa.Column("mode", gateway_account_mode_enum, nullable=False), + sa.Column("platform", external_chat_platform_enum, nullable=False), + sa.Column("mode", external_chat_account_mode_enum, nullable=False), sa.Column("owner_user_id", postgresql.UUID(as_uuid=True), nullable=True), sa.Column("owner_search_space_id", sa.Integer(), nullable=True), sa.Column("is_system_account", sa.Boolean(), nullable=False, server_default="false"), sa.Column("encrypted_credentials", sa.Text(), nullable=True), - sa.Column( - "account_metadata", - postgresql.JSONB(astext_type=sa.Text()), - nullable=False, - server_default=sa.text("'{}'::jsonb"), - ), + sa.Column("bot_username", sa.String(255), nullable=True), + sa.Column("webhook_secret", sa.String(64), nullable=True), sa.Column( "cursor_state", postgresql.JSONB(astext_type=sa.Text()), @@ -216,7 +205,7 @@ def upgrade() -> None: ), sa.Column( "health_status", - gateway_health_status_enum, + external_chat_health_status_enum, nullable=False, server_default="unknown", ), @@ -238,7 +227,7 @@ def upgrade() -> None: sa.CheckConstraint( "(is_system_account = true AND owner_user_id IS NULL) OR " "(is_system_account = false AND owner_user_id IS NOT NULL)", - name="ck_gateway_accounts_owner_shape", + name="ck_external_chat_accounts_owner_shape", ), sa.ForeignKeyConstraint(["owner_user_id"], ["user.id"], ondelete="CASCADE"), sa.ForeignKeyConstraint( @@ -246,32 +235,40 @@ def upgrade() -> None: ), ) op.create_index( - "uq_gateway_accounts_owner_platform", - "gateway_platform_accounts", + "uq_external_chat_accounts_owner_platform", + "external_chat_accounts", ["owner_user_id", "platform"], unique=True, postgresql_where=sa.text("is_system_account = false"), if_not_exists=True, ) op.create_index( - "uq_gateway_accounts_system_platform", - "gateway_platform_accounts", + "uq_external_chat_accounts_system_platform", + "external_chat_accounts", ["platform"], unique=True, postgresql_where=sa.text("is_system_account = true"), if_not_exists=True, ) + op.create_index( + "uq_external_chat_accounts_webhook_secret", + "external_chat_accounts", + ["webhook_secret"], + unique=True, + postgresql_where=sa.text("webhook_secret IS NOT NULL"), + if_not_exists=True, + ) - if not _table_exists(conn, "gateway_conversation_bindings"): + if not _table_exists(conn, "external_chat_bindings"): op.create_table( - "gateway_conversation_bindings", + "external_chat_bindings", sa.Column("id", sa.BigInteger(), primary_key=True), sa.Column("account_id", sa.BigInteger(), nullable=False), sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("search_space_id", sa.Integer(), nullable=False), sa.Column( "state", - gateway_binding_state_enum, + external_chat_binding_state_enum, nullable=False, server_default="pending", ), @@ -280,39 +277,25 @@ def upgrade() -> None: sa.Column("external_peer_id", sa.Text(), nullable=True), sa.Column( "external_peer_kind", - gateway_peer_kind_enum, + external_chat_peer_kind_enum, nullable=False, server_default="unknown", ), - sa.Column("external_thread_id", sa.Text(), nullable=True), + sa.Column( + "external_thread_id", + sa.Text(), + nullable=True, + comment="Reserved for Telegram message_thread_id when group/forum support lands.", + ), sa.Column("external_display_name", sa.Text(), nullable=True), sa.Column("external_username", sa.Text(), nullable=True), - sa.Column("external_pii_hashes", postgresql.JSONB(astext_type=sa.Text()), nullable=True), sa.Column( "external_metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False, server_default=sa.text("'{}'::jsonb"), ), - sa.Column("active_thread_id", sa.Integer(), nullable=True), - sa.Column( - "session_scope", - gateway_session_scope_enum, - nullable=False, - server_default="per_binding", - ), - sa.Column( - "dm_policy", - gateway_dm_policy_enum, - nullable=False, - server_default="enabled", - ), - sa.Column( - "group_policy", - gateway_group_policy_enum, - nullable=False, - server_default="disabled", - ), + sa.Column("new_chat_thread_id", sa.Integer(), nullable=True), sa.Column("revoked_at", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column("suspended_at", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column("suspended_reason", sa.Text(), nullable=True), @@ -329,17 +312,17 @@ def upgrade() -> None: server_default=sa.text("(now() AT TIME ZONE 'utc')"), ), sa.ForeignKeyConstraint( - ["account_id"], ["gateway_platform_accounts.id"], ondelete="CASCADE" + ["account_id"], ["external_chat_accounts.id"], ondelete="CASCADE" ), sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"), sa.ForeignKeyConstraint(["search_space_id"], ["searchspaces.id"], ondelete="CASCADE"), sa.ForeignKeyConstraint( - ["active_thread_id"], ["new_chat_threads.id"], ondelete="SET NULL" + ["new_chat_thread_id"], ["new_chat_threads.id"], ondelete="SET NULL" ), ) op.create_index( - "uq_gateway_bindings_account_peer_active", - "gateway_conversation_bindings", + "uq_external_chat_bindings_account_peer_active", + "external_chat_bindings", ["account_id", "external_peer_id"], unique=True, postgresql_where=sa.text( @@ -348,51 +331,46 @@ def upgrade() -> None: if_not_exists=True, ) op.create_index( - "uq_gateway_bindings_pairing_code_pending", - "gateway_conversation_bindings", + "uq_external_chat_bindings_pairing_code_pending", + "external_chat_bindings", ["pairing_code"], unique=True, postgresql_where=sa.text("state = 'pending'"), if_not_exists=True, ) op.create_index( - "ix_gateway_bindings_user_state", - "gateway_conversation_bindings", + "ix_external_chat_bindings_user_state", + "external_chat_bindings", ["user_id", "state"], if_not_exists=True, ) op.create_index( - "ix_gateway_bindings_search_space_state", - "gateway_conversation_bindings", + "ix_external_chat_bindings_search_space_state", + "external_chat_bindings", ["search_space_id", "state"], if_not_exists=True, ) - if not _table_exists(conn, "gateway_inbound_events"): + if not _table_exists(conn, "external_chat_inbound_events"): op.create_table( - "gateway_inbound_events", + "external_chat_inbound_events", sa.Column("id", sa.BigInteger(), primary_key=True), sa.Column("account_id", sa.BigInteger(), nullable=False), - sa.Column("binding_id", sa.BigInteger(), nullable=True), - sa.Column("platform", gateway_platform_enum, nullable=False), + sa.Column("external_chat_binding_id", sa.BigInteger(), nullable=True), + sa.Column("platform", external_chat_platform_enum, nullable=False), sa.Column("event_dedupe_key", sa.Text(), nullable=False), sa.Column("external_event_id", sa.Text(), nullable=True), sa.Column("external_message_id", sa.Text(), nullable=True), - sa.Column("event_kind", gateway_event_kind_enum, nullable=False), + sa.Column("event_kind", external_chat_event_kind_enum, nullable=False), sa.Column( "raw_payload", postgresql.JSONB(astext_type=sa.Text()), nullable=True, ), - sa.Column( - "processing_metadata", - postgresql.JSONB(astext_type=sa.Text()), - nullable=False, - server_default=sa.text("'{}'::jsonb"), - ), + sa.Column("request_id", sa.String(64), nullable=True), sa.Column( "status", - gateway_event_status_enum, + external_chat_event_status_enum, nullable=False, server_default="received", ), @@ -412,27 +390,34 @@ def upgrade() -> None: server_default=sa.text("(now() AT TIME ZONE 'utc')"), ), sa.ForeignKeyConstraint( - ["account_id"], ["gateway_platform_accounts.id"], ondelete="CASCADE" + ["account_id"], ["external_chat_accounts.id"], ondelete="CASCADE" ), sa.ForeignKeyConstraint( - ["binding_id"], ["gateway_conversation_bindings.id"], ondelete="SET NULL" + ["external_chat_binding_id"], ["external_chat_bindings.id"], ondelete="SET NULL" ), sa.UniqueConstraint( "account_id", "event_dedupe_key", - name="uq_gateway_inbound_account_dedupe_key", + name="uq_external_chat_inbound_account_dedupe_key", ), ) op.create_index( - "ix_gateway_inbound_status_received_at", - "gateway_inbound_events", + "ix_external_chat_inbound_status_received_at", + "external_chat_inbound_events", ["status", "received_at"], if_not_exists=True, ) op.create_index( - "ix_gateway_inbound_binding_received_at", - "gateway_inbound_events", - ["binding_id", "received_at"], + "ix_external_chat_inbound_binding_received_at", + "external_chat_inbound_events", + ["external_chat_binding_id", "received_at"], + if_not_exists=True, + ) + op.create_index( + "ix_external_chat_inbound_request_id", + "external_chat_inbound_events", + ["request_id"], + postgresql_where=sa.text("request_id IS NOT NULL"), if_not_exists=True, ) @@ -442,27 +427,27 @@ def upgrade() -> None: sa.Column("source", sa.Text(), nullable=False, server_default="web"), ) op.alter_column("new_chat_threads", "source", type_=sa.Text()) - if not _column_exists(conn, "new_chat_threads", "binding_id"): + if not _column_exists(conn, "new_chat_threads", "external_chat_binding_id"): op.add_column( "new_chat_threads", - sa.Column("binding_id", sa.BigInteger(), nullable=True), + sa.Column("external_chat_binding_id", sa.BigInteger(), nullable=True), ) if not _constraint_exists( - conn, "new_chat_threads", "fk_new_chat_threads_gateway_binding_id" + conn, "new_chat_threads", "fk_new_chat_threads_external_chat_external_chat_binding_id" ): op.create_foreign_key( - "fk_new_chat_threads_gateway_binding_id", + "fk_new_chat_threads_external_chat_external_chat_binding_id", "new_chat_threads", - "gateway_conversation_bindings", - ["binding_id"], + "external_chat_bindings", + ["external_chat_binding_id"], ["id"], ondelete="SET NULL", ) op.create_index("ix_new_chat_threads_source", "new_chat_threads", ["source"], if_not_exists=True) op.create_index( - "ix_new_chat_threads_binding_id", + "ix_new_chat_threads_external_chat_binding_id", "new_chat_threads", - ["binding_id"], + ["external_chat_binding_id"], if_not_exists=True, ) @@ -510,7 +495,9 @@ def upgrade() -> None: tx = conn.begin_nested() if conn.in_transaction() else conn.begin() with tx: conn.execute( - sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-144-gateway'") + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-144-external-chat'" + ) ) conn.execute( sa.text( @@ -521,7 +508,9 @@ def upgrade() -> None: ) ) conn.execute( - sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-144-gateway'") + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-144-external-chat'" + ) ) @@ -565,59 +554,58 @@ def downgrade() -> None: _drop_column_if_exists("new_chat_messages", "platform_metadata") _drop_column_if_exists("new_chat_messages", "source") - _drop_index_if_exists("ix_new_chat_threads_binding_id", "new_chat_threads") + _drop_index_if_exists("ix_new_chat_threads_external_chat_binding_id", "new_chat_threads") _drop_index_if_exists("ix_new_chat_threads_source", "new_chat_threads") if _constraint_exists( - conn, "new_chat_threads", "fk_new_chat_threads_gateway_binding_id" + conn, "new_chat_threads", "fk_new_chat_threads_external_chat_external_chat_binding_id" ): op.drop_constraint( - "fk_new_chat_threads_gateway_binding_id", + "fk_new_chat_threads_external_chat_external_chat_binding_id", "new_chat_threads", type_="foreignkey", ) - _drop_column_if_exists("new_chat_threads", "binding_id") + _drop_column_if_exists("new_chat_threads", "external_chat_binding_id") _drop_column_if_exists("new_chat_threads", "source") _drop_index_if_exists( - "ix_gateway_inbound_binding_received_at", "gateway_inbound_events" + "ix_external_chat_inbound_binding_received_at", "external_chat_inbound_events" ) - _drop_index_if_exists("ix_gateway_inbound_status_received_at", "gateway_inbound_events") - if _table_exists(conn, "gateway_inbound_events"): - op.drop_table("gateway_inbound_events") + _drop_index_if_exists("ix_external_chat_inbound_request_id", "external_chat_inbound_events") + _drop_index_if_exists("ix_external_chat_inbound_status_received_at", "external_chat_inbound_events") + if _table_exists(conn, "external_chat_inbound_events"): + op.drop_table("external_chat_inbound_events") _drop_index_if_exists( - "ix_gateway_bindings_search_space_state", - "gateway_conversation_bindings", + "ix_external_chat_bindings_search_space_state", + "external_chat_bindings", ) _drop_index_if_exists( - "ix_gateway_bindings_user_state", "gateway_conversation_bindings" + "ix_external_chat_bindings_user_state", "external_chat_bindings" ) _drop_index_if_exists( - "uq_gateway_bindings_pairing_code_pending", - "gateway_conversation_bindings", + "uq_external_chat_bindings_pairing_code_pending", + "external_chat_bindings", ) _drop_index_if_exists( - "uq_gateway_bindings_account_peer_active", - "gateway_conversation_bindings", + "uq_external_chat_bindings_account_peer_active", + "external_chat_bindings", ) - if _table_exists(conn, "gateway_conversation_bindings"): - op.drop_table("gateway_conversation_bindings") + if _table_exists(conn, "external_chat_bindings"): + op.drop_table("external_chat_bindings") - _drop_index_if_exists("uq_gateway_accounts_system_platform", "gateway_platform_accounts") - _drop_index_if_exists("uq_gateway_accounts_owner_platform", "gateway_platform_accounts") - if _table_exists(conn, "gateway_platform_accounts"): - op.drop_table("gateway_platform_accounts") + _drop_index_if_exists("uq_external_chat_accounts_system_platform", "external_chat_accounts") + _drop_index_if_exists("uq_external_chat_accounts_owner_platform", "external_chat_accounts") + _drop_index_if_exists("uq_external_chat_accounts_webhook_secret", "external_chat_accounts") + if _table_exists(conn, "external_chat_accounts"): + op.drop_table("external_chat_accounts") for enum_name in ( - "gateway_event_status", - "gateway_event_kind", - "gateway_group_policy", - "gateway_dm_policy", - "gateway_session_scope", - "gateway_peer_kind", - "gateway_binding_state", - "gateway_health_status", - "gateway_account_mode", - "gateway_platform", + "external_chat_event_status", + "external_chat_event_kind", + "external_chat_peer_kind", + "external_chat_binding_state", + "external_chat_health_status", + "external_chat_account_mode", + "external_chat_platform", ): postgresql.ENUM(name=enum_name).drop(conn, checkfirst=True) diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 82b641ca6..de7792627 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -574,62 +574,45 @@ class ChatVisibility(StrEnum): # PUBLIC = "PUBLIC" # Reserved for future implementation -class GatewayPlatform(StrEnum): +class ExternalChatPlatform(StrEnum): TELEGRAM = "telegram" WHATSAPP = "whatsapp" SIGNAL = "signal" -class GatewayAccountMode(StrEnum): +class ExternalChatAccountMode(StrEnum): CLOUD_SHARED = "cloud_shared" SELF_HOST_BYO = "self_host_byo" -class GatewayHealthStatus(StrEnum): +class ExternalChatHealthStatus(StrEnum): UNKNOWN = "unknown" OK = "ok" FAILING = "failing" -class GatewayBindingState(StrEnum): +class ExternalChatBindingState(StrEnum): PENDING = "pending" BOUND = "bound" REVOKED = "revoked" SUSPENDED = "suspended" -class GatewayPeerKind(StrEnum): +class ExternalChatPeerKind(StrEnum): DIRECT = "direct" GROUP = "group" CHANNEL = "channel" UNKNOWN = "unknown" -class GatewaySessionScope(StrEnum): - PER_BINDING = "per_binding" - PER_USER_SEARCH_SPACE = "per_user_search_space" - EPHEMERAL = "ephemeral" - - -class GatewayDmPolicy(StrEnum): - ENABLED = "enabled" - DISABLED = "disabled" - - -class GatewayGroupPolicy(StrEnum): - DISABLED = "disabled" - ALLOWLIST = "allowlist" - MENTION_REQUIRED = "mention_required" - - -class GatewayEventKind(StrEnum): +class ExternalChatEventKind(StrEnum): MESSAGE = "message" EDITED_MESSAGE = "edited_message" CALLBACK_QUERY = "callback_query" OTHER = "other" -class GatewayEventStatus(StrEnum): +class ExternalChatEventStatus(StrEnum): RECEIVED = "received" PROCESSING = "processing" PROCESSED = "processed" @@ -713,12 +696,12 @@ class NewChatThread(BaseModel, TimestampMixin): # agent_llm_id changes). Unindexed: all reads are by primary key. pinned_llm_config_id = Column(Integer, nullable=True) - # Gateway-originated threads are persisted for the agent, but the UI Zero - # publication only exposes ``source='web'`` rows. + # Surface metadata for web and external chat threads. Zero publishes all + # chat-message sources; the UI can decide which surfaces to render. source = Column(Text, nullable=False, default="web", server_default="web") - binding_id = Column( + external_chat_binding_id = Column( BigInteger, - ForeignKey("gateway_conversation_bindings.id", ondelete="SET NULL"), + ForeignKey("external_chat_bindings.id", ondelete="SET NULL"), nullable=True, index=True, ) @@ -743,9 +726,9 @@ class NewChatThread(BaseModel, TimestampMixin): back_populates="thread", cascade="all, delete-orphan", ) - gateway_binding = relationship( - "GatewayConversationBinding", - foreign_keys=[binding_id], + external_chat_binding = relationship( + "ExternalChatBinding", + foreign_keys=[external_chat_binding_id], back_populates="threads", ) @@ -822,23 +805,23 @@ class NewChatMessage(BaseModel, TimestampMixin): ) -class GatewayPlatformAccount(Base, TimestampMixin): - __tablename__ = "gateway_platform_accounts" +class ExternalChatAccount(Base, TimestampMixin): + __tablename__ = "external_chat_accounts" __allow_unmapped__ = True id = Column(BigInteger, primary_key=True, index=True) platform = Column( SQLAlchemyEnum( - GatewayPlatform, - name="gateway_platform", + ExternalChatPlatform, + name="external_chat_platform", values_callable=_enum_values, ), nullable=False, ) mode = Column( SQLAlchemyEnum( - GatewayAccountMode, - name="gateway_account_mode", + ExternalChatAccountMode, + name="external_chat_account_mode", values_callable=_enum_values, ), nullable=False, @@ -851,17 +834,18 @@ class GatewayPlatformAccount(Base, TimestampMixin): ) is_system_account = Column(Boolean, nullable=False, default=False, server_default="false") encrypted_credentials = Column(Text, nullable=True) - account_metadata = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")) + bot_username = Column(String(255), nullable=True) + webhook_secret = Column(String(64), nullable=True) cursor_state = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")) health_status = Column( SQLAlchemyEnum( - GatewayHealthStatus, - name="gateway_health_status", + ExternalChatHealthStatus, + name="external_chat_health_status", values_callable=_enum_values, ), nullable=False, - default=GatewayHealthStatus.UNKNOWN, - server_default=GatewayHealthStatus.UNKNOWN.value, + default=ExternalChatHealthStatus.UNKNOWN, + server_default=ExternalChatHealthStatus.UNKNOWN.value, ) last_health_check_at = Column(TIMESTAMP(timezone=True), nullable=True) suspended_at = Column(TIMESTAMP(timezone=True), nullable=True) @@ -877,12 +861,12 @@ class GatewayPlatformAccount(Base, TimestampMixin): owner = relationship("User", foreign_keys=[owner_user_id]) owner_search_space = relationship("SearchSpace", foreign_keys=[owner_search_space_id]) bindings = relationship( - "GatewayConversationBinding", + "ExternalChatBinding", back_populates="account", cascade="all, delete-orphan", ) inbound_events = relationship( - "GatewayInboundEvent", + "ExternalChatInboundEvent", back_populates="account", cascade="all, delete-orphan", ) @@ -891,32 +875,38 @@ class GatewayPlatformAccount(Base, TimestampMixin): CheckConstraint( "(is_system_account = true AND owner_user_id IS NULL) OR " "(is_system_account = false AND owner_user_id IS NOT NULL)", - name="ck_gateway_accounts_owner_shape", + name="ck_external_chat_accounts_owner_shape", ), Index( - "uq_gateway_accounts_owner_platform", + "uq_external_chat_accounts_owner_platform", "owner_user_id", "platform", unique=True, postgresql_where=text("is_system_account = false"), ), Index( - "uq_gateway_accounts_system_platform", + "uq_external_chat_accounts_system_platform", "platform", unique=True, postgresql_where=text("is_system_account = true"), ), + Index( + "uq_external_chat_accounts_webhook_secret", + "webhook_secret", + unique=True, + postgresql_where=text("webhook_secret IS NOT NULL"), + ), ) -class GatewayConversationBinding(Base, TimestampMixin): - __tablename__ = "gateway_conversation_bindings" +class ExternalChatBinding(Base, TimestampMixin): + __tablename__ = "external_chat_bindings" __allow_unmapped__ = True id = Column(BigInteger, primary_key=True, index=True) account_id = Column( BigInteger, - ForeignKey("gateway_platform_accounts.id", ondelete="CASCADE"), + ForeignKey("external_chat_accounts.id", ondelete="CASCADE"), nullable=False, index=True, ) @@ -928,68 +918,37 @@ class GatewayConversationBinding(Base, TimestampMixin): ) state = Column( SQLAlchemyEnum( - GatewayBindingState, - name="gateway_binding_state", + ExternalChatBindingState, + name="external_chat_binding_state", values_callable=_enum_values, ), nullable=False, - default=GatewayBindingState.PENDING, - server_default=GatewayBindingState.PENDING.value, + default=ExternalChatBindingState.PENDING, + server_default=ExternalChatBindingState.PENDING.value, ) pairing_code = Column(Text, nullable=True) pairing_code_expires_at = Column(TIMESTAMP(timezone=True), nullable=True) external_peer_id = Column(Text, nullable=True) external_peer_kind = Column( SQLAlchemyEnum( - GatewayPeerKind, - name="gateway_peer_kind", + ExternalChatPeerKind, + name="external_chat_peer_kind", values_callable=_enum_values, ), nullable=False, - default=GatewayPeerKind.UNKNOWN, - server_default=GatewayPeerKind.UNKNOWN.value, + default=ExternalChatPeerKind.UNKNOWN, + server_default=ExternalChatPeerKind.UNKNOWN.value, ) external_thread_id = Column(Text, nullable=True) external_display_name = Column(Text, nullable=True) external_username = Column(Text, nullable=True) - external_pii_hashes = Column(JSONB, nullable=True) external_metadata = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb")) - active_thread_id = Column( + new_chat_thread_id = Column( Integer, ForeignKey("new_chat_threads.id", ondelete="SET NULL"), nullable=True, index=True, ) - session_scope = Column( - SQLAlchemyEnum( - GatewaySessionScope, - name="gateway_session_scope", - values_callable=_enum_values, - ), - nullable=False, - default=GatewaySessionScope.PER_BINDING, - server_default=GatewaySessionScope.PER_BINDING.value, - ) - dm_policy = Column( - SQLAlchemyEnum( - GatewayDmPolicy, - name="gateway_dm_policy", - values_callable=_enum_values, - ), - nullable=False, - default=GatewayDmPolicy.ENABLED, - server_default=GatewayDmPolicy.ENABLED.value, - ) - group_policy = Column( - SQLAlchemyEnum( - GatewayGroupPolicy, - name="gateway_group_policy", - values_callable=_enum_values, - ), - nullable=False, - default=GatewayGroupPolicy.DISABLED, - server_default=GatewayGroupPolicy.DISABLED.value, - ) revoked_at = Column(TIMESTAMP(timezone=True), nullable=True) suspended_at = Column(TIMESTAMP(timezone=True), nullable=True) suspended_reason = Column(Text, nullable=True) @@ -1001,24 +960,24 @@ class GatewayConversationBinding(Base, TimestampMixin): server_default=text("(now() AT TIME ZONE 'utc')"), ) - account = relationship("GatewayPlatformAccount", back_populates="bindings") + account = relationship("ExternalChatAccount", back_populates="bindings") user = relationship("User", foreign_keys=[user_id]) search_space = relationship("SearchSpace", foreign_keys=[search_space_id]) - active_thread = relationship("NewChatThread", foreign_keys=[active_thread_id]) + new_chat_thread = relationship("NewChatThread", foreign_keys=[new_chat_thread_id]) threads = relationship( "NewChatThread", - back_populates="gateway_binding", - foreign_keys="NewChatThread.binding_id", + back_populates="external_chat_binding", + foreign_keys="NewChatThread.external_chat_binding_id", ) inbound_events = relationship( - "GatewayInboundEvent", + "ExternalChatInboundEvent", back_populates="binding", - foreign_keys="GatewayInboundEvent.binding_id", + foreign_keys="ExternalChatInboundEvent.external_chat_binding_id", ) __table_args__ = ( Index( - "uq_gateway_bindings_account_peer_active", + "uq_external_chat_bindings_account_peer_active", "account_id", "external_peer_id", unique=True, @@ -1027,37 +986,37 @@ class GatewayConversationBinding(Base, TimestampMixin): ), ), Index( - "uq_gateway_bindings_pairing_code_pending", + "uq_external_chat_bindings_pairing_code_pending", "pairing_code", unique=True, postgresql_where=text("state = 'pending'"), ), - Index("ix_gateway_bindings_user_state", "user_id", "state"), - Index("ix_gateway_bindings_search_space_state", "search_space_id", "state"), + Index("ix_external_chat_bindings_user_state", "user_id", "state"), + Index("ix_external_chat_bindings_search_space_state", "search_space_id", "state"), ) -class GatewayInboundEvent(Base, TimestampMixin): - __tablename__ = "gateway_inbound_events" +class ExternalChatInboundEvent(Base, TimestampMixin): + __tablename__ = "external_chat_inbound_events" __allow_unmapped__ = True id = Column(BigInteger, primary_key=True, index=True) account_id = Column( BigInteger, - ForeignKey("gateway_platform_accounts.id", ondelete="CASCADE"), + ForeignKey("external_chat_accounts.id", ondelete="CASCADE"), nullable=False, index=True, ) - binding_id = Column( + external_chat_binding_id = Column( BigInteger, - ForeignKey("gateway_conversation_bindings.id", ondelete="SET NULL"), + ForeignKey("external_chat_bindings.id", ondelete="SET NULL"), nullable=True, index=True, ) platform = Column( SQLAlchemyEnum( - GatewayPlatform, - name="gateway_platform", + ExternalChatPlatform, + name="external_chat_platform", values_callable=_enum_values, ), nullable=False, @@ -1067,28 +1026,23 @@ class GatewayInboundEvent(Base, TimestampMixin): external_message_id = Column(Text, nullable=True) event_kind = Column( SQLAlchemyEnum( - GatewayEventKind, - name="gateway_event_kind", + ExternalChatEventKind, + name="external_chat_event_kind", values_callable=_enum_values, ), nullable=False, ) raw_payload = Column(JSONB, nullable=True) - processing_metadata = Column( - JSONB, - nullable=False, - default=dict, - server_default=text("'{}'::jsonb"), - ) + request_id = Column(String(64), nullable=True) status = Column( SQLAlchemyEnum( - GatewayEventStatus, - name="gateway_event_status", + ExternalChatEventStatus, + name="external_chat_event_status", values_callable=_enum_values, ), nullable=False, - default=GatewayEventStatus.RECEIVED, - server_default=GatewayEventStatus.RECEIVED.value, + default=ExternalChatEventStatus.RECEIVED, + server_default=ExternalChatEventStatus.RECEIVED.value, ) attempt_count = Column(Integer, nullable=False, default=0, server_default="0") last_error = Column(Text, nullable=True) @@ -1100,17 +1054,26 @@ class GatewayInboundEvent(Base, TimestampMixin): ) processed_at = Column(TIMESTAMP(timezone=True), nullable=True) - account = relationship("GatewayPlatformAccount", back_populates="inbound_events") - binding = relationship("GatewayConversationBinding", back_populates="inbound_events") + account = relationship("ExternalChatAccount", back_populates="inbound_events") + binding = relationship("ExternalChatBinding", back_populates="inbound_events") __table_args__ = ( UniqueConstraint( "account_id", "event_dedupe_key", - name="uq_gateway_inbound_account_dedupe_key", + name="uq_external_chat_inbound_account_dedupe_key", + ), + Index("ix_external_chat_inbound_status_received_at", "status", "received_at"), + Index( + "ix_external_chat_inbound_binding_received_at", + "external_chat_binding_id", + "received_at", + ), + Index( + "ix_external_chat_inbound_request_id", + "request_id", + postgresql_where=text("request_id IS NOT NULL"), ), - Index("ix_gateway_inbound_status_received_at", "status", "received_at"), - Index("ix_gateway_inbound_binding_received_at", "binding_id", "received_at"), ) diff --git a/surfsense_backend/app/gateway/accounts.py b/surfsense_backend/app/gateway/accounts.py index 727d616c1..3e0d86e46 100644 --- a/surfsense_backend/app/gateway/accounts.py +++ b/surfsense_backend/app/gateway/accounts.py @@ -1,4 +1,4 @@ -"""Gateway account helpers.""" +"""External chat account helpers.""" from __future__ import annotations @@ -7,16 +7,16 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.config import config from app.db import ( - GatewayAccountMode, - GatewayHealthStatus, - GatewayPlatform, - GatewayPlatformAccount, + ExternalChatAccountMode, + ExternalChatHealthStatus, + ExternalChatPlatform, + ExternalChatAccount, ) from app.utils.oauth_security import TokenEncryption -def account_token(account: GatewayPlatformAccount) -> str | None: - if account.is_system_account and account.platform == GatewayPlatform.TELEGRAM: +def account_token(account: ExternalChatAccount) -> str | None: + if account.is_system_account and account.platform == ExternalChatPlatform.TELEGRAM: return config.TELEGRAM_SHARED_BOT_TOKEN if not account.encrypted_credentials: return None @@ -27,26 +27,24 @@ def account_token(account: GatewayPlatformAccount) -> str | None: async def get_or_create_system_telegram_account( session: AsyncSession, -) -> GatewayPlatformAccount: +) -> ExternalChatAccount: result = await session.execute( - select(GatewayPlatformAccount).where( - GatewayPlatformAccount.platform == GatewayPlatform.TELEGRAM, - GatewayPlatformAccount.is_system_account.is_(True), + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.TELEGRAM, + ExternalChatAccount.is_system_account.is_(True), ) ) account = result.scalars().first() if account is not None: return account - account = GatewayPlatformAccount( - platform=GatewayPlatform.TELEGRAM, - mode=GatewayAccountMode.CLOUD_SHARED, + account = ExternalChatAccount( + platform=ExternalChatPlatform.TELEGRAM, + mode=ExternalChatAccountMode.CLOUD_SHARED, is_system_account=True, - account_metadata={ - "bot_username": config.TELEGRAM_SHARED_BOT_USERNAME, - "webhook_secret": config.TELEGRAM_WEBHOOK_SECRET, - }, + bot_username=config.TELEGRAM_SHARED_BOT_USERNAME, + webhook_secret=config.TELEGRAM_WEBHOOK_SECRET, cursor_state={}, - health_status=GatewayHealthStatus.UNKNOWN, + health_status=ExternalChatHealthStatus.UNKNOWN, ) session.add(account) await session.flush() diff --git a/surfsense_backend/app/gateway/auth_invariant.py b/surfsense_backend/app/gateway/auth_invariant.py index 414c69c5c..fba38f64e 100644 --- a/surfsense_backend/app/gateway/auth_invariant.py +++ b/surfsense_backend/app/gateway/auth_invariant.py @@ -1,11 +1,11 @@ -"""Authorization invariants for gateway-routed turns.""" +"""Authorization invariants for external-chat-routed turns.""" from __future__ import annotations from fastapi import HTTPException from sqlalchemy.ext.asyncio import AsyncSession -from app.db import GatewayConversationBinding, Permission, User +from app.db import ExternalChatBinding, Permission, User from app.gateway.bindings import suspend_binding from app.observability.metrics import record_gateway_auth_invariant_failure from app.utils.rbac import check_permission, check_search_space_access @@ -19,7 +19,7 @@ class GatewaySuspendedError(RuntimeError): async def _fail( session: AsyncSession, - binding: GatewayConversationBinding, + binding: ExternalChatBinding, reason: str, ) -> None: suspend_binding(binding, reason) @@ -30,7 +30,7 @@ async def _fail( async def assert_authorization_invariant( session: AsyncSession, - binding: GatewayConversationBinding, + binding: ExternalChatBinding, ) -> User: if binding.state != "bound": await _fail(session, binding, "binding_not_bound") @@ -46,7 +46,7 @@ async def assert_authorization_invariant( user, binding.search_space_id, Permission.CHATS_CREATE.value, - "Gateway owner no longer has permission to chat in this search space", + "External chat owner no longer has permission to chat in this search space", ) except HTTPException as exc: await _fail(session, binding, f"rbac_{exc.status_code}") diff --git a/surfsense_backend/app/gateway/bindings.py b/surfsense_backend/app/gateway/bindings.py index 6f2b641f7..e7205c5f1 100644 --- a/surfsense_backend/app/gateway/bindings.py +++ b/surfsense_backend/app/gateway/bindings.py @@ -1,4 +1,4 @@ -"""Gateway binding helpers.""" +"""External chat binding helpers.""" from __future__ import annotations @@ -9,19 +9,19 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.db import ( ChatVisibility, - GatewayBindingState, - GatewayConversationBinding, + ExternalChatBindingState, + ExternalChatBinding, NewChatThread, ) async def get_or_create_thread_for_binding( session: AsyncSession, - binding: GatewayConversationBinding, + binding: ExternalChatBinding, ) -> NewChatThread: - if binding.active_thread_id is not None: + if binding.new_chat_thread_id is not None: result = await session.execute( - select(NewChatThread).where(NewChatThread.id == binding.active_thread_id) + select(NewChatThread).where(NewChatThread.id == binding.new_chat_thread_id) ) thread = result.scalars().first() if thread is not None and not thread.archived: @@ -33,30 +33,30 @@ async def get_or_create_thread_for_binding( created_by_id=binding.user_id, visibility=ChatVisibility.PRIVATE, source="telegram", - binding_id=binding.id, + external_chat_binding_id=binding.id, ) session.add(thread) await session.flush() - binding.active_thread_id = thread.id + binding.new_chat_thread_id = thread.id return thread -def suspend_binding(binding: GatewayConversationBinding, reason: str) -> None: +def suspend_binding(binding: ExternalChatBinding, reason: str) -> None: now = datetime.now(UTC) - binding.state = GatewayBindingState.SUSPENDED + binding.state = ExternalChatBindingState.SUSPENDED binding.suspended_at = now binding.suspended_reason = reason -def revoke_binding(binding: GatewayConversationBinding) -> None: +def revoke_binding(binding: ExternalChatBinding) -> None: now = datetime.now(UTC) - binding.state = GatewayBindingState.REVOKED + binding.state = ExternalChatBindingState.REVOKED binding.revoked_at = now - binding.active_thread_id = None + binding.new_chat_thread_id = None -def resume_binding(binding: GatewayConversationBinding) -> None: - binding.state = GatewayBindingState.BOUND +def resume_binding(binding: ExternalChatBinding) -> None: + binding.state = ExternalChatBindingState.BOUND binding.suspended_at = None binding.suspended_reason = None diff --git a/surfsense_backend/app/gateway/inbox.py b/surfsense_backend/app/gateway/inbox.py index c98ee5977..9bc660b9d 100644 --- a/surfsense_backend/app/gateway/inbox.py +++ b/surfsense_backend/app/gateway/inbox.py @@ -5,7 +5,7 @@ from __future__ import annotations from sqlalchemy.dialects.postgresql import insert from sqlalchemy.ext.asyncio import AsyncSession -from app.db import GatewayInboundEvent, GatewayPlatform +from app.db import ExternalChatInboundEvent, ExternalChatPlatform def telegram_event_dedupe_key(update_id: int | str) -> str: @@ -16,15 +16,16 @@ async def persist_inbound_event( session: AsyncSession, *, account_id: int, - platform: GatewayPlatform, + platform: ExternalChatPlatform, event_dedupe_key: str, event_kind: str, raw_payload: dict, external_event_id: str | None = None, external_message_id: str | None = None, + request_id: str | None = None, ) -> int | None: stmt = ( - insert(GatewayInboundEvent) + insert(ExternalChatInboundEvent) .values( account_id=account_id, platform=platform, @@ -33,11 +34,12 @@ async def persist_inbound_event( external_message_id=external_message_id, event_kind=event_kind, raw_payload=raw_payload, + request_id=request_id, ) .on_conflict_do_nothing( index_elements=["account_id", "event_dedupe_key"], ) - .returning(GatewayInboundEvent.id) + .returning(ExternalChatInboundEvent.id) ) result = await session.execute(stmt) return result.scalar_one_or_none() diff --git a/surfsense_backend/app/gateway/pairing.py b/surfsense_backend/app/gateway/pairing.py index 55232022e..7818bed12 100644 --- a/surfsense_backend/app/gateway/pairing.py +++ b/surfsense_backend/app/gateway/pairing.py @@ -1,4 +1,4 @@ -"""Pairing code lifecycle for gateway bindings.""" +"""Pairing code lifecycle for external chat bindings.""" from __future__ import annotations @@ -8,7 +8,7 @@ from datetime import UTC, datetime, timedelta from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.db import GatewayBindingState, GatewayConversationBinding +from app.db import ExternalChatBindingState, ExternalChatBinding PAIRING_CODE_TTL = timedelta(minutes=10) @@ -30,19 +30,19 @@ async def redeem_pairing_code( external_display_name: str | None, external_username: str | None, external_metadata: dict | None = None, -) -> GatewayConversationBinding | None: +) -> ExternalChatBinding | None: result = await session.execute( - select(GatewayConversationBinding).where( - GatewayConversationBinding.pairing_code == code, - GatewayConversationBinding.state == GatewayBindingState.PENDING, - GatewayConversationBinding.pairing_code_expires_at > datetime.now(UTC), + select(ExternalChatBinding).where( + ExternalChatBinding.pairing_code == code, + ExternalChatBinding.state == ExternalChatBindingState.PENDING, + ExternalChatBinding.pairing_code_expires_at > datetime.now(UTC), ) ) binding = result.scalars().first() if binding is None: return None - binding.state = GatewayBindingState.BOUND + binding.state = ExternalChatBindingState.BOUND binding.pairing_code = None binding.pairing_code_expires_at = None binding.external_peer_id = external_peer_id diff --git a/surfsense_backend/tests/unit/gateway/test_pairing.py b/surfsense_backend/tests/unit/gateway/test_pairing.py index c50bd6b7c..facf908cd 100644 --- a/surfsense_backend/tests/unit/gateway/test_pairing.py +++ b/surfsense_backend/tests/unit/gateway/test_pairing.py @@ -2,7 +2,7 @@ from datetime import UTC, datetime, timedelta import pytest -from app.db import GatewayBindingState +from app.db import ExternalChatBindingState from app.gateway.pairing import generate_pairing_code, redeem_pairing_code @@ -16,7 +16,7 @@ def test_generate_pairing_code_is_short_display_token(): @pytest.mark.asyncio async def test_redeem_pairing_code_binds_pending_row(mocker): binding = mocker.Mock() - binding.state = GatewayBindingState.PENDING + binding.state = ExternalChatBindingState.PENDING binding.pairing_code_expires_at = datetime.now(UTC) + timedelta(minutes=1) scalars = mocker.Mock() scalars.first.return_value = binding @@ -35,7 +35,7 @@ async def test_redeem_pairing_code_binds_pending_row(mocker): ) assert redeemed is binding - assert binding.state == GatewayBindingState.BOUND + assert binding.state == ExternalChatBindingState.BOUND assert binding.external_peer_id == "telegram:123" assert binding.pairing_code is None From 72024353f9f373a927e3f32ff13d51689be802d3 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 28 May 2026 04:37:41 +0530 Subject: [PATCH 15/63] fix(gateway): harden Telegram webhook intake --- .../app/routes/gateway_webhook_routes.py | 138 ++++++++-------- surfsense_backend/scripts/register_webhook.py | 11 +- .../tests/unit/gateway/test_webhook_routes.py | 149 ++++++++++++++++++ 3 files changed, 231 insertions(+), 67 deletions(-) create mode 100644 surfsense_backend/tests/unit/gateway/test_webhook_routes.py diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py index 86b84f067..f9b6acc93 100644 --- a/surfsense_backend/app/routes/gateway_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -2,6 +2,9 @@ from __future__ import annotations +import hmac +import logging +import uuid from datetime import UTC, datetime from typing import Any @@ -13,10 +16,10 @@ from starlette.responses import Response from app.config import config from app.db import ( - GatewayBindingState, - GatewayConversationBinding, - GatewayPlatform, - GatewayPlatformAccount, + ExternalChatBindingState, + ExternalChatBinding, + ExternalChatPlatform, + ExternalChatAccount, User, get_async_session, ) @@ -24,15 +27,18 @@ from app.gateway.accounts import get_or_create_system_telegram_account from app.gateway.bindings import resume_binding, revoke_binding from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key from app.gateway.pairing import generate_pairing_code, pairing_expires_at -from app.observability.metrics import record_gateway_inbox_write -from app.rate_limiter import limiter +from app.observability.metrics import ( + record_gateway_inbox_write, + record_gateway_webhook_parse_error, +) from app.users import current_active_user router = APIRouter(prefix="/gateway", tags=["gateway"]) +logger = logging.getLogger(__name__) class StartBindingRequest(BaseModel): - platform: GatewayPlatform = GatewayPlatform.TELEGRAM + platform: ExternalChatPlatform = ExternalChatPlatform.TELEGRAM search_space_id: int @@ -60,63 +66,63 @@ def _telegram_message(payload: dict[str, Any]) -> dict[str, Any] | None: async def _resolve_webhook_account( session: AsyncSession, *, - secret: str, + account_id: int, header_secret: str | None, -) -> GatewayPlatformAccount: - if config.TELEGRAM_WEBHOOK_SECRET and secret == config.TELEGRAM_WEBHOOK_SECRET: - if header_secret != config.TELEGRAM_WEBHOOK_SECRET: - raise HTTPException(status_code=403, detail="Invalid Telegram webhook secret") - return await get_or_create_system_telegram_account(session) - - result = await session.execute( - select(GatewayPlatformAccount).where( - GatewayPlatformAccount.platform == GatewayPlatform.TELEGRAM - ) - ) - for account in result.scalars(): - metadata = account.account_metadata or {} - webhook_secret = metadata.get("webhook_secret") - if webhook_secret and webhook_secret == secret: - if header_secret != webhook_secret: - raise HTTPException(status_code=403, detail="Invalid Telegram webhook secret") - return account - - raise HTTPException(status_code=404, detail="Gateway account not found") +) -> ExternalChatAccount: + account = await session.get(ExternalChatAccount, account_id) + if account is None or account.platform != ExternalChatPlatform.TELEGRAM: + raise HTTPException(status_code=404, detail="Gateway account not found") + expected_secret = account.webhook_secret or "" + if not expected_secret or not hmac.compare_digest(header_secret or "", expected_secret): + raise HTTPException(status_code=403, detail="Invalid Telegram webhook secret") + return account -@router.post("/webhooks/telegram/{secret}") -@limiter.limit("60/minute", key_func=lambda request: f"tg-webhook:{request.path_params['secret']}") +@router.post("/webhooks/telegram/{account_id}") async def telegram_webhook( request: Request, - secret: str, + account_id: int, session: AsyncSession = Depends(get_async_session), ) -> Response: - payload = await request.json() - account = await _resolve_webhook_account( - session, - secret=secret, - header_secret=request.headers.get("X-Telegram-Bot-Api-Secret-Token"), - ) - update_id = payload.get("update_id") - if update_id is None: + request_id = f"gateway_{uuid.uuid4().hex[:16]}" + try: + payload = await request.json() + except ValueError: + record_gateway_webhook_parse_error() return Response(status_code=200) - message = _telegram_message(payload) or {} - inbox_id = await persist_inbound_event( + account = await _resolve_webhook_account( session, - account_id=account.id, - platform=GatewayPlatform.TELEGRAM, - event_dedupe_key=telegram_event_dedupe_key(update_id), - external_event_id=str(update_id), - external_message_id=( - str(message["message_id"]) if message.get("message_id") is not None else None - ), - event_kind=_classify_telegram_event(payload), - raw_payload=payload, + account_id=account_id, + header_secret=request.headers.get("X-Telegram-Bot-Api-Secret-Token"), ) - await session.commit() - record_gateway_inbox_write(platform="telegram", dedup_skipped=inbox_id is None) - return Response(status_code=200) + + try: + update_id = payload.get("update_id") + if update_id is None: + return Response(status_code=200) + + message = _telegram_message(payload) or {} + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=ExternalChatPlatform.TELEGRAM, + event_dedupe_key=telegram_event_dedupe_key(update_id), + external_event_id=str(update_id), + external_message_id=( + str(message["message_id"]) if message.get("message_id") is not None else None + ), + event_kind=_classify_telegram_event(payload), + raw_payload=payload, + request_id=request_id, + ) + await session.commit() + record_gateway_inbox_write(platform="telegram", dedup_skipped=inbox_id is None) + return Response(status_code=200) + except Exception: + await session.rollback() + logger.exception("Telegram webhook processing failed account_id=%s", account_id) + return Response(status_code=200) @router.post("/bindings/start", response_model=StartBindingResponse) @@ -125,17 +131,17 @@ async def start_binding( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> StartBindingResponse: - if body.platform != GatewayPlatform.TELEGRAM: + if body.platform != ExternalChatPlatform.TELEGRAM: raise HTTPException(status_code=400, detail="Only Telegram is supported in v1") account = await get_or_create_system_telegram_account(session) code = generate_pairing_code() expires_at = pairing_expires_at() - binding = GatewayConversationBinding( + binding = ExternalChatBinding( account_id=account.id, user_id=user.id, search_space_id=body.search_space_id, - state=GatewayBindingState.PENDING, + state=ExternalChatBindingState.PENDING, pairing_code=code, pairing_code_expires_at=expires_at, ) @@ -143,7 +149,7 @@ async def start_binding( await session.commit() await session.refresh(binding) - username = account.account_metadata.get("bot_username") or config.TELEGRAM_SHARED_BOT_USERNAME + username = account.bot_username or config.TELEGRAM_SHARED_BOT_USERNAME if not username: raise HTTPException(status_code=500, detail="Telegram bot username is not configured") return StartBindingResponse( @@ -160,8 +166,8 @@ async def list_bindings( session: AsyncSession = Depends(get_async_session), ) -> list[dict[str, Any]]: result = await session.execute( - select(GatewayConversationBinding).where( - GatewayConversationBinding.user_id == user.id + select(ExternalChatBinding).where( + ExternalChatBinding.user_id == user.id ) ) return [ @@ -184,9 +190,9 @@ async def list_platforms( session: AsyncSession = Depends(get_async_session), ) -> list[dict[str, Any]]: result = await session.execute( - select(GatewayPlatformAccount).where( - (GatewayPlatformAccount.owner_user_id == user.id) - | (GatewayPlatformAccount.is_system_account.is_(True)) + select(ExternalChatAccount).where( + (ExternalChatAccount.owner_user_id == user.id) + | (ExternalChatAccount.is_system_account.is_(True)) ) ) return [ @@ -194,7 +200,7 @@ async def list_platforms( "id": account.id, "platform": account.platform.value, "mode": account.mode.value, - "bot_username": (account.account_metadata or {}).get("bot_username"), + "bot_username": account.bot_username, "health_status": account.health_status.value, "last_health_check_at": account.last_health_check_at, } @@ -208,7 +214,7 @@ async def delete_binding( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> dict[str, bool]: - binding = await session.get(GatewayConversationBinding, binding_id) + binding = await session.get(ExternalChatBinding, binding_id) if binding is None or binding.user_id != user.id: raise HTTPException(status_code=404, detail="Binding not found") revoke_binding(binding) @@ -217,12 +223,12 @@ async def delete_binding( @router.post("/bindings/{binding_id}/resume") -async def resume_gateway_binding( +async def resume_external_chat_binding( binding_id: int, user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> dict[str, bool]: - binding = await session.get(GatewayConversationBinding, binding_id) + binding = await session.get(ExternalChatBinding, binding_id) if binding is None or binding.user_id != user.id: raise HTTPException(status_code=404, detail="Binding not found") resume_binding(binding) diff --git a/surfsense_backend/scripts/register_webhook.py b/surfsense_backend/scripts/register_webhook.py index 2004ad118..44ead9470 100644 --- a/surfsense_backend/scripts/register_webhook.py +++ b/surfsense_backend/scripts/register_webhook.py @@ -10,6 +10,9 @@ import sys from dotenv import load_dotenv from telegram import Bot +from app.db import async_session_maker +from app.gateway.accounts import get_or_create_system_telegram_account + load_dotenv() WEBHOOK_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{1,256}$") @@ -32,7 +35,13 @@ async def main() -> int: ) return 1 - webhook_url = f"{base_url.rstrip('/')}/api/v1/gateway/webhooks/telegram/{secret}" + async with async_session_maker() as session: + account = await get_or_create_system_telegram_account(session) + account.webhook_secret = secret + await session.commit() + account_id = int(account.id) + + webhook_url = f"{base_url.rstrip('/')}/api/v1/gateway/webhooks/telegram/{account_id}" bot = Bot(token=token) ok = await bot.set_webhook( url=webhook_url, diff --git a/surfsense_backend/tests/unit/gateway/test_webhook_routes.py b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py new file mode 100644 index 000000000..9a62a3cce --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import inspect + +import pytest + +from app.db import ExternalChatPlatform, ExternalChatAccount +from app.routes import gateway_webhook_routes as routes + + +class RequestStub: + def __init__(self, payload=None, *, headers=None, json_exc: Exception | None = None): + self.headers = headers or {} + self._payload = payload + self._json_exc = json_exc + + async def json(self): + if self._json_exc is not None: + raise self._json_exc + return self._payload + + +def _account(secret: str = "secret") -> ExternalChatAccount: + return ExternalChatAccount( + id=123, + platform=ExternalChatPlatform.TELEGRAM, + webhook_secret=secret, + bot_username="surf_bot", + ) + + +async def _call_webhook(*, request: RequestStub, account_id: int, session): + return await routes.telegram_webhook( + request=request, + account_id=account_id, + session=session, + ) + + +@pytest.mark.asyncio +async def test_telegram_webhook_returns_200_on_null_update_id(mocker): + session = mocker.AsyncMock() + session.get.return_value = _account() + request = RequestStub( + {"message": {"message_id": 7}}, + headers={"X-Telegram-Bot-Api-Secret-Token": "secret"}, + ) + + response = await _call_webhook( + request=request, + account_id=123, + session=session, + ) + + assert response.status_code == 200 + session.commit.assert_not_called() + + +@pytest.mark.asyncio +async def test_telegram_webhook_returns_200_on_bad_json(mocker, monkeypatch): + parse_metric = mocker.Mock() + monkeypatch.setattr(routes, "record_gateway_webhook_parse_error", parse_metric) + request = RequestStub(json_exc=ValueError("bad json")) + + response = await _call_webhook( + request=request, + account_id=123, + session=mocker.AsyncMock(), + ) + + assert response.status_code == 200 + parse_metric.assert_called_once_with() + + +@pytest.mark.asyncio +async def test_resolve_webhook_account_rejects_missing_or_wrong_header(mocker): + session = mocker.AsyncMock() + session.get.return_value = _account() + + with pytest.raises(routes.HTTPException) as missing: + await routes._resolve_webhook_account( + session, + account_id=123, + header_secret=None, + ) + assert missing.value.status_code == 403 + + with pytest.raises(routes.HTTPException) as wrong: + await routes._resolve_webhook_account( + session, + account_id=123, + header_secret="wrong", + ) + assert wrong.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_telegram_webhook_persists_for_fastapi_inbox_worker(mocker, monkeypatch): + session = mocker.AsyncMock() + session.get.return_value = _account() + persist = mocker.AsyncMock(return_value=99) + monkeypatch.setattr(routes, "persist_inbound_event", persist) + + request = RequestStub( + { + "update_id": 10, + "message": {"message_id": 7, "chat": {"id": 1}, "from": {"id": 2}}, + }, + headers={"X-Telegram-Bot-Api-Secret-Token": "secret"}, + ) + + response = await _call_webhook( + request=request, + account_id=123, + session=session, + ) + + assert response.status_code == 200 + persist.assert_awaited_once() + session.commit.assert_awaited_once() + assert persist.await_args.kwargs["request_id"].startswith("gateway_") + + +@pytest.mark.asyncio +async def test_telegram_webhook_commits_dedup_without_enqueue(mocker, monkeypatch): + session = mocker.AsyncMock() + session.get.return_value = _account() + monkeypatch.setattr(routes, "persist_inbound_event", mocker.AsyncMock(return_value=None)) + + request = RequestStub( + {"update_id": 10, "message": {"message_id": 7}}, + headers={"X-Telegram-Bot-Api-Secret-Token": "secret"}, + ) + + response = await _call_webhook( + request=request, + account_id=123, + session=session, + ) + + assert response.status_code == 200 + session.commit.assert_awaited_once() + + +def test_telegram_webhook_does_not_use_slowapi_limiter(): + route_source = inspect.getsource(routes) + + assert "@limiter.limit" not in route_source + From 08bf3cc02348b829eba04fa261d1fdafaafbb81e Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 28 May 2026 04:38:00 +0530 Subject: [PATCH 16/63] refactor(gateway): run inbox and BYO polling from FastAPI lifespan --- surfsense_backend/app/app.py | 23 ++- surfsense_backend/app/config/__init__.py | 3 + .../app/gateway/byo_long_poll.py | 94 ++++++++++ surfsense_backend/app/gateway/inbox_worker.py | 55 ++++++ surfsense_backend/app/gateway/runner.py | 80 +++----- surfsense_backend/gateway_runner.py | 16 -- .../scripts/docker/entrypoint.sh | 8 +- .../gateway/test_byo_long_poll_lifespan.py | 172 ++++++++++++++++++ .../tests/unit/gateway/test_inbox_worker.py | 45 +++++ 9 files changed, 415 insertions(+), 81 deletions(-) create mode 100644 surfsense_backend/app/gateway/byo_long_poll.py create mode 100644 surfsense_backend/app/gateway/inbox_worker.py delete mode 100644 surfsense_backend/gateway_runner.py create mode 100644 surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py create mode 100644 surfsense_backend/tests/unit/gateway/test_inbox_worker.py diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index 43b0af7d2..17f4e093e 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -37,6 +37,14 @@ from app.config import ( ) from app.db import User, create_db_and_tables, get_async_session from app.exceptions import GENERIC_5XX_MESSAGE, ISSUES_URL, SurfSenseError +from app.gateway.byo_long_poll import ( + start_byo_long_poll_supervisors, + stop_byo_long_poll_supervisors, +) +from app.gateway.inbox_worker import ( + start_gateway_inbox_worker, + stop_gateway_inbox_worker, +) from app.observability import metrics as ot_metrics from app.observability.bootstrap import init_otel, shutdown_otel from app.rate_limiter import get_real_client_ip, limiter @@ -597,12 +605,17 @@ async def lifespan(app: FastAPI): ) log_system_snapshot("startup_complete") + await start_gateway_inbox_worker() + await start_byo_long_poll_supervisors() - yield - - _stop_openrouter_background_refresh() - await close_checkpointer() - shutdown_otel() + try: + yield + finally: + await stop_byo_long_poll_supervisors() + await stop_gateway_inbox_worker() + _stop_openrouter_background_refresh() + await close_checkpointer() + shutdown_otel() def registration_allowed(): diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index a7739d6c4..89bf4c925 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -546,6 +546,9 @@ class Config: TELEGRAM_SHARED_BOT_USERNAME = os.getenv("TELEGRAM_SHARED_BOT_USERNAME") TELEGRAM_WEBHOOK_SECRET = os.getenv("TELEGRAM_WEBHOOK_SECRET") GATEWAY_BASE_URL = os.getenv("GATEWAY_BASE_URL", BACKEND_URL) + GATEWAY_BYO_LONGPOLL_ENABLED = ( + os.getenv("GATEWAY_BYO_LONGPOLL_ENABLED", "TRUE").upper() == "TRUE" + ) # Stripe checkout for pay-as-you-go page packs STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY") diff --git a/surfsense_backend/app/gateway/byo_long_poll.py b/surfsense_backend/app/gateway/byo_long_poll.py new file mode 100644 index 000000000..d02f19f95 --- /dev/null +++ b/surfsense_backend/app/gateway/byo_long_poll.py @@ -0,0 +1,94 @@ +"""FastAPI lifespan integration for self-hosted BYO Telegram long-polling.""" + +from __future__ import annotations + +import asyncio +import logging + +from sqlalchemy import select + +from app.config import config +from app.db import ExternalChatPlatform, ExternalChatAccount, async_session_maker +from app.gateway.accounts import account_token +from app.gateway.runner import _run_telegram_account + +logger = logging.getLogger(__name__) + +_tasks: set[asyncio.Task[None]] = set() +_shutdown_event: asyncio.Event | None = None + + +async def _sleep_or_shutdown(seconds: float) -> None: + if _shutdown_event is None: + await asyncio.sleep(seconds) + return + try: + await asyncio.wait_for(_shutdown_event.wait(), timeout=seconds) + except TimeoutError: + return + + +async def _byo_account_supervisor(account_id: int, token: str) -> None: + while _shutdown_event is None or not _shutdown_event.is_set(): + try: + await _run_telegram_account(account_id, token) + except asyncio.CancelledError: + raise + except Exception: + logger.exception( + "BYO Telegram long-poll failed account_id=%s; retrying in 30s", + account_id, + ) + await _sleep_or_shutdown(30) + + +async def start_byo_long_poll_supervisors() -> None: + """Start one BYO long-poll supervisor per active non-system Telegram account.""" + + global _shutdown_event + if not config.GATEWAY_BYO_LONGPOLL_ENABLED: + return + if _tasks: + return + + _shutdown_event = asyncio.Event() + async with async_session_maker() as session: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.TELEGRAM, + ExternalChatAccount.is_system_account.is_(False), + ExternalChatAccount.suspended_at.is_(None), + ) + ) + accounts = list(result.scalars()) + + for account in accounts: + token = account_token(account) + if not token: + continue + task = asyncio.create_task( + _byo_account_supervisor(int(account.id), token), + name=f"gateway-byo-telegram-{account.id}", + ) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + logger.info("Started BYO Telegram long-poll supervisor account_id=%s", account.id) + + +async def stop_byo_long_poll_supervisors() -> None: + """Cancel and await all BYO long-poll supervisors.""" + + global _shutdown_event + if _shutdown_event is not None: + _shutdown_event.set() + tasks = list(_tasks) + for task in tasks: + task.cancel() + if tasks: + try: + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=10) + except TimeoutError: + logger.warning("Timed out waiting for BYO Telegram long-poll supervisors to stop") + _tasks.clear() + _shutdown_event = None + diff --git a/surfsense_backend/app/gateway/inbox_worker.py b/surfsense_backend/app/gateway/inbox_worker.py new file mode 100644 index 000000000..e3ea7225c --- /dev/null +++ b/surfsense_backend/app/gateway/inbox_worker.py @@ -0,0 +1,55 @@ +"""FastAPI lifespan worker for gateway inbox processing.""" + +from __future__ import annotations + +import asyncio +import logging +from contextlib import suppress + +from app.gateway.inbox_processor import claim_next_inbound_event, process_inbound_event + +logger = logging.getLogger(__name__) + +_task: asyncio.Task[None] | None = None + + +async def _process_inbox_forever() -> None: + logger.info("Gateway inbox processor started in FastAPI process") + while True: + try: + inbox_id = await claim_next_inbound_event() + if inbox_id is None: + await asyncio.sleep(0.5) + continue + logger.info("Gateway processing inbox_id=%s", inbox_id) + await process_inbound_event(inbox_id) + logger.info("Gateway processed inbox_id=%s", inbox_id) + except asyncio.CancelledError: + raise + except RuntimeError as exc: + if str(exc) == "gateway_thread_busy": + logger.info("Gateway inbox_id busy; will retry from RECEIVED state") + else: + logger.exception("Gateway inbox processor failed one iteration") + await asyncio.sleep(1) + except Exception: + logger.exception("Gateway inbox processor failed one iteration") + await asyncio.sleep(1) + + +async def start_gateway_inbox_worker() -> None: + global _task + if _task is not None and not _task.done(): + return + _task = asyncio.create_task(_process_inbox_forever(), name="gateway-inbox-worker") + + +async def stop_gateway_inbox_worker() -> None: + global _task + if _task is None: + return + _task.cancel() + with suppress(TimeoutError, asyncio.CancelledError): + await asyncio.wait_for(_task, timeout=10) + _task = None + diff --git a/surfsense_backend/app/gateway/runner.py b/surfsense_backend/app/gateway/runner.py index 8ebd89253..83afc2353 100644 --- a/surfsense_backend/app/gateway/runner.py +++ b/surfsense_backend/app/gateway/runner.py @@ -1,18 +1,17 @@ -"""Long-lived messaging gateway runner.""" +"""Telegram BYO long-poll helper for FastAPI lifespan.""" from __future__ import annotations -import asyncio import hashlib import logging +import uuid -from sqlalchemy import select, text +from sqlalchemy import text -from app.db import GatewayPlatform, GatewayPlatformAccount, async_session_maker, engine -from app.gateway.accounts import account_token +from app.db import ExternalChatPlatform, ExternalChatAccount, async_session_maker, engine from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key -from app.gateway.inbox_processor import claim_next_inbound_event, process_inbound_event from app.gateway.telegram.adapter import TelegramAdapter +from app.observability.metrics import record_gateway_byo_longpoll_running_delta logger = logging.getLogger(__name__) @@ -22,76 +21,45 @@ def _lock_key(token: str) -> int: return int.from_bytes(digest[:8], "big", signed=True) -class GatewayRunner: - async def run(self) -> None: - logger.info("Gateway runner started. Waiting for inbound events.") - tasks = [asyncio.create_task(self._process_inbox_forever())] - - async with async_session_maker() as session: - result = await session.execute( - select(GatewayPlatformAccount).where( - GatewayPlatformAccount.platform == GatewayPlatform.TELEGRAM, - GatewayPlatformAccount.is_system_account.is_(False), - GatewayPlatformAccount.suspended_at.is_(None), - ) - ) - accounts = list(result.scalars()) - - for account in accounts: - token = account_token(account) - if not token: - continue - logger.info("Starting Telegram long-poll loop for account_id=%s", account.id) - tasks.append(asyncio.create_task(self._run_telegram_account(account.id, token))) - - await asyncio.gather(*tasks) - - async def _process_inbox_forever(self) -> None: - logger.info("Gateway inbox processor started") - while True: - try: - inbox_id = await claim_next_inbound_event() - if inbox_id is None: - await asyncio.sleep(0.5) - continue - logger.info("Gateway processing inbox_id=%s", inbox_id) - await process_inbound_event(inbox_id) - logger.info("Gateway processed inbox_id=%s", inbox_id) - except Exception: - logger.exception("Gateway inbox processor failed one iteration") - await asyncio.sleep(1) - - async def _run_telegram_account(self, account_id: int, token: str) -> None: - async with engine.connect() as conn: - got_lock = await conn.scalar( - text("SELECT pg_try_advisory_lock(:key)"), - {"key": _lock_key(token)}, - ) - if not got_lock: - logger.warning("Another Telegram gateway runner is active; exiting") - return +async def _run_telegram_account(account_id: int, token: str) -> None: + async with engine.connect() as conn: + lock_key = _lock_key(token) + got_lock = await conn.scalar( + text("SELECT pg_try_advisory_lock(:key)"), + {"key": lock_key}, + ) + if not got_lock: + logger.warning("Another Telegram gateway runner is active; exiting") + return + record_gateway_byo_longpoll_running_delta(1, account_id=account_id) + try: adapter = TelegramAdapter(token) async with async_session_maker() as session: - account = await session.get(GatewayPlatformAccount, account_id) + account = await session.get(ExternalChatAccount, account_id) offset = None if account is not None: offset = int((account.cursor_state or {}).get("last_update_id", 0)) + 1 async for update in adapter.fetch_updates(offset=offset): + request_id = f"gateway_{uuid.uuid4().hex[:16]}" async with async_session_maker() as session: parsed = adapter.parse_inbound(update) inbox_id = await persist_inbound_event( session, account_id=account_id, - platform=GatewayPlatform.TELEGRAM, + platform=ExternalChatPlatform.TELEGRAM, event_dedupe_key=telegram_event_dedupe_key(update["update_id"]), external_event_id=str(update["update_id"]), external_message_id=parsed.external_message_id, event_kind=parsed.event_kind, raw_payload=update, + request_id=request_id, ) await session.commit() if inbox_id is not None: logger.debug("Persisted Telegram polling update inbox_id=%s", inbox_id) + finally: + record_gateway_byo_longpoll_running_delta(-1, account_id=account_id) + await conn.execute(text("SELECT pg_advisory_unlock(:key)"), {"key": lock_key}) diff --git a/surfsense_backend/gateway_runner.py b/surfsense_backend/gateway_runner.py deleted file mode 100644 index 27077ef48..000000000 --- a/surfsense_backend/gateway_runner.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Entrypoint for SERVICE_ROLE=gateway.""" - -from __future__ import annotations - -import asyncio -import logging - -from app.gateway.runner import GatewayRunner - -if __name__ == "__main__": - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s [%(name)s] %(message)s", - ) - asyncio.run(GatewayRunner().run()) - diff --git a/surfsense_backend/scripts/docker/entrypoint.sh b/surfsense_backend/scripts/docker/entrypoint.sh index 81db1ae84..0c1e66790 100644 --- a/surfsense_backend/scripts/docker/entrypoint.sh +++ b/surfsense_backend/scripts/docker/entrypoint.sh @@ -140,11 +140,11 @@ start_worker() { if [ -n "${CELERY_QUEUES}" ]; then QUEUE_ARGS="--queues=${CELERY_QUEUES}" else - # When no queues specified, consume from BOTH the default queue and - # the connectors queue. Without --queues, Celery only consumes from - # the default queue, leaving connector indexing tasks stuck. + # When no queues specified, consume from the default, connectors, and + # gateway maintenance queues. Without --queues, Celery only consumes + # from the default queue, leaving connector/gateway maintenance tasks stuck. DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" - QUEUE_ARGS="--queues=${DEFAULT_Q},${DEFAULT_Q}.connectors" + QUEUE_ARGS="--queues=${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway" fi echo "Starting Celery Worker (autoscale=${CELERY_MAX_WORKERS},${CELERY_MIN_WORKERS}, max-tasks-per-child=${CELERY_MAX_TASKS_PER_CHILD}, queues=${CELERY_QUEUES:-all})..." diff --git a/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py b/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py new file mode 100644 index 000000000..b8212ec9a --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import asyncio + +import pytest +import pytest_asyncio + +from app.gateway import byo_long_poll +from app.gateway import runner + + +class ScalarResult: + def __init__(self, rows): + self._rows = rows + + def scalars(self): + return self + + def __iter__(self): + return iter(self._rows) + + +class SessionContext: + def __init__(self, session): + self.session = session + + async def __aenter__(self): + return self.session + + async def __aexit__(self, exc_type, exc, tb): + return False + + +@pytest_asyncio.fixture(autouse=True) +async def cleanup_supervisors(): + yield + await byo_long_poll.stop_byo_long_poll_supervisors() + + +@pytest.mark.asyncio +async def test_start_byo_long_poll_noops_when_flag_off(monkeypatch): + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", False) + + await byo_long_poll.start_byo_long_poll_supervisors() + + assert byo_long_poll._tasks == set() + + +@pytest.mark.asyncio +async def test_start_byo_long_poll_noops_when_no_byo_accounts(mocker, monkeypatch): + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", True) + session = mocker.AsyncMock() + session.execute.return_value = ScalarResult([]) + monkeypatch.setattr( + byo_long_poll, + "async_session_maker", + lambda: SessionContext(session), + ) + + await byo_long_poll.start_byo_long_poll_supervisors() + + assert byo_long_poll._tasks == set() + + +@pytest.mark.asyncio +async def test_start_byo_long_poll_spawns_one_supervisor_per_account(mocker, monkeypatch): + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", True) + accounts = [mocker.Mock(id=1), mocker.Mock(id=2)] + session = mocker.AsyncMock() + session.execute.return_value = ScalarResult(accounts) + monkeypatch.setattr( + byo_long_poll, + "async_session_maker", + lambda: SessionContext(session), + ) + monkeypatch.setattr(byo_long_poll, "account_token", lambda account: f"token-{account.id}") + + async def forever(_account_id: int, _token: str) -> None: + await asyncio.Event().wait() + + monkeypatch.setattr(byo_long_poll, "_byo_account_supervisor", forever) + + await byo_long_poll.start_byo_long_poll_supervisors() + + assert len(byo_long_poll._tasks) == 2 + + +@pytest.mark.asyncio +async def test_supervisor_retries_after_run_returns(mocker, monkeypatch): + byo_long_poll._shutdown_event = asyncio.Event() + run = mocker.AsyncMock(side_effect=[None, None]) + monkeypatch.setattr(byo_long_poll, "_run_telegram_account", run) + sleep_count = 0 + + async def fake_sleep(_seconds: float) -> None: + nonlocal sleep_count + sleep_count += 1 + if sleep_count >= 2: + assert byo_long_poll._shutdown_event is not None + byo_long_poll._shutdown_event.set() + + monkeypatch.setattr(byo_long_poll, "_sleep_or_shutdown", fake_sleep) + + await byo_long_poll._byo_account_supervisor(7, "token") + + assert run.await_count == 2 + + +@pytest.mark.asyncio +async def test_shutdown_cancels_running_supervisors(mocker, monkeypatch): + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", True) + session = mocker.AsyncMock() + session.execute.return_value = ScalarResult([mocker.Mock(id=1)]) + monkeypatch.setattr( + byo_long_poll, + "async_session_maker", + lambda: SessionContext(session), + ) + monkeypatch.setattr(byo_long_poll, "account_token", lambda _account: "token") + + async def forever(_account_id: int, _token: str) -> None: + await asyncio.Event().wait() + + monkeypatch.setattr(byo_long_poll, "_byo_account_supervisor", forever) + + await byo_long_poll.start_byo_long_poll_supervisors() + await byo_long_poll.stop_byo_long_poll_supervisors() + + assert byo_long_poll._tasks == set() + + +@pytest.mark.asyncio +async def test_run_telegram_account_persists_for_fastapi_inbox_worker(mocker, monkeypatch): + class ConnectionContext: + async def __aenter__(self): + conn = mocker.AsyncMock() + conn.scalar.return_value = True + return conn + + async def __aexit__(self, exc_type, exc, tb): + return False + + class EngineStub: + def connect(self): + return ConnectionContext() + + class AdapterStub: + def __init__(self, _token: str) -> None: + pass + + async def fetch_updates(self, *, offset: int | None): + yield {"update_id": 11, "message": {"message_id": 5}} + + def parse_inbound(self, update): + return mocker.Mock(external_message_id="5", event_kind="message") + + first_session = mocker.AsyncMock() + first_session.get.return_value = mocker.Mock(cursor_state={}) + second_session = mocker.AsyncMock() + contexts = iter([SessionContext(first_session), SessionContext(second_session)]) + monkeypatch.setattr(runner, "engine", EngineStub()) + monkeypatch.setattr(runner, "async_session_maker", lambda: next(contexts)) + monkeypatch.setattr(runner, "TelegramAdapter", AdapterStub) + persist = mocker.AsyncMock(return_value=42) + monkeypatch.setattr(runner, "persist_inbound_event", persist) + + await runner._run_telegram_account(123, "token") + + second_session.commit.assert_awaited_once() + persist.assert_awaited_once() + assert persist.await_args.kwargs["request_id"].startswith("gateway_") + diff --git a/surfsense_backend/tests/unit/gateway/test_inbox_worker.py b/surfsense_backend/tests/unit/gateway/test_inbox_worker.py new file mode 100644 index 000000000..8ecc4d86a --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_inbox_worker.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from app.gateway import inbox_worker + + +@pytest.mark.asyncio +async def test_inbox_worker_claims_and_processes_in_fastapi_process(mocker, monkeypatch): + claim = mocker.AsyncMock(return_value=7) + process = mocker.AsyncMock(side_effect=asyncio.CancelledError) + monkeypatch.setattr(inbox_worker, "claim_next_inbound_event", claim) + monkeypatch.setattr(inbox_worker, "process_inbound_event", process) + + with pytest.raises(asyncio.CancelledError): + await inbox_worker._process_inbox_forever() + + claim.assert_awaited_once() + process.assert_awaited_once_with(7) + + +@pytest.mark.asyncio +async def test_start_stop_gateway_inbox_worker(mocker, monkeypatch): + started = asyncio.Event() + stopped = asyncio.Event() + + async def run_forever(): + started.set() + try: + await asyncio.Event().wait() + finally: + stopped.set() + + monkeypatch.setattr(inbox_worker, "_process_inbox_forever", run_forever) + inbox_worker._task = None + + await inbox_worker.start_gateway_inbox_worker() + await asyncio.wait_for(started.wait(), timeout=1) + await inbox_worker.stop_gateway_inbox_worker() + + assert stopped.is_set() + assert inbox_worker._task is None + From afcadfb4bfcf433dbb67bbc826d5119648b9d239 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 28 May 2026 04:38:20 +0530 Subject: [PATCH 17/63] fix(gateway): preserve request context during inbox processing --- surfsense_backend/app/gateway/agent_invoke.py | 13 ++- .../app/gateway/inbox_processor.py | 101 +++++++++--------- 2 files changed, 60 insertions(+), 54 deletions(-) diff --git a/surfsense_backend/app/gateway/agent_invoke.py b/surfsense_backend/app/gateway/agent_invoke.py index b876d1977..39d1fb299 100644 --- a/surfsense_backend/app/gateway/agent_invoke.py +++ b/surfsense_backend/app/gateway/agent_invoke.py @@ -1,4 +1,4 @@ -"""Invoke SurfSense chat agent for gateway channels.""" +"""Invoke SurfSense chat agent for external chat surfaces.""" from __future__ import annotations @@ -6,9 +6,10 @@ import json import logging from collections.abc import AsyncIterator +from sqlalchemy import update from sqlalchemy.ext.asyncio import AsyncSession -from app.db import GatewayConversationBinding +from app.db import ExternalChatBinding, NewChatMessage from app.gateway.auth_invariant import assert_authorization_invariant from app.gateway.base.translator import GatewayStreamEvent from app.gateway.bindings import get_or_create_thread_for_binding @@ -55,7 +56,7 @@ async def _events_from_sse(chunks: AsyncIterator[str]) -> AsyncIterator[GatewayS async def call_agent_for_gateway( *, session: AsyncSession, - binding: GatewayConversationBinding, + binding: ExternalChatBinding, user_text: str, translator: TelegramStreamTranslator, request_id: str | None = None, @@ -85,6 +86,12 @@ async def call_agent_for_gateway( finally: await events.aclose() await stream.aclose() + await session.execute( + update(NewChatMessage) + .where(NewChatMessage.thread_id == thread.id, NewChatMessage.source == "web") + .values(source="telegram") + ) + await session.commit() record_gateway_turn_latency(0, platform="telegram") finally: release_thread_lock(thread.id) diff --git a/surfsense_backend/app/gateway/inbox_processor.py b/surfsense_backend/app/gateway/inbox_processor.py index 3e3f962b7..c40a6c47c 100644 --- a/surfsense_backend/app/gateway/inbox_processor.py +++ b/surfsense_backend/app/gateway/inbox_processor.py @@ -1,8 +1,7 @@ -"""Long-lived gateway inbox processing. +"""Long-lived external chat inbox processing. -This module owns the agent-turn execution path for messaging gateways. It is -intentionally independent of Celery so LangGraph, async Postgres, Redis, and -Telegram clients all run on one stable event loop in ``GatewayRunner``. +This module owns the agent-turn execution path for external chat surfaces. +FastAPI calls into it after webhook and BYO long-poll intake persist inbox rows. """ from __future__ import annotations @@ -16,12 +15,12 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from app.config import config from app.db import ( - GatewayBindingState, - GatewayConversationBinding, - GatewayEventStatus, - GatewayInboundEvent, - GatewayPeerKind, - GatewayPlatformAccount, + ExternalChatBindingState, + ExternalChatBinding, + ExternalChatEventStatus, + ExternalChatInboundEvent, + ExternalChatPeerKind, + ExternalChatAccount, NewChatThread, async_session_maker, ) @@ -54,16 +53,16 @@ async def claim_next_inbound_event( async with session_maker() as session: result = await session.execute( - select(GatewayInboundEvent) - .where(GatewayInboundEvent.status == GatewayEventStatus.RECEIVED) - .order_by(GatewayInboundEvent.received_at.asc()) + select(ExternalChatInboundEvent) + .where(ExternalChatInboundEvent.status == ExternalChatEventStatus.RECEIVED) + .order_by(ExternalChatInboundEvent.received_at.asc()) .with_for_update(skip_locked=True) .limit(1) ) event = result.scalars().first() if event is None: return None - event.status = GatewayEventStatus.PROCESSING + event.status = ExternalChatEventStatus.PROCESSING event.attempt_count += 1 await session.commit() return int(event.id) @@ -73,22 +72,22 @@ async def process_inbound_event( inbox_id: int, session_maker: SessionMaker = async_session_maker, ) -> None: - """Process one gateway inbox row and mark its terminal status.""" + """Process one external chat inbox row and mark its terminal status.""" async with session_maker() as session: result = await session.execute( - select(GatewayInboundEvent) - .where(GatewayInboundEvent.id == inbox_id) + select(ExternalChatInboundEvent) + .where(ExternalChatInboundEvent.id == inbox_id) .with_for_update(skip_locked=True) ) event = result.scalars().first() if event is None or event.status in { - GatewayEventStatus.PROCESSED, - GatewayEventStatus.IGNORED, + ExternalChatEventStatus.PROCESSED, + ExternalChatEventStatus.IGNORED, }: return - if event.status == GatewayEventStatus.RECEIVED: - event.status = GatewayEventStatus.PROCESSING + if event.status == ExternalChatEventStatus.RECEIVED: + event.status = ExternalChatEventStatus.PROCESSING event.attempt_count += 1 await session.commit() @@ -98,15 +97,15 @@ async def process_inbound_event( if str(exc) == "gateway_thread_busy": async with session_maker() as session: await session.execute( - update(GatewayInboundEvent) - .where(GatewayInboundEvent.id == inbox_id) + update(ExternalChatInboundEvent) + .where(ExternalChatInboundEvent.id == inbox_id) .values( - status=GatewayEventStatus.RECEIVED, + status=ExternalChatEventStatus.RECEIVED, last_error="gateway_thread_busy", ) ) await session.commit() - return + raise await _mark_failed(inbox_id, str(exc), session_maker) raise except Exception as exc: @@ -114,9 +113,9 @@ async def process_inbound_event( raise async with session_maker() as session: - event = await session.get(GatewayInboundEvent, inbox_id) - if event is not None and event.status == GatewayEventStatus.PROCESSING: - event.status = GatewayEventStatus.PROCESSED + event = await session.get(ExternalChatInboundEvent, inbox_id) + if event is not None and event.status == ExternalChatEventStatus.PROCESSING: + event.status = ExternalChatEventStatus.PROCESSED event.processed_at = datetime.now(UTC) await session.commit() record_gateway_inbox_processed(platform=event.platform.value, status="processed") @@ -129,9 +128,9 @@ async def _mark_failed( ) -> None: async with session_maker() as session: await session.execute( - update(GatewayInboundEvent) - .where(GatewayInboundEvent.id == inbox_id) - .values(status=GatewayEventStatus.FAILED, last_error=error) + update(ExternalChatInboundEvent) + .where(ExternalChatInboundEvent.id == inbox_id) + .values(status=ExternalChatEventStatus.FAILED, last_error=error) ) await session.commit() @@ -141,19 +140,19 @@ async def _dispatch_inbound_event( session_maker: SessionMaker, ) -> None: async with session_maker() as session: - event = await session.get(GatewayInboundEvent, inbox_id) + event = await session.get(ExternalChatInboundEvent, inbox_id) if event is None: return - account = await session.get(GatewayPlatformAccount, event.account_id) + account = await session.get(ExternalChatAccount, event.account_id) if account is None: - event.status = GatewayEventStatus.IGNORED + event.status = ExternalChatEventStatus.IGNORED event.last_error = "account_missing" await session.commit() return token = account_token(account) if not token: - event.status = GatewayEventStatus.FAILED + event.status = ExternalChatEventStatus.FAILED event.last_error = "missing_telegram_token" await session.commit() return @@ -161,7 +160,7 @@ async def _dispatch_inbound_event( adapter = TelegramAdapter(token) parsed = adapter.parse_inbound(event.raw_payload or {}) if parsed.external_peer_id is None: - event.status = GatewayEventStatus.IGNORED + event.status = ExternalChatEventStatus.IGNORED event.last_error = "missing_external_peer_id" await session.commit() return @@ -169,19 +168,19 @@ async def _dispatch_inbound_event( _update_account_cursor(account, parsed.metadata.get("update_id")) result = await session.execute( - select(GatewayConversationBinding).where( - GatewayConversationBinding.account_id == account.id, - GatewayConversationBinding.external_peer_id == parsed.external_peer_id, - GatewayConversationBinding.state.in_( - [GatewayBindingState.BOUND, GatewayBindingState.SUSPENDED] + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == parsed.external_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] ), ) ) binding = result.scalars().first() - if parsed.external_peer_kind != GatewayPeerKind.DIRECT.value: + if parsed.external_peer_kind != ExternalChatPeerKind.DIRECT.value: await adapter.leave_chat(external_peer_id=parsed.external_peer_id) - event.status = GatewayEventStatus.IGNORED + event.status = ExternalChatEventStatus.IGNORED event.last_error = "group_rejected" await session.commit() return @@ -201,30 +200,30 @@ async def _dispatch_inbound_event( event=parsed, dashboard_url=_dashboard_url(), ) - event.status = GatewayEventStatus.IGNORED + event.status = ExternalChatEventStatus.IGNORED event.last_error = "unbound_chat" await session.commit() return - event.binding_id = binding.id + event.external_chat_binding_id = binding.id if cmd == "/help": await handle_help_command(adapter=adapter, event=parsed) - event.status = GatewayEventStatus.PROCESSED + event.status = ExternalChatEventStatus.PROCESSED await session.commit() return if cmd == "/new": - binding.active_thread_id = None + binding.new_chat_thread_id = None await adapter.send_message( external_peer_id=parsed.external_peer_id, text="Started a new SurfSense conversation.", ) - event.status = GatewayEventStatus.PROCESSED + event.status = ExternalChatEventStatus.PROCESSED await session.commit() return if not parsed.text: - event.status = GatewayEventStatus.IGNORED + event.status = ExternalChatEventStatus.IGNORED event.last_error = "empty_message" await session.commit() return @@ -241,7 +240,7 @@ async def _dispatch_inbound_event( binding=binding, user_text=parsed.text, translator=translator, - request_id=f"gateway:{inbox_id}", + request_id=event.request_id or f"gateway:{inbox_id}", ) thread = await session.get(NewChatThread, thread.id) @@ -250,7 +249,7 @@ async def _dispatch_inbound_event( await session.commit() -def _update_account_cursor(account: GatewayPlatformAccount, update_id: object) -> None: +def _update_account_cursor(account: ExternalChatAccount, update_id: object) -> None: if update_id is None: return account.cursor_state = { From 2a41a157f7ba7dfbfabe62c42aa92b3c8e82817c Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 28 May 2026 04:39:54 +0530 Subject: [PATCH 18/63] refactor(gateway): model external chat surfaces over canonical chats --- surfsense_backend/app/celery_app.py | 1 - .../app/tasks/celery_tasks/gateway_tasks.py | 70 +++++++++---------- .../gateway/test_enqueue_received_sweep.py | 16 +++++ .../test_process_inbound_event_task.py | 13 ++++ 4 files changed, 64 insertions(+), 36 deletions(-) create mode 100644 surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py create mode 100644 surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py diff --git a/surfsense_backend/app/celery_app.py b/surfsense_backend/app/celery_app.py index 2423133fb..73746f04f 100644 --- a/surfsense_backend/app/celery_app.py +++ b/surfsense_backend/app/celery_app.py @@ -243,7 +243,6 @@ celery_app.conf.update( "index_obsidian_attachment": {"queue": CONNECTORS_QUEUE}, # Everything else (document processing, podcasts, reindexing, # schedule checker, cleanup) stays on the default fast queue. - "gateway.process_inbound_event": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, "gateway.reconcile_inbox": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, "gateway.health_check": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, "gateway.retention_sweep": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, diff --git a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py index b8076b5a7..aeb3d721e 100644 --- a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py @@ -1,4 +1,4 @@ -"""Celery tasks for messaging gateway intake and maintenance.""" +"""Celery maintenance tasks for external chat surfaces.""" from __future__ import annotations @@ -9,11 +9,11 @@ from sqlalchemy import select, update from app.celery_app import celery_app from app.db import ( - GatewayEventStatus, - GatewayHealthStatus, - GatewayInboundEvent, - GatewayPlatform, - GatewayPlatformAccount, + ExternalChatEventStatus, + ExternalChatHealthStatus, + ExternalChatInboundEvent, + ExternalChatPlatform, + ExternalChatAccount, ) from app.gateway.accounts import account_token from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key @@ -27,17 +27,11 @@ from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_ta logger = logging.getLogger(__name__) -@celery_app.task( - bind=True, - name="gateway.process_inbound_event", - acks_late=True, - max_retries=5, - retry_backoff=True, -) -def process_inbound_event_task(self, inbox_id: int) -> None: +@celery_app.task(name="gateway.process_inbound_event") +def process_inbound_event_task(inbox_id: int) -> None: logger.warning( - "Ignoring Celery gateway.process_inbound_event for inbox_id=%s; " - "GatewayRunner owns agent turn processing.", + "Ignoring gateway.process_inbound_event for inbox_id=%s; " + "FastAPI owns external chat agent turn processing.", inbox_id, ) return None @@ -50,14 +44,14 @@ def reconcile_inbox_task() -> None: async with session_maker() as session: stale_threshold = datetime.now(UTC) - timedelta(minutes=10) result = await session.execute( - update(GatewayInboundEvent) + update(ExternalChatInboundEvent) .where( - GatewayInboundEvent.status == GatewayEventStatus.PROCESSING, - GatewayInboundEvent.received_at < stale_threshold, + ExternalChatInboundEvent.status == ExternalChatEventStatus.PROCESSING, + ExternalChatInboundEvent.received_at < stale_threshold, ) .values( - status=GatewayEventStatus.RECEIVED, - last_error="stale processing reset for gateway runner", + status=ExternalChatEventStatus.RECEIVED, + last_error="stale processing reset for FastAPI inbox worker", ) ) for _ in range(result.rowcount or 0): @@ -72,22 +66,19 @@ def gateway_health_check_task() -> None: async def _run() -> None: session_maker = get_celery_session_maker() async with session_maker() as session: - result = await session.execute(select(GatewayPlatformAccount)) + result = await session.execute(select(ExternalChatAccount)) accounts = list(result.scalars()) for account in accounts: token = account_token(account) - if not token or account.platform != GatewayPlatform.TELEGRAM: + if not token or account.platform != ExternalChatPlatform.TELEGRAM: continue try: metadata = await TelegramAdapter(token).validate_credentials() - account.health_status = GatewayHealthStatus.OK - account.account_metadata = { - **(account.account_metadata or {}), - "bot_username": metadata.get("username"), - } + account.health_status = ExternalChatHealthStatus.OK + account.bot_username = metadata.get("username") except Exception: - logger.warning("Gateway Telegram health check failed", exc_info=True) - account.health_status = GatewayHealthStatus.FAILING + logger.warning("External chat Telegram health check failed", exc_info=True) + account.health_status = ExternalChatHealthStatus.FAILING record_gateway_health_check_failure(platform=account.platform.value) account.last_health_check_at = datetime.now(UTC) await session.commit() @@ -95,6 +86,15 @@ def gateway_health_check_task() -> None: return run_async_celery_task(_run) +@celery_app.task(name="gateway.enqueue_received_sweep") +def enqueue_received_sweep_task() -> int: + logger.info( + "Skipping gateway.enqueue_received_sweep; " + "FastAPI inbox worker scans RECEIVED rows directly." + ) + return 0 + + @celery_app.task(name="gateway.retention_sweep") def gateway_retention_sweep_task() -> None: async def _run() -> None: @@ -103,13 +103,13 @@ def gateway_retention_sweep_task() -> None: raw_cutoff = datetime.now(UTC) - timedelta(days=30) delete_cutoff = datetime.now(UTC) - timedelta(days=365) await session.execute( - update(GatewayInboundEvent) - .where(GatewayInboundEvent.received_at < raw_cutoff) + update(ExternalChatInboundEvent) + .where(ExternalChatInboundEvent.received_at < raw_cutoff) .values(raw_payload=None) ) result = await session.execute( - select(GatewayInboundEvent).where( - GatewayInboundEvent.received_at < delete_cutoff + select(ExternalChatInboundEvent).where( + ExternalChatInboundEvent.received_at < delete_cutoff ) ) for event in result.scalars(): @@ -126,7 +126,7 @@ async def enqueue_telegram_update(account_id: int, raw_update: dict) -> int | No inbox_id = await persist_inbound_event( session, account_id=account_id, - platform=GatewayPlatform.TELEGRAM, + platform=ExternalChatPlatform.TELEGRAM, event_dedupe_key=telegram_event_dedupe_key(raw_update["update_id"]), external_event_id=str(raw_update["update_id"]), external_message_id=parsed.external_message_id, diff --git a/surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py b/surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py new file mode 100644 index 000000000..5fe46502f --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from app.tasks.celery_tasks import gateway_tasks + + +def test_enqueue_received_sweep_is_noop_guard(mocker): + apply_async = mocker.Mock() + mocker.patch.object(gateway_tasks.process_inbound_event_task, "apply_async", apply_async) + info = mocker.patch.object(gateway_tasks.logger, "info") + + replayed = gateway_tasks.enqueue_received_sweep_task.run() + + apply_async.assert_not_called() + assert replayed == 0 + info.assert_called_once() + diff --git a/surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py b/surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py new file mode 100644 index 000000000..484eacd1a --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from app.tasks.celery_tasks import gateway_tasks + + +def test_process_inbound_event_task_is_noop_guard(mocker): + warning = mocker.patch.object(gateway_tasks.logger, "warning") + + assert gateway_tasks.process_inbound_event_task.run(123) is None + + warning.assert_called_once() + assert "FastAPI owns external chat agent turn processing" in warning.call_args.args[0] + From 7ff0120fc9b6c6133a693fd321ace851d3d78d5d Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 28 May 2026 04:40:13 +0530 Subject: [PATCH 19/63] feat(gateway): add worker and webhook metrics --- .../app/observability/metrics.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/surfsense_backend/app/observability/metrics.py b/surfsense_backend/app/observability/metrics.py index 8098ac307..5ba3be059 100644 --- a/surfsense_backend/app/observability/metrics.py +++ b/surfsense_backend/app/observability/metrics.py @@ -411,6 +411,38 @@ def _gateway_active_bindings(): ) +@lru_cache(maxsize=1) +def _gateway_inbox_enqueued(): + return _get_meter().create_counter( + "gateway_inbox_enqueued_total", + description="Count of gateway inbox rows enqueued for worker processing.", + ) + + +@lru_cache(maxsize=1) +def _gateway_inbox_sweep_replayed(): + return _get_meter().create_counter( + "gateway_inbox_sweep_replayed_total", + description="Count of received gateway inbox rows replayed by the sweep.", + ) + + +@lru_cache(maxsize=1) +def _gateway_byo_longpoll_running(): + return _get_meter().create_up_down_counter( + "gateway_byo_longpoll_running", + description="Current change in BYO Telegram long-poll supervisors holding a poll loop.", + ) + + +@lru_cache(maxsize=1) +def _gateway_webhook_parse_errors(): + return _get_meter().create_counter( + "gateway_webhook_parse_error_total", + description="Count of malformed gateway webhook payloads.", + ) + + def record_model_call_duration( duration_ms: float, *, model: str | None, provider: str | None ) -> None: @@ -722,6 +754,22 @@ def record_gateway_active_bindings_delta(delta: int, *, platform: str) -> None: _add(_gateway_active_bindings(), delta, {"platform": platform}) +def record_gateway_inbox_enqueued(*, intake: str, outcome: str) -> None: + _add(_gateway_inbox_enqueued(), 1, {"intake": intake, "outcome": outcome}) + + +def record_gateway_inbox_sweep_replayed() -> None: + _add(_gateway_inbox_sweep_replayed(), 1, {}) + + +def record_gateway_byo_longpoll_running_delta(delta: int, *, account_id: int) -> None: + _add(_gateway_byo_longpoll_running(), delta, {"account_id": account_id}) + + +def record_gateway_webhook_parse_error() -> None: + _add(_gateway_webhook_parse_errors(), 1, {}) + + def _runtime_snapshot_value(key: str, transform: Any = None) -> list[Any]: from opentelemetry.metrics import Observation From c958fe5bc6d98cccf9428d401cbadeff0f411ac7 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 28 May 2026 05:02:07 +0530 Subject: [PATCH 20/63] feat(gateway): introduce GATEWAY_TELEGRAM_INTAKE_MODE for Telegram integration --- surfsense_backend/.env.example | 2 + surfsense_backend/app/config/__init__.py | 10 +++-- .../app/gateway/byo_long_poll.py | 2 +- .../gateway/test_byo_long_poll_lifespan.py | 10 ++--- .../content/docs/manual-installation.mdx | 43 ++++++++++++++++--- 5 files changed, 53 insertions(+), 14 deletions(-) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 6fef9b20e..340e5b51a 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -17,10 +17,12 @@ REDIS_APP_URL=redis://localhost:6379/0 # Telegram Gateway # TELEGRAM_WEBHOOK_SECRET must be 1-256 chars and contain only A-Z, a-z, 0-9, _ or - +# GATEWAY_TELEGRAM_INTAKE_MODE: webhook for production, longpoll for single-replica self-host fallback, disabled to skip Telegram intake TELEGRAM_SHARED_BOT_TOKEN= TELEGRAM_SHARED_BOT_USERNAME= TELEGRAM_WEBHOOK_SECRET= GATEWAY_BASE_URL=http://localhost:8000 +GATEWAY_TELEGRAM_INTAKE_MODE=webhook # Platform Web Search (SearXNG) # Set this to enable built-in web search. Docker Compose sets it automatically. diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 89bf4c925..c77b95fde 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -546,9 +546,13 @@ class Config: TELEGRAM_SHARED_BOT_USERNAME = os.getenv("TELEGRAM_SHARED_BOT_USERNAME") TELEGRAM_WEBHOOK_SECRET = os.getenv("TELEGRAM_WEBHOOK_SECRET") GATEWAY_BASE_URL = os.getenv("GATEWAY_BASE_URL", BACKEND_URL) - GATEWAY_BYO_LONGPOLL_ENABLED = ( - os.getenv("GATEWAY_BYO_LONGPOLL_ENABLED", "TRUE").upper() == "TRUE" - ) + GATEWAY_TELEGRAM_INTAKE_MODE = os.getenv( + "GATEWAY_TELEGRAM_INTAKE_MODE", "webhook" + ).lower() + if GATEWAY_TELEGRAM_INTAKE_MODE not in {"webhook", "longpoll", "disabled"}: + raise ValueError( + "GATEWAY_TELEGRAM_INTAKE_MODE must be one of: webhook, longpoll, disabled" + ) # Stripe checkout for pay-as-you-go page packs STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY") diff --git a/surfsense_backend/app/gateway/byo_long_poll.py b/surfsense_backend/app/gateway/byo_long_poll.py index d02f19f95..0be448ae3 100644 --- a/surfsense_backend/app/gateway/byo_long_poll.py +++ b/surfsense_backend/app/gateway/byo_long_poll.py @@ -46,7 +46,7 @@ async def start_byo_long_poll_supervisors() -> None: """Start one BYO long-poll supervisor per active non-system Telegram account.""" global _shutdown_event - if not config.GATEWAY_BYO_LONGPOLL_ENABLED: + if config.GATEWAY_TELEGRAM_INTAKE_MODE != "longpoll": return if _tasks: return diff --git a/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py b/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py index b8212ec9a..951c2d124 100644 --- a/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py +++ b/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py @@ -38,8 +38,8 @@ async def cleanup_supervisors(): @pytest.mark.asyncio -async def test_start_byo_long_poll_noops_when_flag_off(monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", False) +async def test_start_byo_long_poll_noops_when_mode_is_webhook(monkeypatch): + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "webhook") await byo_long_poll.start_byo_long_poll_supervisors() @@ -48,7 +48,7 @@ async def test_start_byo_long_poll_noops_when_flag_off(monkeypatch): @pytest.mark.asyncio async def test_start_byo_long_poll_noops_when_no_byo_accounts(mocker, monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", True) + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll") session = mocker.AsyncMock() session.execute.return_value = ScalarResult([]) monkeypatch.setattr( @@ -64,7 +64,7 @@ async def test_start_byo_long_poll_noops_when_no_byo_accounts(mocker, monkeypatc @pytest.mark.asyncio async def test_start_byo_long_poll_spawns_one_supervisor_per_account(mocker, monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", True) + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll") accounts = [mocker.Mock(id=1), mocker.Mock(id=2)] session = mocker.AsyncMock() session.execute.return_value = ScalarResult(accounts) @@ -108,7 +108,7 @@ async def test_supervisor_retries_after_run_returns(mocker, monkeypatch): @pytest.mark.asyncio async def test_shutdown_cancels_running_supervisors(mocker, monkeypatch): - monkeypatch.setattr(byo_long_poll.config, "GATEWAY_BYO_LONGPOLL_ENABLED", True) + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll") session = mocker.AsyncMock() session.execute.return_value = ScalarResult([mocker.Mock(id=1)]) monkeypatch.setattr( diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index 599cb6238..f977ef11d 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -39,6 +39,39 @@ Complete all the [setup steps](/docs), including: The backend is the core of SurfSense. Follow these steps to set it up: +### Optional: Telegram External Chat Surface + +SurfSense can expose the same canonical chat agent through Telegram. The `external_chat_*` tables store adapter identity, delivery configuration, and durable inbox rows. The actual chat thread and messages remain in `new_chat_threads` and `new_chat_messages`, and all chat-message sources are eligible for Zero replication so a future SurfSense UI layer can render external chat surfaces. The web app initially shows pairing, health, revoke, and resume controls under **User Settings > Messaging Channels**. + +Add these variables to `surfsense_backend/.env` when enabling the Telegram surface: + +```bash +TELEGRAM_SHARED_BOT_TOKEN=123456:bot-token-from-botfather +TELEGRAM_SHARED_BOT_USERNAME=your_bot_username +TELEGRAM_WEBHOOK_SECRET=generate-a-long-random-secret +GATEWAY_BASE_URL=https://api.example.com +GATEWAY_TELEGRAM_INTAKE_MODE=webhook +``` + +`GATEWAY_TELEGRAM_INTAKE_MODE` must be `webhook`, `longpoll`, or `disabled`. Use `webhook` for production/SaaS deployments, `longpoll` only for single-replica self-host installs that cannot expose a public HTTPS webhook, and `disabled` to skip Telegram intake. `TELEGRAM_WEBHOOK_SECRET` must use only `A-Z`, `a-z`, `0-9`, `_`, and `-` characters. `REDIS_APP_URL` is reused for external chat rate limits and per-thread locks. The webhook URL shape is: + +```text +${GATEWAY_BASE_URL}/api/v1/gateway/webhooks/telegram/{account_id} +``` + +After deploying the backend, register the webhook: + +```bash +cd surfsense_backend +uv run python scripts/register_webhook.py +``` + +Keep the FastAPI backend, Celery worker, and Celery beat running. Telegram webhooks write inbound updates into `external_chat_inbound_events`. The FastAPI process owns external chat inbox processing and runs the same SurfSense agent used by web UI chats, then replies back to Telegram. Celery remains maintenance-only for external chat reconciliation, health checks, and retention sweeps. There is no separate gateway service, `SERVICE_ROLE=gateway` process, or Celery agent-processing path. + +For self-hosted BYO Telegram bots without a public HTTPS URL, set `GATEWAY_TELEGRAM_INTAKE_MODE=longpoll`. The FastAPI process starts one lifespan long-poll supervisor per non-system Telegram account and writes updates into the same durable inbox. The FastAPI inbox worker then processes those rows in-process through the canonical `new_chat_*` surface. This fallback is intended for single-replica self-hosted installs. For SaaS-style multi-replica deployments, prefer public webhooks and keep `GATEWAY_TELEGRAM_INTAKE_MODE=webhook` so API replicas skip BYO polling entirely. Telegram does not allow `get_updates()` while a webhook is active, so delete any existing webhook for a BYO bot before relying on long polling. + +When upgrading from an older gateway-runner deployment, apply the rewritten migration 144 external chat schema, deploy the new backend, worker, and beat images, then stop the old `gateway` service. Wait about 30 seconds for any old Telegram `getUpdates` long-poll request to release its advisory lock before starting the new API process. Register each webhook again with the account-id URL above and the per-account `webhook_secret`. If you roll back before using the migration in production, restore the old image and downgrade the schema first. + ### 1. Environment Configuration First, create and configure your environment variables by copying the example file: @@ -350,7 +383,7 @@ redis-cli ping ### 6. Start Celery Worker -In a new terminal window, start the Celery worker to handle background tasks: +In a new terminal window, start the Celery worker to handle background tasks. For external chat surfaces, Celery only runs maintenance tasks; agent turns run inside the FastAPI process. **If using uv:** @@ -358,9 +391,9 @@ In a new terminal window, start the Celery worker to handle background tasks: # Make sure you're in the surfsense_backend directory cd surfsense_backend -# Start Celery worker (consume both default and connectors queues) +# Start Celery worker (consume default, connectors, and external chat maintenance queues) DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" -uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors" +uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway" ``` **If using pip/venv:** @@ -374,9 +407,9 @@ source .venv/bin/activate # Linux/macOS # OR .venv\Scripts\activate # Windows -# Start Celery worker (consume both default and connectors queues) +# Start Celery worker (consume default, connectors, and external chat maintenance queues) DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" -celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors" +celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway" ``` **Optional: Start Flower for monitoring Celery tasks:** From 3faaa25af620d4090a68962e3c64819c7156cd59 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 28 May 2026 13:11:05 +0530 Subject: [PATCH 21/63] refactor(database): update default source values to 'surfsense' for chat threads and messages --- .../alembic/versions/144_add_gateway_tables.py | 4 ++-- surfsense_backend/app/db.py | 8 ++++---- surfsense_backend/app/gateway/agent_invoke.py | 5 ++++- surfsense_web/content/docs/manual-installation.mdx | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/surfsense_backend/alembic/versions/144_add_gateway_tables.py b/surfsense_backend/alembic/versions/144_add_gateway_tables.py index 35eb662c8..91d06f815 100644 --- a/surfsense_backend/alembic/versions/144_add_gateway_tables.py +++ b/surfsense_backend/alembic/versions/144_add_gateway_tables.py @@ -424,7 +424,7 @@ def upgrade() -> None: if not _column_exists(conn, "new_chat_threads", "source"): op.add_column( "new_chat_threads", - sa.Column("source", sa.Text(), nullable=False, server_default="web"), + sa.Column("source", sa.Text(), nullable=False, server_default="surfsense"), ) op.alter_column("new_chat_threads", "source", type_=sa.Text()) if not _column_exists(conn, "new_chat_threads", "external_chat_binding_id"): @@ -454,7 +454,7 @@ def upgrade() -> None: if not _column_exists(conn, "new_chat_messages", "source"): op.add_column( "new_chat_messages", - sa.Column("source", sa.Text(), nullable=False, server_default="web"), + sa.Column("source", sa.Text(), nullable=False, server_default="surfsense"), ) op.alter_column("new_chat_messages", "source", type_=sa.Text()) if not _column_exists(conn, "new_chat_messages", "platform_metadata"): diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index de7792627..e9b301ece 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -696,9 +696,9 @@ class NewChatThread(BaseModel, TimestampMixin): # agent_llm_id changes). Unindexed: all reads are by primary key. pinned_llm_config_id = Column(Integer, nullable=True) - # Surface metadata for web and external chat threads. Zero publishes all - # chat-message sources; the UI can decide which surfaces to render. - source = Column(Text, nullable=False, default="web", server_default="web") + # Surface metadata for first-party SurfSense and external chat threads. + # Zero publishes all chat-message sources; the UI can decide which surfaces to render. + source = Column(Text, nullable=False, default="surfsense", server_default="surfsense") external_chat_binding_id = Column( BigInteger, ForeignKey("external_chat_bindings.id", ondelete="SET NULL"), @@ -786,7 +786,7 @@ class NewChatMessage(BaseModel, TimestampMixin): # Mirrors the parent thread source for publication-level filtering. # This denormalization avoids join-dependent logical replication rules. - source = Column(Text, nullable=False, default="web", server_default="web") + source = Column(Text, nullable=False, default="surfsense", server_default="surfsense") platform_metadata = Column(JSONB, nullable=True) # Relationships diff --git a/surfsense_backend/app/gateway/agent_invoke.py b/surfsense_backend/app/gateway/agent_invoke.py index 39d1fb299..b195f3bce 100644 --- a/surfsense_backend/app/gateway/agent_invoke.py +++ b/surfsense_backend/app/gateway/agent_invoke.py @@ -88,7 +88,10 @@ async def call_agent_for_gateway( await stream.aclose() await session.execute( update(NewChatMessage) - .where(NewChatMessage.thread_id == thread.id, NewChatMessage.source == "web") + .where( + NewChatMessage.thread_id == thread.id, + NewChatMessage.source == "surfsense", + ) .values(source="telegram") ) await session.commit() diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index f977ef11d..602c9a30b 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -41,7 +41,7 @@ The backend is the core of SurfSense. Follow these steps to set it up: ### Optional: Telegram External Chat Surface -SurfSense can expose the same canonical chat agent through Telegram. The `external_chat_*` tables store adapter identity, delivery configuration, and durable inbox rows. The actual chat thread and messages remain in `new_chat_threads` and `new_chat_messages`, and all chat-message sources are eligible for Zero replication so a future SurfSense UI layer can render external chat surfaces. The web app initially shows pairing, health, revoke, and resume controls under **User Settings > Messaging Channels**. +SurfSense can expose the same canonical chat agent through Telegram. The `external_chat_*` tables store adapter identity, delivery configuration, and durable inbox rows. The actual chat thread and messages remain in `new_chat_threads` and `new_chat_messages`, with first-party web/desktop chats marked as `source="surfsense"` and external surfaces marked by platform, such as `source="telegram"`. All chat-message sources are eligible for Zero replication so a future SurfSense UI layer can render external chat surfaces. The web app initially shows pairing, health, revoke, and resume controls under **User Settings > Messaging Channels**. Add these variables to `surfsense_backend/.env` when enabling the Telegram surface: From f6eb955676ae5c8b2d06f7dc03c9c4defd61e045 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:17:28 +0530 Subject: [PATCH 22/63] refactor(gateway): share outbound text splitting --- .../app/gateway/base/formatting.py | 38 +++++++++++++++++++ .../app/gateway/telegram/formatting.py | 20 ++++++---- 2 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 surfsense_backend/app/gateway/base/formatting.py diff --git a/surfsense_backend/app/gateway/base/formatting.py b/surfsense_backend/app/gateway/base/formatting.py new file mode 100644 index 000000000..d0ea6a52d --- /dev/null +++ b/surfsense_backend/app/gateway/base/formatting.py @@ -0,0 +1,38 @@ +"""Provider-neutral message formatting helpers.""" + +from __future__ import annotations + +MAX_GATEWAY_TEXT_CHARS = 4096 + + +def split_text_message( + text: str, + *, + max_chars: int = MAX_GATEWAY_TEXT_CHARS, +) -> list[str]: + """Split outbound text at readable boundaries without exceeding platform caps.""" + if not text: + return [""] + + chunks: list[str] = [] + remaining = text + while remaining: + if len(remaining) <= max_chars: + chunks.append(remaining) + break + + candidate = remaining[:max_chars] + boundary = max( + candidate.rfind("\n\n"), + candidate.rfind("\n"), + candidate.rfind(". "), + candidate.rfind(" "), + ) + if boundary <= max(200, max_chars // 2): + boundary = max_chars + split_at = boundary + (2 if candidate[boundary : boundary + 2] == ". " else 1) + chunk = remaining[:split_at].rstrip() + chunks.append(chunk or remaining[:max_chars]) + remaining = remaining[split_at:].lstrip() + + return chunks diff --git a/surfsense_backend/app/gateway/telegram/formatting.py b/surfsense_backend/app/gateway/telegram/formatting.py index ecc7064bd..a9bb73ed5 100644 --- a/surfsense_backend/app/gateway/telegram/formatting.py +++ b/surfsense_backend/app/gateway/telegram/formatting.py @@ -4,6 +4,8 @@ from __future__ import annotations import re +from app.gateway.base.formatting import split_text_message + MARKDOWN_V2_RESERVED = r"_*[]()~`>#+-=|{}.!" MAX_TELEGRAM_MESSAGE_UNITS = 4096 @@ -43,13 +45,15 @@ def chunk_message( max_units: int = MAX_TELEGRAM_MESSAGE_UNITS, ) -> list[str]: """Split a Telegram message at paragraph/sentence boundaries.""" - if not text: - return [""] + if max_units == MAX_TELEGRAM_MESSAGE_UNITS: + if not text: + return [""] - chunks: list[str] = [] - remaining = text - while remaining: - chunk, remaining = _split_at_boundary(remaining, max_units) - chunks.append(chunk) - return chunks + chunks: list[str] = [] + remaining = text + while remaining: + chunk, remaining = _split_at_boundary(remaining, max_units) + chunks.append(chunk) + return chunks + return split_text_message(text, max_chars=max_units) From e953be5e99da6734919a778a37707907a20f59f5 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:17:41 +0530 Subject: [PATCH 23/63] refactor(gateway): abstract platform command handling --- .../app/gateway/base/commands.py | 41 +++++++++++++++++++ .../app/gateway/telegram/commands.py | 38 ++++++++++++++--- 2 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 surfsense_backend/app/gateway/base/commands.py diff --git a/surfsense_backend/app/gateway/base/commands.py b/surfsense_backend/app/gateway/base/commands.py new file mode 100644 index 000000000..ea5d09e20 --- /dev/null +++ b/surfsense_backend/app/gateway/base/commands.py @@ -0,0 +1,41 @@ +"""Provider-neutral command hooks for external chat gateways.""" + +from __future__ import annotations + +from app.gateway.base.adapter import BasePlatformAdapter, ParsedInboundEvent + + +def command_name(text: str | None) -> str | None: + if not text or not text.startswith("/"): + return None + return text.split(maxsplit=1)[0].split("@", 1)[0].lower() + + +class BaseGatewayCommands: + """Default command behavior for platforms without slash-command onboarding.""" + + async def handle_start_command( + self, + *, + session, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + ) -> bool: + return False + + async def handle_help_command( + self, + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + ) -> bool: + return False + + async def send_unbound_onboarding( + self, + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + dashboard_url: str, + ) -> None: + return None diff --git a/surfsense_backend/app/gateway/telegram/commands.py b/surfsense_backend/app/gateway/telegram/commands.py index bc4a64377..903330fd8 100644 --- a/surfsense_backend/app/gateway/telegram/commands.py +++ b/surfsense_backend/app/gateway/telegram/commands.py @@ -3,6 +3,7 @@ from __future__ import annotations from app.gateway.base.adapter import ParsedInboundEvent +from app.gateway.base.commands import BaseGatewayCommands from app.gateway.pairing import redeem_pairing_code from app.gateway.ratelimit import acquire_token from app.gateway.telegram.adapter import TelegramAdapter @@ -15,12 +16,6 @@ HELP_TEXT = ( ) -def command_name(text: str | None) -> str | None: - if not text or not text.startswith("/"): - return None - return text.split(maxsplit=1)[0].split("@", 1)[0].lower() - - async def handle_start_command( *, session, @@ -89,3 +84,34 @@ async def send_unbound_onboarding( ), ) + +class TelegramGatewayCommands(BaseGatewayCommands): + async def handle_start_command( + self, + *, + session, + adapter: TelegramAdapter, + event: ParsedInboundEvent, + ) -> bool: + return await handle_start_command(session=session, adapter=adapter, event=event) + + async def handle_help_command( + self, + *, + adapter: TelegramAdapter, + event: ParsedInboundEvent, + ) -> bool: + return await handle_help_command(adapter=adapter, event=event) + + async def send_unbound_onboarding( + self, + *, + adapter: TelegramAdapter, + event: ParsedInboundEvent, + dashboard_url: str, + ) -> None: + await send_unbound_onboarding( + adapter=adapter, + event=event, + dashboard_url=dashboard_url, + ) \ No newline at end of file From 5048b0fd7c7667887d885aee89edd85d57a7cc24 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:17:57 +0530 Subject: [PATCH 24/63] refactor(gateway): route inbound events through platform bundles --- surfsense_backend/app/gateway/agent_invoke.py | 10 +- .../app/gateway/inbox_processor.py | 79 +++++++------ surfsense_backend/app/gateway/registry.py | 111 ++++++++++++++++++ 3 files changed, 160 insertions(+), 40 deletions(-) create mode 100644 surfsense_backend/app/gateway/registry.py diff --git a/surfsense_backend/app/gateway/agent_invoke.py b/surfsense_backend/app/gateway/agent_invoke.py index b195f3bce..7a2219b1d 100644 --- a/surfsense_backend/app/gateway/agent_invoke.py +++ b/surfsense_backend/app/gateway/agent_invoke.py @@ -11,10 +11,9 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.db import ExternalChatBinding, NewChatMessage from app.gateway.auth_invariant import assert_authorization_invariant -from app.gateway.base.translator import GatewayStreamEvent +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent from app.gateway.bindings import get_or_create_thread_for_binding from app.gateway.hitl_filter import DEFAULT_HITL_TOOL_NAMES -from app.gateway.telegram.translator import TelegramStreamTranslator from app.gateway.thread_lock import acquire_thread_lock, release_thread_lock from app.observability.metrics import record_gateway_turn_latency from app.tasks.chat.stream_new_chat import stream_new_chat @@ -58,7 +57,8 @@ async def call_agent_for_gateway( session: AsyncSession, binding: ExternalChatBinding, user_text: str, - translator: TelegramStreamTranslator, + translator: BaseStreamTranslator, + platform_label: str = "telegram", request_id: str | None = None, ) -> None: user = await assert_authorization_invariant(session, binding) @@ -92,10 +92,10 @@ async def call_agent_for_gateway( NewChatMessage.thread_id == thread.id, NewChatMessage.source == "surfsense", ) - .values(source="telegram") + .values(source=platform_label) ) await session.commit() - record_gateway_turn_latency(0, platform="telegram") + record_gateway_turn_latency(0, platform=platform_label) finally: release_thread_lock(thread.id) diff --git a/surfsense_backend/app/gateway/inbox_processor.py b/surfsense_backend/app/gateway/inbox_processor.py index c40a6c47c..bdf768d61 100644 --- a/surfsense_backend/app/gateway/inbox_processor.py +++ b/surfsense_backend/app/gateway/inbox_processor.py @@ -15,26 +15,19 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from app.config import config from app.db import ( - ExternalChatBindingState, + ExternalChatAccount, ExternalChatBinding, + ExternalChatBindingState, ExternalChatEventStatus, ExternalChatInboundEvent, ExternalChatPeerKind, - ExternalChatAccount, NewChatThread, async_session_maker, ) -from app.gateway.accounts import account_token from app.gateway.agent_invoke import call_agent_for_gateway +from app.gateway.base.commands import command_name from app.gateway.bindings import get_or_create_thread_for_binding -from app.gateway.telegram.adapter import TelegramAdapter -from app.gateway.telegram.commands import ( - command_name, - handle_help_command, - handle_start_command, - send_unbound_onboarding, -) -from app.gateway.telegram.translator import TelegramStreamTranslator +from app.gateway.registry import resolve_platform_bundle from app.observability.metrics import record_gateway_inbox_processed logger = logging.getLogger(__name__) @@ -150,14 +143,15 @@ async def _dispatch_inbound_event( await session.commit() return - token = account_token(account) - if not token: + try: + bundle = resolve_platform_bundle(account) + except RuntimeError as exc: event.status = ExternalChatEventStatus.FAILED - event.last_error = "missing_telegram_token" + event.last_error = str(exc) await session.commit() return - adapter = TelegramAdapter(token) + adapter = bundle.adapter parsed = adapter.parse_inbound(event.raw_payload or {}) if parsed.external_peer_id is None: event.status = ExternalChatEventStatus.IGNORED @@ -179,7 +173,8 @@ async def _dispatch_inbound_event( binding = result.scalars().first() if parsed.external_peer_kind != ExternalChatPeerKind.DIRECT.value: - await adapter.leave_chat(external_peer_id=parsed.external_peer_id) + if hasattr(adapter, "leave_chat"): + await adapter.leave_chat(external_peer_id=parsed.external_peer_id) event.status = ExternalChatEventStatus.IGNORED event.last_error = "group_rejected" await session.commit() @@ -187,7 +182,7 @@ async def _dispatch_inbound_event( cmd = command_name(parsed.text) if cmd == "/start": - handled = await handle_start_command( + handled = await bundle.commands.handle_start_command( session=session, adapter=adapter, event=parsed ) await session.commit() @@ -195,23 +190,39 @@ async def _dispatch_inbound_event( return if binding is None: - await send_unbound_onboarding( - adapter=adapter, - event=parsed, - dashboard_url=_dashboard_url(), - ) - event.status = ExternalChatEventStatus.IGNORED - event.last_error = "unbound_chat" - await session.commit() - return + if bundle.auto_bind_owner and account.owner_user_id and account.owner_search_space_id: + binding = ExternalChatBinding( + account_id=account.id, + user_id=account.owner_user_id, + search_space_id=account.owner_search_space_id, + state=ExternalChatBindingState.BOUND, + external_peer_id=parsed.external_peer_id, + external_peer_kind=parsed.external_peer_kind, + external_display_name=parsed.display_name, + external_username=parsed.username, + external_metadata=parsed.metadata, + ) + session.add(binding) + await session.flush() + else: + await bundle.commands.send_unbound_onboarding( + adapter=adapter, + event=parsed, + dashboard_url=_dashboard_url(), + ) + event.status = ExternalChatEventStatus.IGNORED + event.last_error = "unbound_chat" + await session.commit() + return event.external_chat_binding_id = binding.id if cmd == "/help": - await handle_help_command(adapter=adapter, event=parsed) - event.status = ExternalChatEventStatus.PROCESSED - await session.commit() - return + handled = await bundle.commands.handle_help_command(adapter=adapter, event=parsed) + if handled: + event.status = ExternalChatEventStatus.PROCESSED + await session.commit() + return if cmd == "/new": binding.new_chat_thread_id = None await adapter.send_message( @@ -231,21 +242,19 @@ async def _dispatch_inbound_event( thread = await get_or_create_thread_for_binding(session, binding) await session.commit() - translator = TelegramStreamTranslator( - adapter=adapter, - external_peer_id=parsed.external_peer_id, - ) + translator = bundle.translator_factory(adapter, parsed) await call_agent_for_gateway( session=session, binding=binding, user_text=parsed.text, translator=translator, + platform_label=bundle.platform_label, request_id=event.request_id or f"gateway:{inbox_id}", ) thread = await session.get(NewChatThread, thread.id) if thread is not None: - thread.source = "telegram" + thread.source = bundle.platform_label await session.commit() diff --git a/surfsense_backend/app/gateway/registry.py b/surfsense_backend/app/gateway/registry.py new file mode 100644 index 000000000..db334b7f1 --- /dev/null +++ b/surfsense_backend/app/gateway/registry.py @@ -0,0 +1,111 @@ +"""Resolve gateway platform implementations from account rows.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +from app.db import ExternalChatAccount, ExternalChatAccountMode, ExternalChatPlatform +from app.gateway.accounts import account_token +from app.gateway.base.adapter import BasePlatformAdapter, ParsedInboundEvent +from app.gateway.base.commands import BaseGatewayCommands +from app.gateway.base.translator import BaseStreamTranslator +from app.gateway.telegram.adapter import TelegramAdapter +from app.gateway.telegram.commands import TelegramGatewayCommands +from app.gateway.telegram.translator import TelegramStreamTranslator + +TranslatorFactory = Callable[ + [BasePlatformAdapter, ParsedInboundEvent], + BaseStreamTranslator, +] + + +@dataclass(frozen=True) +class PlatformBundle: + adapter: BasePlatformAdapter + translator_factory: TranslatorFactory + platform_label: str + commands: BaseGatewayCommands + auto_bind_owner: bool = False + + +def _telegram_translator_factory( + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> BaseStreamTranslator: + if event.external_peer_id is None: + raise RuntimeError("missing_external_peer_id") + return TelegramStreamTranslator( + adapter=adapter, # type: ignore[arg-type] + external_peer_id=event.external_peer_id, + ) + + +def _whatsapp_cloud_translator_factory( + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> BaseStreamTranslator: + if event.external_peer_id is None: + raise RuntimeError("missing_external_peer_id") + from app.gateway.whatsapp.translator import WhatsAppCloudStreamTranslator + + return WhatsAppCloudStreamTranslator( + adapter=adapter, + external_peer_id=event.external_peer_id, + inbound_message_id=event.external_message_id, + ) + + +def _whatsapp_baileys_translator_factory( + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> BaseStreamTranslator: + if event.external_peer_id is None: + raise RuntimeError("missing_external_peer_id") + from app.gateway.whatsapp.translator_baileys import WhatsAppBaileysStreamTranslator + + return WhatsAppBaileysStreamTranslator( + adapter=adapter, + external_peer_id=event.external_peer_id, + ) + + +def resolve_platform_bundle(account: ExternalChatAccount) -> PlatformBundle: + if account.platform == ExternalChatPlatform.TELEGRAM: + token = account_token(account) + if not token: + raise RuntimeError("missing_telegram_token") + return PlatformBundle( + adapter=TelegramAdapter(token), + translator_factory=_telegram_translator_factory, + platform_label="telegram", + commands=TelegramGatewayCommands(), + ) + + if account.platform == ExternalChatPlatform.WHATSAPP: + if account.mode == ExternalChatAccountMode.CLOUD_SHARED: + from app.gateway.whatsapp.adapter_cloud import WhatsAppCloudAdapter + from app.gateway.whatsapp.commands import WhatsAppGatewayCommands + from app.gateway.whatsapp.credentials import ( + load_system_whatsapp_credentials, + ) + + return PlatformBundle( + adapter=WhatsAppCloudAdapter(load_system_whatsapp_credentials()), + translator_factory=_whatsapp_cloud_translator_factory, + platform_label="whatsapp", + commands=WhatsAppGatewayCommands(), + auto_bind_owner=False, + ) + if account.mode == ExternalChatAccountMode.SELF_HOST_BYO: + from app.gateway.whatsapp.adapter_baileys import WhatsAppBaileysAdapter + + return PlatformBundle( + adapter=WhatsAppBaileysAdapter(), + translator_factory=_whatsapp_baileys_translator_factory, + platform_label="whatsapp", + commands=BaseGatewayCommands(), + auto_bind_owner=True, + ) + + raise RuntimeError(f"unsupported_gateway_platform:{account.platform.value}:{account.mode.value}") From a6b2882275436037a1cea902dd12df11cdf812ef Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:18:11 +0530 Subject: [PATCH 25/63] feat(gateway): add WhatsApp gateway configuration --- surfsense_backend/.env.example | 12 +++++++++ surfsense_backend/app/config/__init__.py | 17 +++++++++++++ surfsense_backend/app/gateway/accounts.py | 30 ++++++++++++++++++++++- 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 340e5b51a..bc96cc948 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -24,6 +24,18 @@ TELEGRAM_WEBHOOK_SECRET= GATEWAY_BASE_URL=http://localhost:8000 GATEWAY_TELEGRAM_INTAKE_MODE=webhook +# WhatsApp Gateway +# GATEWAY_WHATSAPP_INTAKE_MODE: cloud for Meta Cloud API, baileys for self-hosted bridge, disabled to skip WhatsApp intake +GATEWAY_WHATSAPP_INTAKE_MODE=disabled +WHATSAPP_SHARED_BUSINESS_TOKEN= +WHATSAPP_SHARED_PHONE_NUMBER_ID= +WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER= +WHATSAPP_SHARED_WABA_ID= +WHATSAPP_GRAPH_API_VERSION=v25.0 +WHATSAPP_WEBHOOK_VERIFY_TOKEN= +WHATSAPP_WEBHOOK_APP_SECRET= +WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:3000 + # Platform Web Search (SearXNG) # Set this to enable built-in web search. Docker Compose sets it automatically. # Only uncomment if running the backend outside Docker (e.g. uvicorn on host). diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index c77b95fde..afccb190b 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -553,6 +553,23 @@ class Config: raise ValueError( "GATEWAY_TELEGRAM_INTAKE_MODE must be one of: webhook, longpoll, disabled" ) + WHATSAPP_SHARED_BUSINESS_TOKEN = os.getenv("WHATSAPP_SHARED_BUSINESS_TOKEN") + WHATSAPP_SHARED_PHONE_NUMBER_ID = os.getenv("WHATSAPP_SHARED_PHONE_NUMBER_ID") + WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER = os.getenv( + "WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER" + ) + WHATSAPP_SHARED_WABA_ID = os.getenv("WHATSAPP_SHARED_WABA_ID") + WHATSAPP_GRAPH_API_VERSION = os.getenv("WHATSAPP_GRAPH_API_VERSION", "v25.0") + WHATSAPP_WEBHOOK_VERIFY_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFY_TOKEN") + WHATSAPP_WEBHOOK_APP_SECRET = os.getenv("WHATSAPP_WEBHOOK_APP_SECRET") + WHATSAPP_BRIDGE_URL = os.getenv("WHATSAPP_BRIDGE_URL", "http://whatsapp-bridge:3000") + GATEWAY_WHATSAPP_INTAKE_MODE = os.getenv( + "GATEWAY_WHATSAPP_INTAKE_MODE", "disabled" + ).lower() + if GATEWAY_WHATSAPP_INTAKE_MODE not in {"cloud", "baileys", "disabled"}: + raise ValueError( + "GATEWAY_WHATSAPP_INTAKE_MODE must be one of: cloud, baileys, disabled" + ) # Stripe checkout for pay-as-you-go page packs STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY") diff --git a/surfsense_backend/app/gateway/accounts.py b/surfsense_backend/app/gateway/accounts.py index 3e0d86e46..7379336a7 100644 --- a/surfsense_backend/app/gateway/accounts.py +++ b/surfsense_backend/app/gateway/accounts.py @@ -7,10 +7,10 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.config import config from app.db import ( + ExternalChatAccount, ExternalChatAccountMode, ExternalChatHealthStatus, ExternalChatPlatform, - ExternalChatAccount, ) from app.utils.oauth_security import TokenEncryption @@ -50,3 +50,31 @@ async def get_or_create_system_telegram_account( await session.flush() return account + +async def get_or_create_system_whatsapp_account( + session: AsyncSession, +) -> ExternalChatAccount: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.is_system_account.is_(True), + ) + ) + account = result.scalars().first() + if account is not None: + return account + account = ExternalChatAccount( + platform=ExternalChatPlatform.WHATSAPP, + mode=ExternalChatAccountMode.CLOUD_SHARED, + is_system_account=True, + cursor_state={ + "phone_number_id": config.WHATSAPP_SHARED_PHONE_NUMBER_ID, + "display_phone_number": config.WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER, + "waba_id": config.WHATSAPP_SHARED_WABA_ID, + }, + health_status=ExternalChatHealthStatus.UNKNOWN, + ) + session.add(account) + await session.flush() + return account + From daa123832e75e20adb7b8226666a2e43c32e755f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:18:28 +0530 Subject: [PATCH 26/63] feat(gateway): add WhatsApp Cloud adapter --- .../app/gateway/whatsapp/__init__.py | 1 + .../app/gateway/whatsapp/adapter_cloud.py | 149 ++++++++++++++++++ .../app/gateway/whatsapp/client_cloud.py | 99 ++++++++++++ .../app/gateway/whatsapp/credentials.py | 31 ++++ 4 files changed, 280 insertions(+) create mode 100644 surfsense_backend/app/gateway/whatsapp/__init__.py create mode 100644 surfsense_backend/app/gateway/whatsapp/adapter_cloud.py create mode 100644 surfsense_backend/app/gateway/whatsapp/client_cloud.py create mode 100644 surfsense_backend/app/gateway/whatsapp/credentials.py diff --git a/surfsense_backend/app/gateway/whatsapp/__init__.py b/surfsense_backend/app/gateway/whatsapp/__init__.py new file mode 100644 index 000000000..5c54d2caf --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/__init__.py @@ -0,0 +1 @@ +"""WhatsApp gateway implementations.""" diff --git a/surfsense_backend/app/gateway/whatsapp/adapter_cloud.py b/surfsense_backend/app/gateway/whatsapp/adapter_cloud.py new file mode 100644 index 000000000..f247db692 --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/adapter_cloud.py @@ -0,0 +1,149 @@ +"""WhatsApp Cloud API platform adapter.""" + +from __future__ import annotations + +from typing import Any + +from app.gateway.base.adapter import ( + BasePlatformAdapter, + ParsedInboundEvent, + PlatformSendResult, +) +from app.gateway.whatsapp.client_cloud import WhatsAppCloudClient +from app.gateway.whatsapp.credentials import WhatsAppCredentials + + +class WhatsAppCloudAdapter(BasePlatformAdapter): + platform = "whatsapp" + + def __init__(self, credentials: WhatsAppCredentials) -> None: + self.credentials = credentials + self.client = WhatsAppCloudClient( + business_token=credentials["business_token"], + phone_number_id=credentials["phone_number_id"], + api_version=credentials.get("api_version"), + ) + + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + message = _first_message(raw_payload) + if message is None: + return ParsedInboundEvent( + platform=self.platform, + event_kind="other", + external_peer_id=None, + external_peer_kind="unknown", + external_message_id=None, + external_user_id=None, + text=None, + raw_payload=raw_payload, + ) + + contact = _first_contact(raw_payload, message.get("from")) + text = _message_text(message) + wa_id = str(message.get("from") or "") + return ParsedInboundEvent( + platform=self.platform, + event_kind=str(message.get("type") or "message"), + external_peer_id=wa_id or None, + external_peer_kind="direct", + external_message_id=str(message.get("id")) if message.get("id") else None, + external_user_id=wa_id or None, + text=text, + raw_payload=raw_payload, + display_name=(contact.get("profile") or {}).get("name"), + username=None, + metadata={ + "phone_number_id": _metadata(raw_payload).get("phone_number_id"), + "display_phone_number": _metadata(raw_payload).get("display_phone_number"), + "timestamp": message.get("timestamp"), + "message_type": message.get("type"), + }, + ) + + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + return await self.client.send_text( + to=external_peer_id, + text=text, + reply_to_message_id=reply_to_message_id, + ) + + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + raise NotImplementedError("WhatsApp Cloud API does not support message edits") + + async def send_typing_indicator(self, *, inbound_message_id: str) -> None: + await self.client.send_typing_indicator(message_id=inbound_message_id) + + async def validate_credentials(self) -> dict[str, Any]: + return await self.client.validate() + + +def _changes(raw_payload: dict[str, Any]) -> list[dict[str, Any]]: + changes: list[dict[str, Any]] = [] + for entry in raw_payload.get("entry") or []: + if isinstance(entry, dict): + changes.extend( + change for change in (entry.get("changes") or []) if isinstance(change, dict) + ) + return changes + + +def _first_message(raw_payload: dict[str, Any]) -> dict[str, Any] | None: + for change in _changes(raw_payload): + value = change.get("value") or {} + messages = value.get("messages") or [] + if messages and isinstance(messages[0], dict): + return messages[0] + if "message" in raw_payload and isinstance(raw_payload["message"], dict): + return raw_payload["message"] + return None + + +def _first_contact( + raw_payload: dict[str, Any], + wa_id: object, +) -> dict[str, Any]: + for change in _changes(raw_payload): + value = change.get("value") or {} + for contact in value.get("contacts") or []: + if isinstance(contact, dict) and ( + wa_id is None or str(contact.get("wa_id")) == str(wa_id) + ): + return contact + return {} + + +def _metadata(raw_payload: dict[str, Any]) -> dict[str, Any]: + for change in _changes(raw_payload): + value = change.get("value") or {} + metadata = value.get("metadata") + if isinstance(metadata, dict): + return metadata + return {} + + +def _message_text(message: dict[str, Any]) -> str | None: + message_type = message.get("type") + if message_type == "text": + return (message.get("text") or {}).get("body") + if message_type == "button": + return (message.get("button") or {}).get("text") + if message_type == "interactive": + interactive = message.get("interactive") or {} + button_reply = interactive.get("button_reply") or {} + list_reply = interactive.get("list_reply") or {} + return button_reply.get("title") or list_reply.get("title") + return None diff --git a/surfsense_backend/app/gateway/whatsapp/client_cloud.py b/surfsense_backend/app/gateway/whatsapp/client_cloud.py new file mode 100644 index 000000000..e39e022aa --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/client_cloud.py @@ -0,0 +1,99 @@ +"""Small httpx wrapper for the WhatsApp Cloud API.""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from app.config import config +from app.gateway.base.adapter import PlatformSendResult +from app.gateway.ratelimit import wait_for_token +from app.observability.metrics import record_gateway_rate_limit_hit + + +class WhatsAppCloudClient: + def __init__( + self, + *, + business_token: str, + phone_number_id: str, + api_version: str | None = None, + ) -> None: + self.business_token = business_token + self.phone_number_id = phone_number_id + self.api_version = api_version or config.WHATSAPP_GRAPH_API_VERSION + self.base_url = f"https://graph.facebook.com/{self.api_version}" + + async def send_text( + self, + *, + to: str, + text: str, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + payload: dict[str, Any] = { + "messaging_product": "whatsapp", + "recipient_type": "individual", + "to": to, + "type": "text", + "text": {"preview_url": True, "body": text}, + } + if reply_to_message_id: + payload["context"] = {"message_id": reply_to_message_id} + data = await self._post(f"/{self.phone_number_id}/messages", json=payload) + message_id = str((data.get("messages") or [{}])[0].get("id") or "") + return PlatformSendResult(external_message_id=message_id, raw_response=data) + + async def send_typing_indicator(self, *, message_id: str) -> dict[str, Any]: + payload = { + "messaging_product": "whatsapp", + "status": "read", + "message_id": message_id, + "typing_indicator": {"type": "text"}, + } + return await self._post(f"/{self.phone_number_id}/messages", json=payload) + + async def validate(self) -> dict[str, Any]: + return await self._get( + f"/{self.phone_number_id}", + params={ + "fields": "verified_name,quality_rating,account_review_status,display_phone_number" + }, + ) + + async def _post(self, path: str, *, json: dict[str, Any]) -> dict[str, Any]: + await self._throttle() + async with httpx.AsyncClient(timeout=20) as client: + response = await client.post( + f"{self.base_url}{path}", + headers={"Authorization": f"Bearer {self.business_token}"}, + json=json, + ) + response.raise_for_status() + return response.json() + + async def _get( + self, + path: str, + *, + params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + await self._throttle() + async with httpx.AsyncClient(timeout=20) as client: + response = await client.get( + f"{self.base_url}{path}", + headers={"Authorization": f"Bearer {self.business_token}"}, + params=params, + ) + response.raise_for_status() + return response.json() + + async def _throttle(self) -> None: + wait_ms = await wait_for_token( + f"wa:phone:{self.phone_number_id}", + capacity=10, + refill_per_sec=10.0, + ) + if wait_ms: + record_gateway_rate_limit_hit(bucket="wa:phone") diff --git a/surfsense_backend/app/gateway/whatsapp/credentials.py b/surfsense_backend/app/gateway/whatsapp/credentials.py new file mode 100644 index 000000000..fba79d470 --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/credentials.py @@ -0,0 +1,31 @@ +"""Credential helpers for WhatsApp gateway accounts.""" + +from __future__ import annotations + +from typing import TypedDict + +from app.config import config + + +class WhatsAppCredentials(TypedDict, total=False): + business_token: str + waba_id: str + phone_number_id: str + business_id: str + registration_pin: str + api_version: str + + +def load_system_whatsapp_credentials() -> WhatsAppCredentials: + if not ( + config.WHATSAPP_SHARED_BUSINESS_TOKEN + and config.WHATSAPP_SHARED_PHONE_NUMBER_ID + ): + raise RuntimeError("whatsapp_system_credentials_not_configured") + + return { + "business_token": config.WHATSAPP_SHARED_BUSINESS_TOKEN, + "phone_number_id": config.WHATSAPP_SHARED_PHONE_NUMBER_ID, + "waba_id": config.WHATSAPP_SHARED_WABA_ID, + "api_version": config.WHATSAPP_GRAPH_API_VERSION, + } From 3d9620275b808447bfe577ff483ca10a5e66ec00 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:18:45 +0530 Subject: [PATCH 27/63] feat(gateway): handle WhatsApp Cloud conversations --- .../app/gateway/whatsapp/commands.py | 123 ++++++++++++++++++ .../app/gateway/whatsapp/translator.py | 90 +++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 surfsense_backend/app/gateway/whatsapp/commands.py create mode 100644 surfsense_backend/app/gateway/whatsapp/translator.py diff --git a/surfsense_backend/app/gateway/whatsapp/commands.py b/surfsense_backend/app/gateway/whatsapp/commands.py new file mode 100644 index 000000000..28b765347 --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/commands.py @@ -0,0 +1,123 @@ +"""WhatsApp command handlers.""" + +from __future__ import annotations + +from app.gateway.base.adapter import BasePlatformAdapter, ParsedInboundEvent +from app.gateway.base.commands import BaseGatewayCommands +from app.gateway.pairing import redeem_pairing_code +from app.gateway.ratelimit import acquire_token + +HELP_TEXT = ( + "SurfSense WhatsApp commands:\n" + "/start - pair this chat\n" + "/new - start a fresh conversation\n" + "/help - show this help" +) + + +async def handle_start_command( + *, + session, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> bool: + text = event.text or "" + parts = text.split(maxsplit=1) + if len(parts) != 2 or not event.external_peer_id: + await adapter.send_message( + external_peer_id=event.external_peer_id or "", + text=( + "Generate a pairing code in SurfSense Settings > Messaging Channels, " + "then send /start CODE here." + ), + ) + return True + + binding = await redeem_pairing_code( + session, + code=parts[1].strip(), + external_peer_id=event.external_peer_id, + external_peer_kind=event.external_peer_kind, + external_display_name=event.display_name, + external_username=event.username, + external_metadata=event.metadata, + ) + if binding is None: + await adapter.send_message( + external_peer_id=event.external_peer_id, + text="That pairing code is invalid or expired. Generate a new code in SurfSense.", + ) + return True + + await adapter.send_message( + external_peer_id=event.external_peer_id, + text="SurfSense is connected. Send a message here to chat with your agent.", + ) + return True + + +async def handle_help_command( + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> bool: + if not event.external_peer_id: + return True + await adapter.send_message(external_peer_id=event.external_peer_id, text=HELP_TEXT) + return True + + +async def send_unbound_onboarding( + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + dashboard_url: str, +) -> None: + if not event.external_peer_id: + return + wait_ms = await acquire_token( + f"wa:onboarded:{event.external_peer_id}", + capacity=1, + refill_per_sec=1 / 3600, + ) + if wait_ms > 0: + return + await adapter.send_message( + external_peer_id=event.external_peer_id, + text=( + "Hi! To use SurfSense via WhatsApp, generate a pairing code at " + f"{dashboard_url} and send /start CODE here." + ), + ) + + +class WhatsAppGatewayCommands(BaseGatewayCommands): + async def handle_start_command( + self, + *, + session, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + ) -> bool: + return await handle_start_command(session=session, adapter=adapter, event=event) + + async def handle_help_command( + self, + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + ) -> bool: + return await handle_help_command(adapter=adapter, event=event) + + async def send_unbound_onboarding( + self, + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + dashboard_url: str, + ) -> None: + await send_unbound_onboarding( + adapter=adapter, + event=event, + dashboard_url=dashboard_url, + ) diff --git a/surfsense_backend/app/gateway/whatsapp/translator.py b/surfsense_backend/app/gateway/whatsapp/translator.py new file mode 100644 index 000000000..deef8b452 --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/translator.py @@ -0,0 +1,90 @@ +"""Translate agent stream events into WhatsApp Cloud API messages.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator + +from app.gateway.base.adapter import BasePlatformAdapter, PlatformSendResult +from app.gateway.base.formatting import split_text_message +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.whatsapp.adapter_cloud import WhatsAppCloudAdapter +from app.observability.metrics import ( + record_gateway_hitl_aborted, + record_gateway_outbound, +) + +logger = logging.getLogger(__name__) + +HITL_UNSUPPORTED_MESSAGE = ( + "This action requires approval and is not yet supported from WhatsApp. " + "Try again with a different request." +) + + +class WhatsAppCloudStreamTranslator(BaseStreamTranslator): + def __init__( + self, + *, + adapter: BasePlatformAdapter, + external_peer_id: str, + inbound_message_id: str | None = None, + ) -> None: + self.adapter = adapter + self.external_peer_id = external_peer_id + self.inbound_message_id = inbound_message_id + self._buffer = "" + self._typing_sent = False + + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + async for event in events: + if event.type in {"text-delta", "text_delta", "text"}: + if not self._typing_sent: + await self._send_typing_indicator() + self._buffer += str(event.data.get("text") or event.data.get("delta") or "") + elif event.type in {"data-interrupt-request", "interrupt"}: + await self._handle_hitl_interrupt() + return + elif event.type in {"finish", "done"}: + break + + await self._flush_final() + + async def _flush_final(self) -> None: + if not self._buffer: + return + for chunk in split_text_message(self._buffer): + await self._send_text(chunk) + + async def _send_typing_indicator(self) -> None: + self._typing_sent = True + if not self.inbound_message_id: + return + if not isinstance(self.adapter, WhatsAppCloudAdapter): + return + try: + await self.adapter.send_typing_indicator( + inbound_message_id=self.inbound_message_id + ) + record_gateway_outbound(platform="whatsapp", kind="typing", status="sent") + except Exception: + logger.debug("WhatsApp typing indicator failed", exc_info=True) + record_gateway_outbound(platform="whatsapp", kind="typing", status="failed") + + async def _send_text(self, text: str) -> PlatformSendResult: + try: + result = await self.adapter.send_message( + external_peer_id=self.external_peer_id, + text=text, + ) + except Exception: + record_gateway_outbound(platform="whatsapp", kind="send", status="failed") + raise + record_gateway_outbound(platform="whatsapp", kind="send", status="sent") + return result + + async def _handle_hitl_interrupt(self) -> None: + if self._buffer: + await self._flush_final() + await self._send_text(HITL_UNSUPPORTED_MESSAGE) + record_gateway_hitl_aborted(platform="whatsapp") From dbd996621909ca108b6054eec01bd6f85c7e8f9a Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:18:58 +0530 Subject: [PATCH 28/63] feat(gateway): add WhatsApp Cloud webhook intake --- .../routes/gateway_whatsapp_webhook_routes.py | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py diff --git a/surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py b/surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py new file mode 100644 index 000000000..bb7b49712 --- /dev/null +++ b/surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py @@ -0,0 +1,205 @@ +"""WhatsApp Cloud API webhook routes.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import logging +import uuid +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from sqlalchemy.ext.asyncio import AsyncSession +from starlette.responses import Response + +from app.config import config +from app.db import ( + ExternalChatHealthStatus, + ExternalChatPlatform, + get_async_session, +) +from app.gateway.accounts import get_or_create_system_whatsapp_account +from app.gateway.inbox import persist_inbound_event +from app.observability.metrics import ( + record_gateway_inbox_write, + record_gateway_outbound, + record_gateway_webhook_parse_error, +) + +router = APIRouter(prefix="/gateway/webhooks/whatsapp", tags=["gateway"]) +logger = logging.getLogger(__name__) + + +def _ensure_whatsapp_enabled() -> None: + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "disabled": + raise HTTPException(status_code=404, detail="WhatsApp gateway is disabled") + + +@router.get("") +async def verify_whatsapp_webhook( + hub_mode: str = Query(alias="hub.mode"), + hub_verify_token: str = Query(alias="hub.verify_token"), + hub_challenge: str = Query(alias="hub.challenge"), +) -> Response: + _ensure_whatsapp_enabled() + if ( + hub_mode == "subscribe" + and config.WHATSAPP_WEBHOOK_VERIFY_TOKEN + and hmac.compare_digest(hub_verify_token, config.WHATSAPP_WEBHOOK_VERIFY_TOKEN) + ): + return Response(content=hub_challenge, media_type="text/plain") + raise HTTPException(status_code=403, detail="Invalid WhatsApp webhook token") + + +@router.post("") +async def whatsapp_webhook( + request: Request, + session: AsyncSession = Depends(get_async_session), +) -> Response: + _ensure_whatsapp_enabled() + raw_body = await request.body() + _verify_signature(raw_body, request.headers.get("X-Hub-Signature-256")) + try: + payload = json.loads(raw_body) + except ValueError: + record_gateway_webhook_parse_error() + return Response(status_code=200) + + try: + await _process_payload(session, payload) + await session.commit() + except Exception: + await session.rollback() + logger.exception("WhatsApp webhook processing failed") + return Response(status_code=200) + return Response(status_code=200) + + +def _verify_signature(raw_body: bytes, header_signature: str | None) -> None: + if not config.WHATSAPP_WEBHOOK_APP_SECRET: + raise HTTPException(status_code=500, detail="WhatsApp app secret is not configured") + received = (header_signature or "").removeprefix("sha256=") + expected = hmac.new( + config.WHATSAPP_WEBHOOK_APP_SECRET.encode(), + raw_body, + hashlib.sha256, + ).hexdigest() + if not received or not hmac.compare_digest(received, expected): + raise HTTPException(status_code=403, detail="Invalid WhatsApp webhook signature") + + +async def _process_payload(session: AsyncSession, payload: dict[str, Any]) -> None: + for entry in payload.get("entry") or []: + if not isinstance(entry, dict): + continue + for change in entry.get("changes") or []: + if not isinstance(change, dict): + continue + field = change.get("field") + value = change.get("value") or {} + if field == "messages": + await _process_messages_change(session, payload, entry, change, value) + elif field == "account_update": + await _handle_account_update(session, entry, value) + elif field == "phone_number_quality_update": + await _handle_phone_number_quality_update(session, entry, value) + + +async def _process_messages_change( + session: AsyncSession, + payload: dict[str, Any], + entry: dict[str, Any], + change: dict[str, Any], + value: dict[str, Any], +) -> None: + statuses = [status for status in value.get("statuses") or [] if isinstance(status, dict)] + for status in statuses: + record_gateway_outbound( + platform="whatsapp", + kind="status", + status=str(status.get("status") or "unknown"), + ) + + messages = [msg for msg in value.get("messages") or [] if isinstance(msg, dict)] + if not messages: + return + + account = await get_or_create_system_whatsapp_account(session) + metadata = value.get("metadata") or {} + if isinstance(metadata, dict): + cursor_state = dict(account.cursor_state or {}) + for key in ("phone_number_id", "display_phone_number"): + if metadata.get(key): + cursor_state[key] = metadata[key] + account.cursor_state = cursor_state + + for msg in messages: + message_id = str(msg.get("id") or "") + if not message_id: + continue + request_id = f"gateway_{uuid.uuid4().hex[:16]}" + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=ExternalChatPlatform.WHATSAPP, + event_dedupe_key=f"wamid:{message_id}", + external_event_id=message_id, + external_message_id=message_id, + event_kind="message", + raw_payload=_single_message_payload(payload, entry, change, msg), + request_id=request_id, + ) + record_gateway_inbox_write(platform="whatsapp", dedup_skipped=inbox_id is None) + + +async def _handle_account_update( + session: AsyncSession, + entry: dict[str, Any], + value: dict[str, Any], +) -> None: + account = await get_or_create_system_whatsapp_account(session) + cursor_state = dict(account.cursor_state or {}) + if entry.get("id"): + cursor_state["waba_id"] = str(entry.get("id")) + cursor_state["account_update"] = value + account.cursor_state = cursor_state + event = str(value.get("event") or value.get("type") or "").upper() + if event in {"DISABLED_UPDATE", "ACCOUNT_RESTRICTION", "PARTNER_REMOVED"}: + account.health_status = ExternalChatHealthStatus.FAILING + account.suspended_at = datetime.now(UTC) + account.suspended_reason = event.lower() + elif event in {"VERIFIED_ACCOUNT", "ACCOUNT_ENABLED", "REINSTATED"}: + account.health_status = ExternalChatHealthStatus.OK + account.suspended_at = None + account.suspended_reason = None + account.last_health_check_at = datetime.now(UTC) + + +async def _handle_phone_number_quality_update( + session: AsyncSession, + entry: dict[str, Any], + value: dict[str, Any], +) -> None: + account = await get_or_create_system_whatsapp_account(session) + cursor_state = dict(account.cursor_state or {}) + if entry.get("id"): + cursor_state["waba_id"] = str(entry.get("id")) + cursor_state["quality_update"] = value + account.cursor_state = cursor_state + account.last_health_check_at = datetime.now(UTC) + + +def _single_message_payload( + payload: dict[str, Any], + entry: dict[str, Any], + change: dict[str, Any], + message: dict[str, Any], +) -> dict[str, Any]: + value = dict(change.get("value") or {}) + value["messages"] = [message] + value.pop("statuses", None) + single_change = {**change, "value": value} + single_entry = {**entry, "changes": [single_change]} + return {"object": payload.get("object"), "entry": [single_entry]} From 3b529a3ab27a8ea1ad6556c90bb2556a6f7a79bc Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:19:13 +0530 Subject: [PATCH 29/63] feat(gateway): add Baileys WhatsApp adapter --- .../app/gateway/whatsapp/adapter_baileys.py | 118 +++++++++++++++++ .../gateway/whatsapp/translator_baileys.py | 123 ++++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100644 surfsense_backend/app/gateway/whatsapp/adapter_baileys.py create mode 100644 surfsense_backend/app/gateway/whatsapp/translator_baileys.py diff --git a/surfsense_backend/app/gateway/whatsapp/adapter_baileys.py b/surfsense_backend/app/gateway/whatsapp/adapter_baileys.py new file mode 100644 index 000000000..99489e27b --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/adapter_baileys.py @@ -0,0 +1,118 @@ +"""Baileys bridge platform adapter.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +import httpx + +from app.config import config +from app.gateway.base.adapter import ( + BasePlatformAdapter, + ParsedInboundEvent, + PlatformSendResult, +) + + +class WhatsAppBaileysAdapter(BasePlatformAdapter): + platform = "whatsapp" + + def __init__(self, bridge_url: str | None = None) -> None: + self.bridge_url = (bridge_url or config.WHATSAPP_BRIDGE_URL).rstrip("/") + + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + chat_id = str(raw_payload.get("chatId") or "") + sender_id = str(raw_payload.get("senderId") or chat_id) + message_id = str(raw_payload.get("messageId") or "") + body = raw_payload.get("body") + is_group = bool(raw_payload.get("isGroup")) + return ParsedInboundEvent( + platform=self.platform, + event_kind="message", + external_peer_id=chat_id or None, + external_peer_kind="group" if is_group else "direct", + external_message_id=message_id or None, + external_user_id=sender_id or None, + text=str(body) if body is not None else None, + raw_payload=raw_payload, + display_name=str(raw_payload.get("chatName") or sender_id or chat_id) or None, + username=None, + metadata={ + "sender_id": sender_id, + "from_me": bool(raw_payload.get("fromMe")), + "timestamp": raw_payload.get("timestamp"), + }, + ) + + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + payload: dict[str, Any] = {"chatId": external_peer_id, "message": text} + if reply_to_message_id: + payload["replyTo"] = reply_to_message_id + data = await self._post("/send", payload) + return PlatformSendResult( + external_message_id=str(data.get("messageId") or ""), + raw_response=data, + ) + + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + data = await self._post( + "/edit", + { + "chatId": external_peer_id, + "messageId": external_message_id, + "message": text, + }, + ) + return PlatformSendResult( + external_message_id=str(data.get("messageId") or external_message_id), + raw_response=data, + ) + + async def send_typing_indicator(self, *, external_peer_id: str) -> None: + await self._post("/typing", {"chatId": external_peer_id}, expect_json=False) + + async def validate_credentials(self) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=10) as client: + response = await client.get(f"{self.bridge_url}/health") + response.raise_for_status() + return response.json() + + async def fetch_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]: + async with httpx.AsyncClient(timeout=35) as client: + response = await client.get(f"{self.bridge_url}/messages") + response.raise_for_status() + for message in response.json(): + if isinstance(message, dict): + yield message + + async def request_pairing_code(self, *, phone_number: str) -> dict[str, Any]: + return await self._post("/pair", {"phoneNumber": phone_number}) + + async def _post( + self, + path: str, + payload: dict[str, Any], + *, + expect_json: bool = True, + ) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=30) as client: + response = await client.post(f"{self.bridge_url}{path}", json=payload) + response.raise_for_status() + if not expect_json or response.status_code == 204: + return {} + return response.json() diff --git a/surfsense_backend/app/gateway/whatsapp/translator_baileys.py b/surfsense_backend/app/gateway/whatsapp/translator_baileys.py new file mode 100644 index 000000000..8a4c8acfa --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/translator_baileys.py @@ -0,0 +1,123 @@ +"""Translate agent stream events into Baileys bridge messages.""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator + +from app.gateway.base.adapter import BasePlatformAdapter, PlatformSendResult +from app.gateway.base.formatting import split_text_message +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.whatsapp.adapter_baileys import WhatsAppBaileysAdapter +from app.observability.metrics import ( + record_gateway_hitl_aborted, + record_gateway_outbound, +) + +logger = logging.getLogger(__name__) + +HITL_UNSUPPORTED_MESSAGE = ( + "This action requires approval and is not yet supported from WhatsApp. " + "Try again with a different request." +) + + +class WhatsAppBaileysStreamTranslator(BaseStreamTranslator): + def __init__( + self, + *, + adapter: BasePlatformAdapter, + external_peer_id: str, + debounce_seconds: float = 1.5, + ) -> None: + self.adapter = adapter + self.external_peer_id = external_peer_id + self.debounce_seconds = debounce_seconds + self._buffer = "" + self._last_flush_at = 0.0 + self._external_message_ids: list[str] = [] + + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + await self._send_typing_indicator() + async for event in events: + if event.type in {"text-delta", "text_delta", "text"}: + self._buffer += str(event.data.get("text") or event.data.get("delta") or "") + await self._maybe_flush() + elif event.type in {"data-interrupt-request", "interrupt"}: + await self._handle_hitl_interrupt() + return + elif event.type in {"finish", "done"}: + break + + await self._flush(final=True) + + async def _maybe_flush(self) -> None: + now = asyncio.get_running_loop().time() + if now - self._last_flush_at < self.debounce_seconds: + return + await self._flush(final=False) + self._last_flush_at = now + + async def _flush(self, *, final: bool) -> None: + if not self._buffer: + return + + chunks = split_text_message(self._buffer) + if len(chunks) > 1: + for chunk in chunks[:-1]: + result = await self._send_text(chunk) + self._external_message_ids.append(result.external_message_id) + self._buffer = chunks[-1] + + if self._external_message_ids: + await self._edit_text(self._external_message_ids[-1], self._buffer) + else: + result = await self._send_text(self._buffer) + self._external_message_ids.append(result.external_message_id) + + if final: + logger.debug( + "WhatsApp Baileys finalized external_ids=%s", + self._external_message_ids, + ) + + async def _send_typing_indicator(self) -> None: + if not isinstance(self.adapter, WhatsAppBaileysAdapter): + return + try: + await self.adapter.send_typing_indicator(external_peer_id=self.external_peer_id) + record_gateway_outbound(platform="whatsapp", kind="typing", status="sent") + except Exception: + logger.debug("WhatsApp Baileys typing indicator failed", exc_info=True) + + async def _send_text(self, text: str) -> PlatformSendResult: + try: + result = await self.adapter.send_message( + external_peer_id=self.external_peer_id, + text=text, + ) + except Exception: + record_gateway_outbound(platform="whatsapp", kind="send", status="failed") + raise + record_gateway_outbound(platform="whatsapp", kind="send", status="sent") + return result + + async def _edit_text(self, message_id: str, text: str) -> PlatformSendResult: + try: + result = await self.adapter.edit_message( + external_peer_id=self.external_peer_id, + external_message_id=message_id, + text=text, + ) + except Exception: + record_gateway_outbound(platform="whatsapp", kind="edit", status="failed") + raise + record_gateway_outbound(platform="whatsapp", kind="edit", status="edited") + return result + + async def _handle_hitl_interrupt(self) -> None: + if self._buffer: + await self._flush(final=False) + await self._send_text(HITL_UNSUPPORTED_MESSAGE) + record_gateway_hitl_aborted(platform="whatsapp") From 63f9fe61b5ff25713f0ae7a64fe62e07f4acfdf2 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:19:29 +0530 Subject: [PATCH 30/63] feat(gateway): expose Baileys pairing endpoints --- .../routes/gateway_whatsapp_baileys_routes.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py diff --git a/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py new file mode 100644 index 000000000..6a7680c0d --- /dev/null +++ b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py @@ -0,0 +1,99 @@ +"""Routes for the self-hosted WhatsApp Baileys bridge.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + ExternalChatAccount, + ExternalChatAccountMode, + ExternalChatHealthStatus, + ExternalChatPlatform, + User, + get_async_session, +) +from app.gateway.whatsapp.adapter_baileys import WhatsAppBaileysAdapter +from app.users import current_active_user + +router = APIRouter(prefix="/gateway/whatsapp/baileys", tags=["gateway"]) + + +class BaileysPairRequest(BaseModel): + search_space_id: int + phone_number: str + + +def _ensure_baileys_enabled() -> None: + if config.GATEWAY_WHATSAPP_INTAKE_MODE != "baileys": + raise HTTPException(status_code=404, detail="WhatsApp Baileys gateway is disabled") + if config.is_cloud(): + raise HTTPException( + status_code=403, + detail="Baileys is only available for self-hosted SurfSense installs", + ) + + +async def _get_user_whatsapp_account( + session: AsyncSession, + user: User, +) -> ExternalChatAccount | None: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.owner_user_id == user.id, + ExternalChatAccount.platform == ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.is_system_account.is_(False), + ) + ) + return result.scalars().first() + + +@router.post("/pair") +async def request_pairing_code( + body: BaileysPairRequest, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, Any]: + _ensure_baileys_enabled() + adapter = WhatsAppBaileysAdapter() + try: + pairing = await adapter.request_pairing_code(phone_number=body.phone_number) + except Exception as exc: + raise HTTPException(status_code=502, detail=str(exc)) from exc + + account = await _get_user_whatsapp_account(session, user) + if account is None: + account = ExternalChatAccount( + platform=ExternalChatPlatform.WHATSAPP, + mode=ExternalChatAccountMode.SELF_HOST_BYO, + owner_user_id=user.id, + owner_search_space_id=body.search_space_id, + is_system_account=False, + cursor_state={}, + health_status=ExternalChatHealthStatus.UNKNOWN, + ) + session.add(account) + else: + account.mode = ExternalChatAccountMode.SELF_HOST_BYO + account.owner_search_space_id = body.search_space_id + account.health_status = ExternalChatHealthStatus.UNKNOWN + account.last_health_check_at = datetime.now(UTC) + await session.commit() + await session.refresh(account) + return {"account_id": account.id, **pairing} + + +@router.get("/health") +async def bridge_health() -> dict[str, Any]: + _ensure_baileys_enabled() + adapter = WhatsAppBaileysAdapter() + try: + return await adapter.validate_credentials() + except Exception as exc: + raise HTTPException(status_code=502, detail=str(exc)) from exc From 76a594ac606903496d955d9a4593a6084ecd9193 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:20:07 +0530 Subject: [PATCH 31/63] feat(gateway): add self-hosted WhatsApp bridge service --- .../scripts/whatsapp-bridge/Dockerfile | 15 + .../scripts/whatsapp-bridge/bridge.js | 262 ++ .../scripts/whatsapp-bridge/package-lock.json | 2150 +++++++++++++++++ .../scripts/whatsapp-bridge/package.json | 16 + 4 files changed, 2443 insertions(+) create mode 100644 surfsense_backend/scripts/whatsapp-bridge/Dockerfile create mode 100644 surfsense_backend/scripts/whatsapp-bridge/bridge.js create mode 100644 surfsense_backend/scripts/whatsapp-bridge/package-lock.json create mode 100644 surfsense_backend/scripts/whatsapp-bridge/package.json diff --git a/surfsense_backend/scripts/whatsapp-bridge/Dockerfile b/surfsense_backend/scripts/whatsapp-bridge/Dockerfile new file mode 100644 index 000000000..4a7e3f3fd --- /dev/null +++ b/surfsense_backend/scripts/whatsapp-bridge/Dockerfile @@ -0,0 +1,15 @@ +FROM node:20-alpine + +WORKDIR /app + +COPY package*.json ./ +RUN npm ci --silent + +COPY . . + +ENV WHATSAPP_SESSION_DIR=/data/sessions +EXPOSE 3000 + +HEALTHCHECK --interval=30s --timeout=5s CMD wget -qO- http://127.0.0.1:3000/health || exit 1 + +CMD ["node", "bridge.js"] diff --git a/surfsense_backend/scripts/whatsapp-bridge/bridge.js b/surfsense_backend/scripts/whatsapp-bridge/bridge.js new file mode 100644 index 000000000..5cef6b980 --- /dev/null +++ b/surfsense_backend/scripts/whatsapp-bridge/bridge.js @@ -0,0 +1,262 @@ +#!/usr/bin/env node + +import { + DisconnectReason, + fetchLatestBaileysVersion, + makeWASocket, + useMultiFileAuthState, +} from "@whiskeysockets/baileys"; +import { Boom } from "@hapi/boom"; +import express from "express"; +import { mkdirSync } from "node:fs"; +import path from "node:path"; +import pino from "pino"; +import qrcode from "qrcode-terminal"; + +const PORT = Number(process.env.PORT || "3000"); +const SESSION_DIR = process.env.WHATSAPP_SESSION_DIR || "/data/sessions"; +const SEND_TIMEOUT_MS = Number(process.env.WHATSAPP_SEND_TIMEOUT_MS || "60000"); +const MAX_QUEUE_SIZE = Number(process.env.WHATSAPP_MAX_QUEUE_SIZE || "100"); +const WHATSAPP_MODE = process.env.WHATSAPP_MODE || "self-chat"; +const SENT_ECHO_TTL_MS = 60_000; + +mkdirSync(SESSION_DIR, { recursive: true }); + +const app = express(); +app.use(express.json({ limit: "2mb" })); + +const logger = pino({ level: process.env.WHATSAPP_DEBUG ? "debug" : "warn" }); +const messageQueue = []; +const sentKeys = new Map(); +const recentlySentIds = new Set(); + +let sock = null; +let connectionState = "disconnected"; +let latestQr = null; +let starting = null; + +function normalizeText(message) { + const content = message?.message || {}; + return ( + content.conversation || + content.extendedTextMessage?.text || + content.imageMessage?.caption || + content.videoMessage?.caption || + content.documentMessage?.caption || + "" + ); +} + +function enqueueMessage(message) { + const remoteJid = message?.key?.remoteJid; + const id = message?.key?.id; + if (!remoteJid || !id || !message?.message) return; + if (messageQueue.length >= MAX_QUEUE_SIZE) messageQueue.shift(); + messageQueue.push({ + event: "messages.upsert", + key: message.key, + chatId: remoteJid, + senderId: message.key.participant || remoteJid, + messageId: id, + fromMe: Boolean(message.key.fromMe), + isGroup: remoteJid.endsWith("@g.us"), + body: normalizeText(message), + timestamp: Number(message.messageTimestamp || Date.now() / 1000), + raw: message, + }); +} + +function rememberSentMessage(sent) { + const sentId = sent?.key?.id; + if (!sentId) return; + sentKeys.set(sentId, sent.key); + recentlySentIds.add(sentId); + setTimeout(() => { + recentlySentIds.delete(sentId); + }, SENT_ECHO_TTL_MS).unref?.(); +} + +function withTimeout(promise, timeoutMs) { + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`sendMessage timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + +async function startSocket() { + if (starting) return starting; + starting = (async () => { + connectionState = "connecting"; + const { state, saveCreds } = await useMultiFileAuthState(SESSION_DIR); + const { version } = await fetchLatestBaileysVersion(); + sock = makeWASocket({ + version, + auth: state, + logger, + printQRInTerminal: false, + browser: ["SurfSense", "Chrome", "120.0"], + syncFullHistory: false, + markOnlineOnConnect: false, + getMessage: async () => ({ conversation: "" }), + }); + + sock.ev.on("creds.update", saveCreds); + sock.ev.on("connection.update", (update) => { + const { connection, lastDisconnect, qr } = update; + if (qr) { + latestQr = qr; + connectionState = "qr"; + qrcode.generate(qr, { small: true }); + } + if (connection === "open") { + latestQr = null; + connectionState = "connected"; + console.log("WhatsApp connected"); + } + if (connection === "close") { + const reason = new Boom(lastDisconnect?.error)?.output?.statusCode; + connectionState = "disconnected"; + if (reason === DisconnectReason.loggedOut) { + console.error("WhatsApp logged out; clear the session volume and pair again."); + process.exit(1); + } + setTimeout(() => { + starting = null; + void startSocket(); + }, reason === 515 ? 1000 : 3000); + } + }); + + sock.ev.on("messages.upsert", ({ messages, type }) => { + if (type !== "notify" && type !== "append") return; + for (const message of messages || []) { + const chatId = message?.key?.remoteJid; + if (!chatId) continue; + if (chatId.endsWith("@g.us") || chatId.includes("status@broadcast")) continue; + + if (message?.key?.fromMe) { + if (WHATSAPP_MODE !== "self-chat") continue; + if (recentlySentIds.has(message.key.id)) continue; + + const myNumber = (sock.user?.id || "").replace(/:.*@/, "@").replace(/@.*/, ""); + const myLid = (sock.user?.lid || "").replace(/:.*@/, "@").replace(/@.*/, ""); + const chatNumber = chatId.replace(/@.*/, ""); + const isSelfChat = + (myNumber && chatNumber === myNumber) || (myLid && chatNumber === myLid); + if (!isSelfChat) continue; + } else if (WHATSAPP_MODE === "self-chat") { + continue; + } + + enqueueMessage(message); + } + }); + })(); + try { + await starting; + } finally { + starting = null; + } +} + +app.get("/health", (_req, res) => { + res.json({ + status: connectionState, + hasQr: Boolean(latestQr), + queueDepth: messageQueue.length, + user: sock?.user || null, + }); +}); + +app.get("/messages", (_req, res) => { + const messages = messageQueue.splice(0, messageQueue.length); + res.json(messages); +}); + +app.post("/send", async (req, res) => { + try { + if (!sock || connectionState !== "connected") { + return res.status(503).json({ error: "WhatsApp is not connected" }); + } + const { chatId, message, replyTo } = req.body || {}; + if (!chatId || !message) { + return res.status(400).json({ error: "chatId and message are required" }); + } + const payload = { text: String(message) }; + if (replyTo) { + payload.contextInfo = { stanzaId: String(replyTo) }; + } + const sent = await withTimeout(sock.sendMessage(chatId, payload), SEND_TIMEOUT_MS); + rememberSentMessage(sent); + res.json({ messageId: sent?.key?.id || null, raw: sent }); + } catch (error) { + res.status(500).json({ error: error?.message || "send failed" }); + } +}); + +app.post("/edit", async (req, res) => { + try { + if (!sock || connectionState !== "connected") { + return res.status(503).json({ error: "WhatsApp is not connected" }); + } + const { chatId, messageId, message } = req.body || {}; + if (!chatId || !messageId || !message) { + return res.status(400).json({ error: "chatId, messageId and message are required" }); + } + const key = sentKeys.get(String(messageId)) || { + remoteJid: chatId, + id: String(messageId), + fromMe: true, + }; + const sent = await withTimeout( + sock.sendMessage(chatId, { text: String(message), edit: key }), + SEND_TIMEOUT_MS, + ); + rememberSentMessage(sent); + res.json({ messageId: sent?.key?.id || messageId, raw: sent }); + } catch (error) { + res.status(500).json({ error: error?.message || "edit failed" }); + } +}); + +app.post("/typing", async (req, res) => { + try { + if (!sock || connectionState !== "connected") return res.status(204).end(); + const { chatId } = req.body || {}; + if (chatId) { + await sock.sendPresenceUpdate("composing", chatId); + } + res.status(204).end(); + } catch { + res.status(204).end(); + } +}); + +app.post("/pair", async (req, res) => { + try { + await startSocket(); + const phoneNumber = String(req.body?.phoneNumber || req.body?.phone_number || "").replace(/\D/g, ""); + if (connectionState === "connected") { + return res.json({ status: "connected", pairing_code: null, expires_in: 0 }); + } + if (!phoneNumber) { + return res.status(400).json({ error: "phoneNumber is required for pairing code" }); + } + connectionState = "pairing"; + const code = await sock.requestPairingCode(phoneNumber); + res.json({ status: "pairing", pairing_code: code, expires_in: 60 }); + } catch (error) { + res.status(500).json({ error: error?.message || "pairing failed" }); + } +}); + +app.listen(PORT, "0.0.0.0", () => { + console.log( + `SurfSense WhatsApp bridge listening on ${PORT}; session=${path.resolve(SESSION_DIR)}; mode=${WHATSAPP_MODE}`, + ); + void startSocket(); +}); diff --git a/surfsense_backend/scripts/whatsapp-bridge/package-lock.json b/surfsense_backend/scripts/whatsapp-bridge/package-lock.json new file mode 100644 index 000000000..52c6900d7 --- /dev/null +++ b/surfsense_backend/scripts/whatsapp-bridge/package-lock.json @@ -0,0 +1,2150 @@ +{ + "name": "surfsense-whatsapp-bridge", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "surfsense-whatsapp-bridge", + "version": "1.0.0", + "dependencies": { + "@hapi/boom": "latest", + "@whiskeysockets/baileys": "latest", + "express": "latest", + "pino": "latest", + "qrcode-terminal": "latest" + } + }, + "node_modules/@borewit/text-codec": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@borewit/text-codec/-/text-codec-0.2.2.tgz", + "integrity": "sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@cacheable/memory": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.0.9.tgz", + "integrity": "sha512-HdMx6DoGywB30vacDbBsITbIX4pgFqj1zsrV58jZBUw3klzkNoXhj7qOqAgledhxG7YZI5rBSJg7Zp8/VG0DuA==", + "license": "MIT", + "dependencies": { + "@cacheable/utils": "^2.4.1", + "@keyv/bigmap": "^1.3.1", + "hookified": "^1.15.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@cacheable/node-cache": { + "version": "1.7.6", + "resolved": "https://registry.npmjs.org/@cacheable/node-cache/-/node-cache-1.7.6.tgz", + "integrity": "sha512-6Omk2SgNnjtxB5f/E6bTIWIt5xhdpx39fGNRQgU9lojvRxU68v+qY+SXXLsp3ZGukqoPjsK21wZ6XABFr/Ge3A==", + "license": "MIT", + "dependencies": { + "cacheable": "^2.3.1", + "hookified": "^1.14.0", + "keyv": "^5.5.5" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@cacheable/utils": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/@cacheable/utils/-/utils-2.4.1.tgz", + "integrity": "sha512-eiFgzCbIneyMlLOmNG4g9xzF7Hv3Mga4LjxjcSC/ues6VYq2+gUbQI8JqNuw/ZM8tJIeIaBGpswAsqV2V7ApgA==", + "license": "MIT", + "dependencies": { + "hashery": "^1.5.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@hapi/boom": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@hapi/boom/-/boom-10.0.1.tgz", + "integrity": "sha512-ERcCZaEjdH3OgSJlyjVk8pHIFeus91CjKP3v+MpgBNp5IvGzP2l/bRiD78nqYcKPaZdbKkK5vDBVPd2ohHBlsA==", + "license": "BSD-3-Clause", + "dependencies": { + "@hapi/hoek": "^11.0.2" + } + }, + "node_modules/@hapi/hoek": { + "version": "11.0.7", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-11.0.7.tgz", + "integrity": "sha512-HV5undWkKzcB4RZUusqOpcgxOaq6VOAH7zhhIr2g3G8NF/MlFO75SjOr2NfuSx0Mh40+1FqCkagKLJRykUWoFQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@keyv/bigmap": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@keyv/bigmap/-/bigmap-1.3.1.tgz", + "integrity": "sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ==", + "license": "MIT", + "dependencies": { + "hashery": "^1.4.0", + "hookified": "^1.15.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "keyv": "^5.6.0" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "license": "MIT" + }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.2.tgz", + "integrity": "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, + "node_modules/@tokenizer/inflate": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.4.1.tgz", + "integrity": "sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "token-types": "^6.1.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "25.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.1.tgz", + "integrity": "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==", + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@whiskeysockets/baileys": { + "version": "7.0.0-rc13", + "resolved": "https://registry.npmjs.org/@whiskeysockets/baileys/-/baileys-7.0.0-rc13.tgz", + "integrity": "sha512-8JPc8gaaCRykkjW2jxLGQ7/RZGrc7awO7WU+QJocf58eSUI9jAdcuYLynzhAbyU4UWvJJsHImZ+5E/JaZj5ypA==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "@cacheable/node-cache": "^1.4.0", + "@hapi/boom": "^9.1.3", + "async-mutex": "^0.5.0", + "libsignal": "^6.0.0", + "lru-cache": "^11.1.0", + "music-metadata": "^11.12.3", + "p-queue": "^9.0.0", + "pino": "^9.6", + "protobufjs": "^7.5.6", + "whatsapp-rust-bridge": "0.5.4", + "ws": "^8.13.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "audio-decode": "^2.1.3", + "jimp": "^1.6.1", + "link-preview-js": "^3.0.0", + "sharp": "*" + }, + "peerDependenciesMeta": { + "audio-decode": { + "optional": true + }, + "jimp": { + "optional": true + }, + "link-preview-js": { + "optional": true + } + } + }, + "node_modules/@whiskeysockets/baileys/node_modules/@hapi/boom": { + "version": "9.1.4", + "resolved": "https://registry.npmjs.org/@hapi/boom/-/boom-9.1.4.tgz", + "integrity": "sha512-Ls1oH8jaN1vNsqcaHVYJrKmgMcKsC1wcp8bujvXrHaAqD2iDYq3HoOwsxwo09Cuda5R5nC0o0IxlrlTuvPuzSw==", + "license": "BSD-3-Clause", + "dependencies": { + "@hapi/hoek": "9.x.x" + } + }, + "node_modules/@whiskeysockets/baileys/node_modules/@hapi/hoek": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", + "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@whiskeysockets/baileys/node_modules/pino": { + "version": "9.14.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.14.0.tgz", + "integrity": "sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^2.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^3.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/@whiskeysockets/baileys/node_modules/pino-abstract-transport": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-2.0.0.tgz", + "integrity": "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/@whiskeysockets/baileys/node_modules/thread-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.1.0.tgz", + "integrity": "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/async-mutex": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.5.0.tgz", + "integrity": "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/body-parser": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.1", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cacheable": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-2.3.5.tgz", + "integrity": "sha512-EQfaKe09tl615iNvq/TBRWTFf1AKJNXYQSsMx0Z3EI0nA+pVsVPS8wJhnRlkbdacKPh1d0qVIhwTc2zsQNFEEg==", + "license": "MIT", + "dependencies": { + "@cacheable/memory": "^2.0.8", + "@cacheable/utils": "^2.4.1", + "hookified": "^1.15.0", + "keyv": "^5.6.0", + "qified": "^0.10.1" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/curve25519-js": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/curve25519-js/-/curve25519-js-0.0.4.tgz", + "integrity": "sha512-axn2UMEnkhyDUPWOwVKBMVIzSQy2ejH2xRGy1wq81dqRwApXfIzfbE3hIX0ZRFBIihf/KDqK158DLwESu4AK1w==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/file-type": { + "version": "21.3.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-21.3.4.tgz", + "integrity": "sha512-Ievi/yy8DS3ygGvT47PjSfdFoX+2isQueoYP1cntFW1JLYAuS4GD7NUPGg4zv2iZfV52uDyk5w5Z0TdpRS6Q1g==", + "license": "MIT", + "dependencies": { + "@tokenizer/inflate": "^0.4.1", + "strtok3": "^10.3.4", + "token-types": "^6.1.1", + "uint8array-extras": "^1.4.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hashery": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/hashery/-/hashery-1.5.1.tgz", + "integrity": "sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ==", + "license": "MIT", + "dependencies": { + "hookified": "^1.15.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/hasown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hookified": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.15.1.tgz", + "integrity": "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==", + "license": "MIT" + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/keyv": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", + "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/libsignal": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/libsignal/-/libsignal-6.0.0.tgz", + "integrity": "sha512-d/5V3YFtDljbFMufz4ncyUYGYhJl+vzAe+c2EFFBQ6bz1h8Q3IOMEGXYMzlibU60I+e8GagMMpji18iez3P1hA==", + "license": "GPL-3.0", + "dependencies": { + "curve25519-js": "^0.0.4", + "protobufjs": "^7.5.5" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "11.5.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.1.tgz", + "integrity": "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/music-metadata": { + "version": "11.12.3", + "resolved": "https://registry.npmjs.org/music-metadata/-/music-metadata-11.12.3.tgz", + "integrity": "sha512-n6hSTZkuD59qWgHh6IP5dtDlDZQXoxk/bcA85Jywg8Z1iFrlNgl2+GTFgjZyn52W5UgQpV42V4XqrQZZAMbZTQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + }, + { + "type": "buymeacoffee", + "url": "https://buymeacoffee.com/borewit" + } + ], + "license": "MIT", + "dependencies": { + "@borewit/text-codec": "^0.2.2", + "@tokenizer/token": "^0.3.0", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "file-type": "^21.3.1", + "media-typer": "^1.1.0", + "strtok3": "^10.3.4", + "token-types": "^6.1.2", + "uint8array-extras": "^1.5.0", + "win-guid": "^0.2.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-queue": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.0.tgz", + "integrity": "sha512-7NED7xhQ74Ngp4JP/2e0VZHp7vSWfJfqeiR92jPgxsz6m0Se4P03YoTKa9dDXyZ3r6P616gUXttrB6nnHYKang==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.4", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", + "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pino": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", + "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", + "license": "MIT" + }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/protobufjs": { + "version": "7.6.1", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.1.tgz", + "integrity": "sha512-4K0myLaWL5EteuSAro91EGFgcfVgxb64Jx+7oDAY6GOkXD4M69yuSEljNcInGVCA5sOPxmZ/EqDLj2x0Q0+Ygg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qified": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/qified/-/qified-0.10.1.tgz", + "integrity": "sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==", + "license": "MIT", + "dependencies": { + "hookified": "^2.1.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/qified/node_modules/hookified": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-2.2.0.tgz", + "integrity": "sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA==", + "license": "MIT" + }, + "node_modules/qrcode-terminal": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/qrcode-terminal/-/qrcode-terminal-0.12.0.tgz", + "integrity": "sha512-EXtzRZmC+YGmGlDFbXKxQiMZNwCLEO6BANKXG4iCtSIM0yqc/pappSx3RIKr4r0uh5JsBckOXeKrB3Iz7mdQpQ==", + "bin": { + "qrcode-terminal": "bin/qrcode-terminal.js" + } + }, + "node_modules/qs": { + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.1.tgz", + "integrity": "sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==", + "license": "ISC", + "peer": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/strtok3": { + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.5.tgz", + "integrity": "sha512-ki4hZQfh5rX0QDLLkOCj+h+CVNkqmp/CMf8v8kZpkNVK6jGQooMytqzLZYUVYIZcFZ6yDB70EfD8POcFXiF5oA==", + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/thread-stream": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", + "integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==", + "license": "MIT", + "dependencies": { + "real-require": "^1.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/thread-stream/node_modules/real-require": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz", + "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", + "license": "MIT" + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/token-types": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.1.2.tgz", + "integrity": "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==", + "license": "MIT", + "dependencies": { + "@borewit/text-codec": "^0.2.1", + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/uint8array-extras": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.5.0.tgz", + "integrity": "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/whatsapp-rust-bridge": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/whatsapp-rust-bridge/-/whatsapp-rust-bridge-0.5.4.tgz", + "integrity": "sha512-yYO1qSs0Fe7tGtnxOFHomocUD6IZtoAgmA4oDFyGIRZ67D3QZk3w7swA6XXFXNQngiyrg2k7tul6IrM3eUFh7A==", + "license": "MIT" + }, + "node_modules/win-guid": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/win-guid/-/win-guid-0.2.1.tgz", + "integrity": "sha512-gEIQU4mkgl2OPeoNrWflcJFJ3Ae2BPd4eCsHHA/XikslkIVms/nHhvnvzIZV7VLmBvtFlDOzLt9rrZT+n6D67A==", + "license": "MIT" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/surfsense_backend/scripts/whatsapp-bridge/package.json b/surfsense_backend/scripts/whatsapp-bridge/package.json new file mode 100644 index 000000000..214ebacc6 --- /dev/null +++ b/surfsense_backend/scripts/whatsapp-bridge/package.json @@ -0,0 +1,16 @@ +{ + "name": "surfsense-whatsapp-bridge", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "start": "node bridge.js" + }, + "dependencies": { + "@hapi/boom": "latest", + "@whiskeysockets/baileys": "latest", + "express": "latest", + "pino": "latest", + "qrcode-terminal": "latest" + } +} From 51bf2a8361b24e9a97ad77e788a282b324e82c79 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:20:25 +0530 Subject: [PATCH 32/63] feat(gateway): wire WhatsApp bridge runtime --- docker/docker-compose.yml | 22 +++++ .../app/gateway/byo_long_poll.py | 97 +++++++++++++++---- 2 files changed, 102 insertions(+), 17 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 06a3ac79a..12b7a6a33 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -118,6 +118,7 @@ services: UNSTRUCTURED_HAS_PATCHED_LOOP: "1" NEXT_FRONTEND_URL: ${NEXT_FRONTEND_URL:-http://localhost:${FRONTEND_PORT:-3929}} SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080} + WHATSAPP_BRIDGE_URL: ${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:3000} # Daytona Sandbox – uncomment and set credentials to enable cloud code execution # DAYTONA_SANDBOX_ENABLED: "TRUE" # DAYTONA_API_KEY: ${DAYTONA_API_KEY:-} @@ -143,6 +144,23 @@ services: retries: 30 start_period: 200s + whatsapp-bridge: + build: ../surfsense_backend/scripts/whatsapp-bridge + profiles: + - whatsapp + volumes: + - whatsapp_sessions:/data/sessions + environment: + WHATSAPP_MODE: ${WHATSAPP_MODE:-self-chat} + WHATSAPP_SESSION_DIR: /data/sessions + mem_limit: 512m + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/health"] + interval: 30s + timeout: 5s + retries: 5 + celery_worker: image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest} volumes: @@ -264,6 +282,8 @@ services: NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${AUTH_TYPE:-LOCAL} NEXT_PUBLIC_ETL_SERVICE: ${ETL_SERVICE:-DOCLING} NEXT_PUBLIC_DEPLOYMENT_MODE: ${DEPLOYMENT_MODE:-self-hosted} + NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE: ${GATEWAY_WHATSAPP_INTAKE_MODE:-disabled} + NEXT_PUBLIC_WHATSAPP_DISPLAY_PHONE_NUMBER: ${WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER:-} FASTAPI_BACKEND_INTERNAL_URL: ${FASTAPI_BACKEND_INTERNAL_URL:-http://backend:8000} labels: - "com.centurylinklabs.watchtower.enable=true" @@ -285,3 +305,5 @@ volumes: name: surfsense-zero-cache zero_init: name: surfsense-zero-init + whatsapp_sessions: + name: surfsense-whatsapp-sessions diff --git a/surfsense_backend/app/gateway/byo_long_poll.py b/surfsense_backend/app/gateway/byo_long_poll.py index 0be448ae3..bb7ba53ad 100644 --- a/surfsense_backend/app/gateway/byo_long_poll.py +++ b/surfsense_backend/app/gateway/byo_long_poll.py @@ -8,9 +8,17 @@ import logging from sqlalchemy import select from app.config import config -from app.db import ExternalChatPlatform, ExternalChatAccount, async_session_maker +from app.db import ( + ExternalChatAccount, + ExternalChatAccountMode, + ExternalChatPlatform, + async_session_maker, +) from app.gateway.accounts import account_token +from app.gateway.inbox import persist_inbound_event from app.gateway.runner import _run_telegram_account +from app.gateway.whatsapp.adapter_baileys import WhatsAppBaileysAdapter +from app.observability.metrics import record_gateway_inbox_write logger = logging.getLogger(__name__) @@ -42,37 +50,92 @@ async def _byo_account_supervisor(account_id: int, token: str) -> None: await _sleep_or_shutdown(30) +async def _whatsapp_baileys_supervisor() -> None: + adapter = WhatsAppBaileysAdapter() + while _shutdown_event is None or not _shutdown_event.is_set(): + try: + async for raw_event in adapter.fetch_updates(offset=None): + async with async_session_maker() as session: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.mode == ExternalChatAccountMode.SELF_HOST_BYO, + ExternalChatAccount.is_system_account.is_(False), + ExternalChatAccount.suspended_at.is_(None), + ) + ) + account = result.scalars().first() + if account is None: + continue + message_id = str(raw_event.get("messageId") or "") + if not message_id: + continue + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=ExternalChatPlatform.WHATSAPP, + event_dedupe_key=f"baileys:{message_id}", + external_event_id=message_id, + external_message_id=message_id, + event_kind="message", + raw_payload=raw_event, + ) + await session.commit() + record_gateway_inbox_write( + platform="whatsapp", + dedup_skipped=inbox_id is None, + ) + except asyncio.CancelledError: + raise + except Exception: + logger.exception("WhatsApp Baileys intake failed; retrying in 10s") + await _sleep_or_shutdown(10) + + async def start_byo_long_poll_supervisors() -> None: """Start one BYO long-poll supervisor per active non-system Telegram account.""" global _shutdown_event - if config.GATEWAY_TELEGRAM_INTAKE_MODE != "longpoll": + if ( + config.GATEWAY_TELEGRAM_INTAKE_MODE != "longpoll" + and config.GATEWAY_WHATSAPP_INTAKE_MODE != "baileys" + ): return if _tasks: return _shutdown_event = asyncio.Event() - async with async_session_maker() as session: - result = await session.execute( - select(ExternalChatAccount).where( - ExternalChatAccount.platform == ExternalChatPlatform.TELEGRAM, - ExternalChatAccount.is_system_account.is_(False), - ExternalChatAccount.suspended_at.is_(None), + if config.GATEWAY_TELEGRAM_INTAKE_MODE == "longpoll": + async with async_session_maker() as session: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.TELEGRAM, + ExternalChatAccount.is_system_account.is_(False), + ExternalChatAccount.suspended_at.is_(None), + ) ) - ) - accounts = list(result.scalars()) + accounts = list(result.scalars()) - for account in accounts: - token = account_token(account) - if not token: - continue + for account in accounts: + token = account_token(account) + if not token: + continue + task = asyncio.create_task( + _byo_account_supervisor(int(account.id), token), + name=f"gateway-byo-telegram-{account.id}", + ) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + logger.info("Started BYO Telegram long-poll supervisor account_id=%s", account.id) + + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "baileys": task = asyncio.create_task( - _byo_account_supervisor(int(account.id), token), - name=f"gateway-byo-telegram-{account.id}", + _whatsapp_baileys_supervisor(), + name="gateway-byo-whatsapp-baileys", ) _tasks.add(task) task.add_done_callback(_tasks.discard) - logger.info("Started BYO Telegram long-poll supervisor account_id=%s", account.id) + logger.info("Started WhatsApp Baileys bridge intake supervisor") async def stop_byo_long_poll_supervisors() -> None: From 185759de1fa66923fb024b1d6f3ebda9a09eca04 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:20:43 +0530 Subject: [PATCH 33/63] feat(gateway): register multi-platform gateway routes --- surfsense_backend/app/routes/__init__.py | 4 ++ .../app/routes/gateway_webhook_routes.py | 63 ++++++++++++++----- .../app/tasks/celery_tasks/gateway_tasks.py | 30 ++++++--- 3 files changed, 72 insertions(+), 25 deletions(-) diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index f46b6fc65..369c988c7 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -19,6 +19,8 @@ from .editor_routes import router as editor_router from .export_routes import router as export_router from .folders_routes import router as folders_router from .gateway_webhook_routes import router as gateway_router +from .gateway_whatsapp_baileys_routes import router as gateway_whatsapp_baileys_router +from .gateway_whatsapp_webhook_routes import router as gateway_whatsapp_webhook_router from .google_calendar_add_connector_route import ( router as google_calendar_add_connector_router, ) @@ -70,6 +72,8 @@ router.include_router(export_router) router.include_router(documents_router) router.include_router(folders_router) router.include_router(gateway_router) +router.include_router(gateway_whatsapp_webhook_router) +router.include_router(gateway_whatsapp_baileys_router) router.include_router(notes_router) router.include_router(new_chat_router) # Chat with assistant-ui persistence router.include_router(agent_revert_router) # POST /threads/{id}/revert/{action_id} diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py index f9b6acc93..5508e534c 100644 --- a/surfsense_backend/app/routes/gateway_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -7,6 +7,7 @@ import logging import uuid from datetime import UTC, datetime from typing import Any +from urllib.parse import quote from fastapi import APIRouter, Depends, HTTPException, Request from pydantic import BaseModel @@ -16,14 +17,17 @@ from starlette.responses import Response from app.config import config from app.db import ( - ExternalChatBindingState, - ExternalChatBinding, - ExternalChatPlatform, ExternalChatAccount, + ExternalChatBinding, + ExternalChatBindingState, + ExternalChatPlatform, User, get_async_session, ) -from app.gateway.accounts import get_or_create_system_telegram_account +from app.gateway.accounts import ( + get_or_create_system_telegram_account, + get_or_create_system_whatsapp_account, +) from app.gateway.bindings import resume_binding, revoke_binding from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key from app.gateway.pairing import generate_pairing_code, pairing_expires_at @@ -131,11 +135,39 @@ async def start_binding( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> StartBindingResponse: - if body.platform != ExternalChatPlatform.TELEGRAM: - raise HTTPException(status_code=400, detail="Only Telegram is supported in v1") - - account = await get_or_create_system_telegram_account(session) code = generate_pairing_code() + if body.platform == ExternalChatPlatform.TELEGRAM: + account = await get_or_create_system_telegram_account(session) + username = account.bot_username or config.TELEGRAM_SHARED_BOT_USERNAME + if not username: + raise HTTPException( + status_code=500, + detail="Telegram bot username is not configured", + ) + deep_link = f"https://t.me/{username}?start={code}" + elif body.platform == ExternalChatPlatform.WHATSAPP: + if config.GATEWAY_WHATSAPP_INTAKE_MODE != "cloud": + raise HTTPException( + status_code=400, + detail="WhatsApp /start pairing requires GATEWAY_WHATSAPP_INTAKE_MODE=cloud", + ) + account = await get_or_create_system_whatsapp_account(session) + phone = config.WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER + if not phone: + raise HTTPException( + status_code=500, + detail="WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER is not configured", + ) + normalized_phone = "".join(ch for ch in phone if ch.isdigit()) + if not normalized_phone: + raise HTTPException( + status_code=500, + detail="WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER must contain digits", + ) + deep_link = f"https://wa.me/{normalized_phone}?text={quote(f'/start {code}')}" + else: + raise HTTPException(status_code=400, detail="Unsupported platform") + expires_at = pairing_expires_at() binding = ExternalChatBinding( account_id=account.id, @@ -149,13 +181,10 @@ async def start_binding( await session.commit() await session.refresh(binding) - username = account.bot_username or config.TELEGRAM_SHARED_BOT_USERNAME - if not username: - raise HTTPException(status_code=500, detail="Telegram bot username is not configured") return StartBindingResponse( binding_id=binding.id, code=code, - deep_link=f"https://t.me/{username}?start={code}", + deep_link=deep_link, expires_at=expires_at, ) @@ -166,21 +195,21 @@ async def list_bindings( session: AsyncSession = Depends(get_async_session), ) -> list[dict[str, Any]]: result = await session.execute( - select(ExternalChatBinding).where( - ExternalChatBinding.user_id == user.id - ) + select(ExternalChatBinding, ExternalChatAccount) + .join(ExternalChatAccount, ExternalChatBinding.account_id == ExternalChatAccount.id) + .where(ExternalChatBinding.user_id == user.id) ) return [ { "id": binding.id, - "platform": "telegram", + "platform": account.platform.value, "state": binding.state.value, "search_space_id": binding.search_space_id, "external_display_name": binding.external_display_name, "external_username": binding.external_username, "suspended_reason": binding.suspended_reason, } - for binding in result.scalars() + for binding, account in result.all() ] diff --git a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py index aeb3d721e..1c2bb166f 100644 --- a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py @@ -9,14 +9,14 @@ from sqlalchemy import select, update from app.celery_app import celery_app from app.db import ( + ExternalChatAccount, ExternalChatEventStatus, ExternalChatHealthStatus, ExternalChatInboundEvent, ExternalChatPlatform, - ExternalChatAccount, ) -from app.gateway.accounts import account_token from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key +from app.gateway.registry import resolve_platform_bundle from app.gateway.telegram.adapter import TelegramAdapter from app.observability.metrics import ( record_gateway_health_check_failure, @@ -69,15 +69,29 @@ def gateway_health_check_task() -> None: result = await session.execute(select(ExternalChatAccount)) accounts = list(result.scalars()) for account in accounts: - token = account_token(account) - if not token or account.platform != ExternalChatPlatform.TELEGRAM: - continue try: - metadata = await TelegramAdapter(token).validate_credentials() + bundle = resolve_platform_bundle(account) + metadata = await bundle.adapter.validate_credentials() account.health_status = ExternalChatHealthStatus.OK - account.bot_username = metadata.get("username") + if account.platform == ExternalChatPlatform.TELEGRAM: + account.bot_username = metadata.get("username") + elif account.platform == ExternalChatPlatform.WHATSAPP: + cursor_state = dict(account.cursor_state or {}) + for key in ( + "quality_rating", + "account_review_status", + "status", + ): + if key in metadata: + cursor_state[key] = metadata[key] + account.cursor_state = cursor_state except Exception: - logger.warning("External chat Telegram health check failed", exc_info=True) + logger.warning( + "External chat health check failed platform=%s account_id=%s", + account.platform.value, + account.id, + exc_info=True, + ) account.health_status = ExternalChatHealthStatus.FAILING record_gateway_health_check_failure(platform=account.platform.value) account.last_health_check_at = datetime.now(UTC) From bba33b59471adc1ddd24a96a1b2247cdbc85674a Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 10:21:37 +0530 Subject: [PATCH 34/63] feat(web): add WhatsApp messaging channel controls --- surfsense_web/.env.example | 1 + .../components/MessagingChannelsContent.tsx | 128 +++++++++++++++--- 2 files changed, 111 insertions(+), 18 deletions(-) diff --git a/surfsense_web/.env.example b/surfsense_web/.env.example index 5fb9d07d1..12d81ee3f 100644 --- a/surfsense_web/.env.example +++ b/surfsense_web/.env.example @@ -6,6 +6,7 @@ FASTAPI_BACKEND_INTERNAL_URL=https://your-internal-backend.example.com NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL or GOOGLE NEXT_PUBLIC_ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848 +NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE=disabled # Contact Form Vars (optional) DATABASE_URL=postgresql://postgres:[YOUR-PASSWORD]@db.sdsf.supabase.co:5432/postgres diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx index 0c35533c6..248abb121 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx @@ -2,7 +2,7 @@ import { MessageCircle, RefreshCw, ShieldAlert } from "lucide-react"; import { useParams } from "next/navigation"; -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useState, useTransition } from "react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; @@ -11,6 +11,7 @@ import { BACKEND_URL } from "@/lib/env-config"; type Binding = { id: number; + platform?: string; state: string; search_space_id: number; external_display_name?: string | null; @@ -34,13 +35,20 @@ type Pairing = { expires_at: string; }; +type PairingPlatform = "telegram" | "whatsapp"; + export function MessagingChannelsContent() { const params = useParams<{ search_space_id: string }>(); const searchSpaceId = Number(params.search_space_id); const [bindings, setBindings] = useState([]); const [platforms, setPlatforms] = useState([]); const [pairing, setPairing] = useState(null); + const [pairingPlatform, setPairingPlatform] = useState(null); + const [whatsappStatus, setWhatsappStatus] = useState(null); + const [baileysPhone, setBaileysPhone] = useState(""); + const [baileysCode, setBaileysCode] = useState(null); const [loading, setLoading] = useState(true); + const [isPending, startTransition] = useTransition(); const refresh = useCallback(async () => { setLoading(true); @@ -57,16 +65,40 @@ export function MessagingChannelsContent() { void refresh(); }, [refresh]); - async function startPairing() { + async function startPairing(platform: PairingPlatform) { const res = await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings/start`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ platform: "telegram", search_space_id: searchSpaceId }), + body: JSON.stringify({ platform, search_space_id: searchSpaceId }), }); setPairing(await res.json()); + setPairingPlatform(platform); await refresh(); } + function pairBaileys() { + startTransition(async () => { + setWhatsappStatus("Requesting WhatsApp pairing code..."); + const res = await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/whatsapp/baileys/pair`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ search_space_id: searchSpaceId, phone_number: baileysPhone }), + }); + if (!res.ok) { + setWhatsappStatus("Unable to request pairing code. Check the whatsapp-bridge service."); + return; + } + const data = await res.json(); + setBaileysCode(data.pairing_code ?? null); + setWhatsappStatus( + data.status === "connected" + ? "WhatsApp bridge is connected." + : "Enter the pairing code in WhatsApp.", + ); + await refresh(); + }); + } + async function revoke(id: number) { await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings/${id}`, { method: "DELETE", @@ -82,7 +114,26 @@ export function MessagingChannelsContent() { } const telegram = platforms.find((p) => p.platform === "telegram"); + const whatsapp = platforms.find((p) => p.platform === "whatsapp"); + const whatsappMode = process.env.NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE ?? "disabled"; const activeBindings = bindings.filter((binding) => binding.search_space_id === searchSpaceId); + const renderPairingPanel = (platform: PairingPlatform) => { + if (!pairing || pairingPlatform !== platform) return null; + + return ( +
+

Pairing code

+

{pairing.code}

+ + Open {platform === "whatsapp" ? "WhatsApp" : "Telegram"} pairing link + +

+ Expires at {new Date(pairing.expires_at).toLocaleString()}. SurfSense stores this + channel's messages for agent memory and operational debugging. +

+
+ ); + }; return (
@@ -104,36 +155,77 @@ export function MessagingChannelsContent() {
- +
- {pairing ? ( -
-

Pairing code

-

{pairing.code}

- - Open Telegram pairing link - -

- Expires at {new Date(pairing.expires_at).toLocaleString()}. SurfSense stores this - channel's messages for agent memory and operational debugging. -

-
- ) : null} + {renderPairingPanel("telegram")} + {whatsappMode !== "disabled" ? ( + + +
+ + + WhatsApp + + + {whatsapp?.health_status ?? "not configured"} + +
+

+ Pair this search space with WhatsApp using the configured gateway mode. +

+ + + {whatsappMode === "cloud" ? ( +
+ + {renderPairingPanel("whatsapp")} +
+ ) : null} + {whatsappMode === "baileys" ? ( +
+

+ Self-hosted WhatsApp uses Message Yourself mode. After pairing, send messages in + your own WhatsApp chat with yourself; messages from other chats are ignored. +

+ setBaileysPhone(event.target.value)} + /> + + {baileysCode ? ( +
+

WhatsApp pairing code

+

{baileysCode}

+
+ ) : null} +
+ ) : null} + {whatsappStatus ? ( +

{whatsappStatus}

+ ) : null} + + + ) : null} + Active Chats {activeBindings.length === 0 ? ( -

No Telegram chats paired yet.

+

No external chats connected yet.

) : ( activeBindings.map((binding) => (
Date: Fri, 29 May 2026 11:39:11 +0530 Subject: [PATCH 35/63] feat(gateway): update WhatsApp bridge configuration and expose port 9929 --- docker/.env.example | 3 +++ docker/docker-compose.dev.yml | 22 +++++++++++++++++++ docker/docker-compose.yml | 7 ++++-- surfsense_backend/.env.example | 2 +- surfsense_backend/app/config/__init__.py | 2 +- .../scripts/whatsapp-bridge/Dockerfile | 4 ++-- .../scripts/whatsapp-bridge/bridge.js | 2 +- 7 files changed, 35 insertions(+), 7 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 748f03048..39fd8989b 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -55,6 +55,9 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # -- Redis exposed port (dev only; Redis is internal-only in prod) -- # REDIS_PORT=6379 +# -- WhatsApp bridge exposed port (dev/hybrid only; prod keeps it Docker-internal) -- +# WHATSAPP_BRIDGE_PORT=9929 + # -- Frontend Build Args -- # In dev, the frontend is built from source and these are passed as build args. # In prod, they are automatically derived from AUTH_TYPE, ETL_SERVICE, and the port settings above. diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 58cb7b42f..818611138 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -126,6 +126,7 @@ services: - AUTH_TYPE=${AUTH_TYPE:-LOCAL} - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000} - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080} + - WHATSAPP_BRIDGE_URL=${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:9929} # Daytona Sandbox – uncomment and set credentials to enable cloud code execution # - DAYTONA_SANDBOX_ENABLED=TRUE # - DAYTONA_API_KEY=${DAYTONA_API_KEY:-} @@ -148,6 +149,25 @@ services: retries: 30 start_period: 200s + whatsapp-bridge: + build: ../surfsense_backend/scripts/whatsapp-bridge + profiles: + - whatsapp + ports: + - "127.0.0.1:${WHATSAPP_BRIDGE_PORT:-9929}:9929" + volumes: + - whatsapp_sessions:/data/sessions + environment: + - PORT=9929 + - WHATSAPP_MODE=${WHATSAPP_MODE:-self-chat} + - WHATSAPP_SESSION_DIR=/data/sessions + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:9929/health"] + interval: 30s + timeout: 5s + retries: 5 + celery_worker: build: *backend-build volumes: @@ -282,3 +302,5 @@ volumes: name: surfsense-dev-zero-cache zero_init: name: surfsense-dev-zero-init + whatsapp_sessions: + name: surfsense-dev-whatsapp-sessions diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 12b7a6a33..66ad55b77 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -118,7 +118,7 @@ services: UNSTRUCTURED_HAS_PATCHED_LOOP: "1" NEXT_FRONTEND_URL: ${NEXT_FRONTEND_URL:-http://localhost:${FRONTEND_PORT:-3929}} SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080} - WHATSAPP_BRIDGE_URL: ${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:3000} + WHATSAPP_BRIDGE_URL: ${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:9929} # Daytona Sandbox – uncomment and set credentials to enable cloud code execution # DAYTONA_SANDBOX_ENABLED: "TRUE" # DAYTONA_API_KEY: ${DAYTONA_API_KEY:-} @@ -148,15 +148,18 @@ services: build: ../surfsense_backend/scripts/whatsapp-bridge profiles: - whatsapp + expose: + - "9929" volumes: - whatsapp_sessions:/data/sessions environment: + PORT: 9929 WHATSAPP_MODE: ${WHATSAPP_MODE:-self-chat} WHATSAPP_SESSION_DIR: /data/sessions mem_limit: 512m restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/health"] + test: ["CMD", "wget", "-qO-", "http://localhost:9929/health"] interval: 30s timeout: 5s retries: 5 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index bc96cc948..6ddf18ebb 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -34,7 +34,7 @@ WHATSAPP_SHARED_WABA_ID= WHATSAPP_GRAPH_API_VERSION=v25.0 WHATSAPP_WEBHOOK_VERIFY_TOKEN= WHATSAPP_WEBHOOK_APP_SECRET= -WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:3000 +WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:9929 # Platform Web Search (SearXNG) # Set this to enable built-in web search. Docker Compose sets it automatically. diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index afccb190b..c8ffa802e 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -562,7 +562,7 @@ class Config: WHATSAPP_GRAPH_API_VERSION = os.getenv("WHATSAPP_GRAPH_API_VERSION", "v25.0") WHATSAPP_WEBHOOK_VERIFY_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFY_TOKEN") WHATSAPP_WEBHOOK_APP_SECRET = os.getenv("WHATSAPP_WEBHOOK_APP_SECRET") - WHATSAPP_BRIDGE_URL = os.getenv("WHATSAPP_BRIDGE_URL", "http://whatsapp-bridge:3000") + WHATSAPP_BRIDGE_URL = os.getenv("WHATSAPP_BRIDGE_URL", "http://whatsapp-bridge:9929") GATEWAY_WHATSAPP_INTAKE_MODE = os.getenv( "GATEWAY_WHATSAPP_INTAKE_MODE", "disabled" ).lower() diff --git a/surfsense_backend/scripts/whatsapp-bridge/Dockerfile b/surfsense_backend/scripts/whatsapp-bridge/Dockerfile index 4a7e3f3fd..42bcd6b21 100644 --- a/surfsense_backend/scripts/whatsapp-bridge/Dockerfile +++ b/surfsense_backend/scripts/whatsapp-bridge/Dockerfile @@ -8,8 +8,8 @@ RUN npm ci --silent COPY . . ENV WHATSAPP_SESSION_DIR=/data/sessions -EXPOSE 3000 +EXPOSE 9929 -HEALTHCHECK --interval=30s --timeout=5s CMD wget -qO- http://127.0.0.1:3000/health || exit 1 +HEALTHCHECK --interval=30s --timeout=5s CMD wget -qO- http://127.0.0.1:9929/health || exit 1 CMD ["node", "bridge.js"] diff --git a/surfsense_backend/scripts/whatsapp-bridge/bridge.js b/surfsense_backend/scripts/whatsapp-bridge/bridge.js index 5cef6b980..84b28030b 100644 --- a/surfsense_backend/scripts/whatsapp-bridge/bridge.js +++ b/surfsense_backend/scripts/whatsapp-bridge/bridge.js @@ -13,7 +13,7 @@ import path from "node:path"; import pino from "pino"; import qrcode from "qrcode-terminal"; -const PORT = Number(process.env.PORT || "3000"); +const PORT = Number(process.env.PORT || "9929"); const SESSION_DIR = process.env.WHATSAPP_SESSION_DIR || "/data/sessions"; const SEND_TIMEOUT_MS = Number(process.env.WHATSAPP_SEND_TIMEOUT_MS || "60000"); const MAX_QUEUE_SIZE = Number(process.env.WHATSAPP_MAX_QUEUE_SIZE || "100"); From 389a51d4942fd7bbb6c740e678d38634e0ea3afc Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 29 May 2026 13:37:45 +0530 Subject: [PATCH 36/63] feat(gateway): enhance WhatsApp bridge with pairing timeout and health check integration --- .../routes/gateway_whatsapp_baileys_routes.py | 4 +- .../scripts/whatsapp-bridge/bridge.js | 101 ++++++++++++++++-- .../components/MessagingChannelsContent.tsx | 80 +++++++------- surfsense_web/package.json | 1 + surfsense_web/pnpm-lock.yaml | 88 +++++++++++++++ 5 files changed, 227 insertions(+), 47 deletions(-) diff --git a/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py index 6a7680c0d..24209f86f 100644 --- a/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py +++ b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py @@ -90,7 +90,9 @@ async def request_pairing_code( @router.get("/health") -async def bridge_health() -> dict[str, Any]: +async def bridge_health( + user: User = Depends(current_active_user), +) -> dict[str, Any]: _ensure_baileys_enabled() adapter = WhatsAppBaileysAdapter() try: diff --git a/surfsense_backend/scripts/whatsapp-bridge/bridge.js b/surfsense_backend/scripts/whatsapp-bridge/bridge.js index 84b28030b..017456654 100644 --- a/surfsense_backend/scripts/whatsapp-bridge/bridge.js +++ b/surfsense_backend/scripts/whatsapp-bridge/bridge.js @@ -8,7 +8,7 @@ import { } from "@whiskeysockets/baileys"; import { Boom } from "@hapi/boom"; import express from "express"; -import { mkdirSync } from "node:fs"; +import { mkdirSync, readdirSync, rmSync } from "node:fs"; import path from "node:path"; import pino from "pino"; import qrcode from "qrcode-terminal"; @@ -16,6 +16,7 @@ import qrcode from "qrcode-terminal"; const PORT = Number(process.env.PORT || "9929"); const SESSION_DIR = process.env.WHATSAPP_SESSION_DIR || "/data/sessions"; const SEND_TIMEOUT_MS = Number(process.env.WHATSAPP_SEND_TIMEOUT_MS || "60000"); +const PAIRING_TIMEOUT_MS = Number(process.env.WHATSAPP_PAIRING_TIMEOUT_MS || "30000"); const MAX_QUEUE_SIZE = Number(process.env.WHATSAPP_MAX_QUEUE_SIZE || "100"); const WHATSAPP_MODE = process.env.WHATSAPP_MODE || "self-chat"; const SENT_ECHO_TTL_MS = 60_000; @@ -34,6 +35,82 @@ let sock = null; let connectionState = "disconnected"; let latestQr = null; let starting = null; +let pendingPairing = null; + +function resetSessionState() { + sock = null; + latestQr = null; + sentKeys.clear(); + recentlySentIds.clear(); + mkdirSync(SESSION_DIR, { recursive: true }); + for (const entry of readdirSync(SESSION_DIR)) { + rmSync(path.join(SESSION_DIR, entry), { recursive: true, force: true }); + } +} + +function resolvePendingPairing(payload) { + if (!pendingPairing) return; + clearTimeout(pendingPairing.timer); + pendingPairing.resolve(payload); + pendingPairing = null; +} + +function rejectPendingPairing(error) { + if (!pendingPairing) return; + clearTimeout(pendingPairing.timer); + pendingPairing.reject(error); + pendingPairing = null; +} + +async function maybeRequestPairingCode(update = {}) { + if (!pendingPairing || pendingPairing.inFlight || !sock) return; + + const canRequestPairingCode = + update.connection === "connecting" || + Boolean(update.qr) || + Boolean(latestQr); + + if (!canRequestPairingCode) return; + + pendingPairing.inFlight = true; + connectionState = "pairing"; + try { + const code = await sock.requestPairingCode(pendingPairing.phoneNumber); + resolvePendingPairing({ status: "pairing", pairing_code: code, expires_in: 60 }); + } catch (error) { + rejectPendingPairing(error); + } +} + +function requestPairingCodeWhenReady(phoneNumber) { + if (connectionState === "connected") { + return Promise.resolve({ status: "connected", pairing_code: null, expires_in: 0 }); + } + + if (pendingPairing) { + return Promise.reject(new Error("A WhatsApp pairing request is already in progress")); + } + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pendingPairing = null; + reject(new Error("Timed out waiting for WhatsApp to become ready for pairing")); + }, PAIRING_TIMEOUT_MS); + + pendingPairing = { + phoneNumber, + resolve, + reject, + timer, + inFlight: false, + }; + + void startSocket() + .then(() => maybeRequestPairingCode()) + .catch((error) => rejectPendingPairing(error)); + void maybeRequestPairingCode(); + }); +} function normalizeText(message) { const content = message?.message || {}; @@ -111,24 +188,33 @@ async function startSocket() { latestQr = qr; connectionState = "qr"; qrcode.generate(qr, { small: true }); + void maybeRequestPairingCode(update); } if (connection === "open") { latestQr = null; connectionState = "connected"; console.log("WhatsApp connected"); + resolvePendingPairing({ status: "connected", pairing_code: null, expires_in: 0 }); } if (connection === "close") { const reason = new Boom(lastDisconnect?.error)?.output?.statusCode; connectionState = "disconnected"; if (reason === DisconnectReason.loggedOut) { - console.error("WhatsApp logged out; clear the session volume and pair again."); - process.exit(1); + console.error("WhatsApp logged out; clearing session and waiting for pairing."); + connectionState = "logged_out"; + resetSessionState(); + setTimeout(() => { + starting = null; + void startSocket(); + }, 1000); + return; } setTimeout(() => { starting = null; void startSocket(); }, reason === 515 ? 1000 : 3000); } + void maybeRequestPairingCode(update); }); sock.ev.on("messages.upsert", ({ messages, type }) => { @@ -167,6 +253,7 @@ app.get("/health", (_req, res) => { res.json({ status: connectionState, hasQr: Boolean(latestQr), + qr: latestQr, queueDepth: messageQueue.length, user: sock?.user || null, }); @@ -238,17 +325,11 @@ app.post("/typing", async (req, res) => { app.post("/pair", async (req, res) => { try { - await startSocket(); const phoneNumber = String(req.body?.phoneNumber || req.body?.phone_number || "").replace(/\D/g, ""); - if (connectionState === "connected") { - return res.json({ status: "connected", pairing_code: null, expires_in: 0 }); - } if (!phoneNumber) { return res.status(400).json({ error: "phoneNumber is required for pairing code" }); } - connectionState = "pairing"; - const code = await sock.requestPairingCode(phoneNumber); - res.json({ status: "pairing", pairing_code: code, expires_in: 60 }); + res.json(await requestPairingCodeWhenReady(phoneNumber)); } catch (error) { res.status(500).json({ error: error?.message || "pairing failed" }); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx index 248abb121..b44f3ecbb 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx @@ -2,6 +2,7 @@ import { MessageCircle, RefreshCw, ShieldAlert } from "lucide-react"; import { useParams } from "next/navigation"; +import { QRCodeSVG } from "qrcode.react"; import { useCallback, useEffect, useState, useTransition } from "react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -37,16 +38,23 @@ type Pairing = { type PairingPlatform = "telegram" | "whatsapp"; +type BaileysHealth = { + status: string; + hasQr: boolean; + qr?: string | null; + queueDepth?: number; + user?: unknown; +}; + export function MessagingChannelsContent() { const params = useParams<{ search_space_id: string }>(); const searchSpaceId = Number(params.search_space_id); + const whatsappMode = process.env.NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE ?? "disabled"; const [bindings, setBindings] = useState([]); const [platforms, setPlatforms] = useState([]); const [pairing, setPairing] = useState(null); const [pairingPlatform, setPairingPlatform] = useState(null); - const [whatsappStatus, setWhatsappStatus] = useState(null); - const [baileysPhone, setBaileysPhone] = useState(""); - const [baileysCode, setBaileysCode] = useState(null); + const [baileysHealth, setBaileysHealth] = useState(null); const [loading, setLoading] = useState(true); const [isPending, startTransition] = useTransition(); @@ -65,6 +73,18 @@ export function MessagingChannelsContent() { void refresh(); }, [refresh]); + const refreshBaileysHealth = useCallback(async () => { + if (whatsappMode !== "baileys") return; + const res = await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/whatsapp/baileys/health`); + if (!res.ok) return; + const data = (await res.json()) as BaileysHealth; + setBaileysHealth(data); + }, [whatsappMode]); + + useEffect(() => { + void refreshBaileysHealth(); + }, [refreshBaileysHealth]); + async function startPairing(platform: PairingPlatform) { const res = await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings/start`, { method: "POST", @@ -76,25 +96,9 @@ export function MessagingChannelsContent() { await refresh(); } - function pairBaileys() { + function refreshBaileys() { startTransition(async () => { - setWhatsappStatus("Requesting WhatsApp pairing code..."); - const res = await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/whatsapp/baileys/pair`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ search_space_id: searchSpaceId, phone_number: baileysPhone }), - }); - if (!res.ok) { - setWhatsappStatus("Unable to request pairing code. Check the whatsapp-bridge service."); - return; - } - const data = await res.json(); - setBaileysCode(data.pairing_code ?? null); - setWhatsappStatus( - data.status === "connected" - ? "WhatsApp bridge is connected." - : "Enter the pairing code in WhatsApp.", - ); + await refreshBaileysHealth(); await refresh(); }); } @@ -115,7 +119,7 @@ export function MessagingChannelsContent() { const telegram = platforms.find((p) => p.platform === "telegram"); const whatsapp = platforms.find((p) => p.platform === "whatsapp"); - const whatsappMode = process.env.NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE ?? "disabled"; + const baileysQr = baileysHealth?.qr || null; const activeBindings = bindings.filter((binding) => binding.search_space_id === searchSpaceId); const renderPairingPanel = (platform: PairingPlatform) => { if (!pairing || pairingPlatform !== platform) return null; @@ -195,26 +199,30 @@ export function MessagingChannelsContent() { Self-hosted WhatsApp uses Message Yourself mode. After pairing, send messages in your own WhatsApp chat with yourself; messages from other chats are ignored.

- setBaileysPhone(event.target.value)} - /> - - {baileysCode ? ( + {baileysQr ? (
-

WhatsApp pairing code

-

{baileysCode}

+

WhatsApp QR pairing

+

+ Scan this QR from WhatsApp > Linked Devices > Link a Device. +

+
+ +
) : null} + {baileysHealth ? ( +

+ Bridge status: {baileysHealth.status} + {typeof baileysHealth.queueDepth === "number" + ? `, queue: ${baileysHealth.queueDepth}` + : ""} +

+ ) : null}
) : null} - {whatsappStatus ? ( -

{whatsappStatus}

- ) : null} ) : null} diff --git a/surfsense_web/package.json b/surfsense_web/package.json index 213adbaad..2eb5a45e7 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -126,6 +126,7 @@ "postgres": "^3.4.7", "posthog-js": "^1.336.1", "posthog-node": "^5.24.4", + "qrcode.react": "^4.2.0", "radix-ui": "^1.4.3", "react": "^19.2.3", "react-day-picker": "^9.13.2", diff --git a/surfsense_web/pnpm-lock.yaml b/surfsense_web/pnpm-lock.yaml index 8602feb8d..c2bd77cb5 100644 --- a/surfsense_web/pnpm-lock.yaml +++ b/surfsense_web/pnpm-lock.yaml @@ -299,6 +299,9 @@ importers: posthog-node: specifier: ^5.24.4 version: 5.24.17 + qrcode.react: + specifier: ^4.2.0 + version: 4.2.0(react@19.2.4) radix-ui: specifier: ^1.4.3 version: 1.4.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -1143,24 +1146,28 @@ packages: engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] + libc: [musl] '@biomejs/cli-linux-arm64@2.4.6': resolution: {integrity: sha512-kMLaI7OF5GN1Q8Doymjro1P8rVEoy7BKQALNz6fiR8IC1WKduoNyteBtJlHT7ASIL0Cx2jR6VUOBIbcB1B8pew==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] + libc: [glibc] '@biomejs/cli-linux-x64-musl@2.4.6': resolution: {integrity: sha512-C9s98IPDu7DYarjlZNuzJKTjVHN03RUnmHV5htvqsx6vEUXCDSJ59DNwjKVD5XYoSS4N+BYhq3RTBAL8X6svEg==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] + libc: [musl] '@biomejs/cli-linux-x64@2.4.6': resolution: {integrity: sha512-oHXmUFEoH8Lql1xfc3QkFLiC1hGR7qedv5eKNlC185or+o4/4HiaU7vYODAH3peRCfsuLr1g6v2fK9dFFOYdyw==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] + libc: [glibc] '@biomejs/cli-win32-arm64@2.4.6': resolution: {integrity: sha512-xzThn87Pf3YrOGTEODFGONmqXpTwUNxovQb72iaUOdcw8sBSY3+3WD8Hm9IhMYLnPi0n32s3L3NWU6+eSjfqFg==} @@ -1836,89 +1843,105 @@ packages: resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.4': resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.4': resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-riscv64@1.2.4': resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} cpu: [riscv64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.4': resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.4': resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.2.4': resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.4': resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-linux-arm64@0.34.5': resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-linux-arm@0.34.5': resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-linux-ppc64@0.34.5': resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-linux-riscv64@0.34.5': resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [riscv64] os: [linux] + libc: [glibc] '@img/sharp-linux-s390x@0.34.5': resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-linux-x64@0.34.5': resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-linuxmusl-arm64@0.34.5': resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-linuxmusl-x64@0.34.5': resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-wasm32@0.34.5': resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} @@ -2028,30 +2051,35 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@napi-rs/canvas-linux-arm64-musl@0.1.97': resolution: {integrity: sha512-kKmSkQVnWeqg7qdsiXvYxKhAFuHz3tkBjW/zyQv5YKUPhotpaVhpBGv5LqCngzyuRV85SXoe+OFj+Tv0a0QXkQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@napi-rs/canvas-linux-riscv64-gnu@0.1.97': resolution: {integrity: sha512-Jc7I3A51jnEOIAXeLsN/M/+Z28LUeakcsXs07FLq9prXc0eYOtVwsDEv913Gr+06IRo34gJJVgT0TXvmz+N2VA==} engines: {node: '>= 10'} cpu: [riscv64] os: [linux] + libc: [glibc] '@napi-rs/canvas-linux-x64-gnu@0.1.97': resolution: {integrity: sha512-iDUBe7AilfuBSRbSa8/IGX38Mf+iCSBqoVKLSQ5XaY2JLOaqz1TVyPFEyIck7wT6mRQhQt5sN6ogfjIDfi74tg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@napi-rs/canvas-linux-x64-musl@0.1.97': resolution: {integrity: sha512-AKLFd/v0Z5fvgqBDqhvqtAdx+fHMJ5t9JcUNKq4FIZ5WH+iegGm8HPdj00NFlCSnm83Fp3Ln8I2f7uq1aIiWaA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@napi-rs/canvas-win32-arm64-msvc@0.1.97': resolution: {integrity: sha512-u883Yr6A6fO7Vpsy9YE4FVCIxzzo5sO+7pIUjjoDLjS3vQaNMkVzx5bdIpEL+ob+gU88WDK4VcxYMZ6nmnoX9A==} @@ -2095,24 +2123,28 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@next/swc-linux-arm64-musl@16.1.6': resolution: {integrity: sha512-S4J2v+8tT3NIO9u2q+S0G5KdvNDjXfAv06OhfOzNDaBn5rw84DGXWndOEB7d5/x852A20sW1M56vhC/tRVbccQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@next/swc-linux-x64-gnu@16.1.6': resolution: {integrity: sha512-2eEBDkFlMMNQnkTyPBhQOAyn2qMxyG2eE7GPH2WIDGEpEILcBPI/jdSv4t6xupSP+ot/jkfrCShLAa7+ZUPcJQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@next/swc-linux-x64-musl@16.1.6': resolution: {integrity: sha512-oicJwRlyOoZXVlxmIMaTq7f8pN9QNbdes0q2FXfRsPhfCi8n8JmOZJm5oo1pwDaFbnnD421rVU409M3evFbIqg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@next/swc-win32-arm64-msvc@16.1.6': resolution: {integrity: sha512-gQmm8izDTPgs+DCWH22kcDmuUp7NyiJgEl18bcr8irXA5N2m2O+JQIr6f3ct42GOs9c0h8QF3L5SzIxcYAAXXw==} @@ -2768,48 +2800,56 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [glibc] '@oxfmt/binding-linux-arm64-musl@0.45.0': resolution: {integrity: sha512-XQKXZIKYJC3GQJ8FnD3iMntpw69Wd9kDDK/Xt79p6xnFYlGGxSNv2vIBvRTDg5CKByWFWWZLCRDOXoP/m6YN4g==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [musl] '@oxfmt/binding-linux-ppc64-gnu@0.45.0': resolution: {integrity: sha512-+g5RiG+xOkdrCWkKodv407nTvMq4vYM18Uox2MhZBm/YoqFxxJpWKsloskFFG5NU13HGPw1wzYjjOVcyd9moCA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@oxfmt/binding-linux-riscv64-gnu@0.45.0': resolution: {integrity: sha512-V7dXKoSyEbWAkkSF4JJNtF+NJZDmJoSarSoP30WCsB3X636Rehd3CvxBj49FIJxEBFWhvcUjGSHVeU8Erck1bQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [riscv64] os: [linux] + libc: [glibc] '@oxfmt/binding-linux-riscv64-musl@0.45.0': resolution: {integrity: sha512-Vdelft1sAEYojVGgcODEFXSWYQYlIvoyIGWebKCuUibd1tvS1TjTx413xG2ZLuHpYj45CkN/ztMLMX6jrgqpgg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [riscv64] os: [linux] + libc: [musl] '@oxfmt/binding-linux-s390x-gnu@0.45.0': resolution: {integrity: sha512-RR7xKgNpqwENnK0aYCGYg0JycY2n93J0reNjHyes+I9Gq52dH95x+CBlnlAQHCPfz6FGnKA9HirgUl14WO6o7w==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] + libc: [glibc] '@oxfmt/binding-linux-x64-gnu@0.45.0': resolution: {integrity: sha512-U/QQ0+BQNSHxjuXR/utvXnQ50Vu5kUuqEomZvQ1/3mhgbBiMc2WU9q5kZ5WwLp3gnFIx9ibkveoRSe2EZubkqg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [glibc] '@oxfmt/binding-linux-x64-musl@0.45.0': resolution: {integrity: sha512-o5TLOUCF0RWQjsIS06yVC+kFgp092/yLe6qBGSUvtnmTVw9gxjpdQSXc3VN5Cnive4K11HNstEZF8ROKHfDFSw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [musl] '@oxfmt/binding-openharmony-arm64@0.45.0': resolution: {integrity: sha512-RnGcV3HgPuOjsGx/k9oyRNKmOp+NBLGzZTdPDYbc19r7NGeYPplnUU/BfU35bX2Y/O4ejvHxcfkvW2WoYL/gsg==} @@ -2864,36 +2904,42 @@ packages: engines: {node: '>= 10.0.0'} cpu: [arm] os: [linux] + libc: [glibc] '@parcel/watcher-linux-arm-musl@2.5.6': resolution: {integrity: sha512-Ve3gUCG57nuUUSyjBq/MAM0CzArtuIOxsBdQ+ftz6ho8n7s1i9E1Nmk/xmP323r2YL0SONs1EuwqBp2u1k5fxg==} engines: {node: '>= 10.0.0'} cpu: [arm] os: [linux] + libc: [musl] '@parcel/watcher-linux-arm64-glibc@2.5.6': resolution: {integrity: sha512-f2g/DT3NhGPdBmMWYoxixqYr3v/UXcmLOYy16Bx0TM20Tchduwr4EaCbmxh1321TABqPGDpS8D/ggOTaljijOA==} engines: {node: '>= 10.0.0'} cpu: [arm64] os: [linux] + libc: [glibc] '@parcel/watcher-linux-arm64-musl@2.5.6': resolution: {integrity: sha512-qb6naMDGlbCwdhLj6hgoVKJl2odL34z2sqkC7Z6kzir8b5W65WYDpLB6R06KabvZdgoHI/zxke4b3zR0wAbDTA==} engines: {node: '>= 10.0.0'} cpu: [arm64] os: [linux] + libc: [musl] '@parcel/watcher-linux-x64-glibc@2.5.6': resolution: {integrity: sha512-kbT5wvNQlx7NaGjzPFu8nVIW1rWqV780O7ZtkjuWaPUgpv2NMFpjYERVi0UYj1msZNyCzGlaCWEtzc+exjMGbQ==} engines: {node: '>= 10.0.0'} cpu: [x64] os: [linux] + libc: [glibc] '@parcel/watcher-linux-x64-musl@2.5.6': resolution: {integrity: sha512-1JRFeC+h7RdXwldHzTsmdtYR/Ku8SylLgTU/reMuqdVD7CtLwf0VR1FqeprZ0eHQkO0vqsbvFLXUmYm/uNKJBg==} engines: {node: '>= 10.0.0'} cpu: [x64] os: [linux] + libc: [musl] '@parcel/watcher-win32-arm64@2.5.6': resolution: {integrity: sha512-3ukyebjc6eGlw9yRt678DxVF7rjXatWiHvTXqphZLvo7aC5NdEgFufVwjFfY51ijYEWpXbqF5jtrK275z52D4Q==} @@ -4222,66 +4268,79 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} @@ -4472,24 +4531,28 @@ packages: engines: {node: '>=10'} cpu: [arm64] os: [linux] + libc: [glibc] '@swc/core-linux-arm64-musl@1.15.13': resolution: {integrity: sha512-SmZ9m+XqCB35NddHCctvHFLqPZDAs5j8IgD36GoutufDJmeq2VNfgk5rQoqNqKmAK3Y7iFdEmI76QoHIWiCLyw==} engines: {node: '>=10'} cpu: [arm64] os: [linux] + libc: [musl] '@swc/core-linux-x64-gnu@1.15.13': resolution: {integrity: sha512-5rij+vB9a29aNkHq72EXI2ihDZPszJb4zlApJY4aCC/q6utgqFA6CkrfTfIb+O8hxtG3zP5KERETz8mfFK6A0A==} engines: {node: '>=10'} cpu: [x64] os: [linux] + libc: [glibc] '@swc/core-linux-x64-musl@1.15.13': resolution: {integrity: sha512-OlSlaOK9JplQ5qn07WiBLibkOw7iml2++ojEXhhR3rbWrNEKCD7sd8+6wSavsInyFdw4PhLA+Hy6YyDBIE23Yw==} engines: {node: '>=10'} cpu: [x64] os: [linux] + libc: [musl] '@swc/core-win32-arm64-msvc@1.15.13': resolution: {integrity: sha512-zwQii5YVdsfG8Ti9gIKgBKZg8qMkRZxl+OlYWUT5D93Jl4NuNBRausP20tfEkQdAPSRrMCSUZBM6FhW7izAZRg==} @@ -4573,24 +4636,28 @@ packages: engines: {node: '>= 20'} cpu: [arm64] os: [linux] + libc: [glibc] '@tailwindcss/oxide-linux-arm64-musl@4.2.1': resolution: {integrity: sha512-WZA0CHRL/SP1TRbA5mp9htsppSEkWuQ4KsSUumYQnyl8ZdT39ntwqmz4IUHGN6p4XdSlYfJwM4rRzZLShHsGAQ==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] + libc: [musl] '@tailwindcss/oxide-linux-x64-gnu@4.2.1': resolution: {integrity: sha512-qMFzxI2YlBOLW5PhblzuSWlWfwLHaneBE0xHzLrBgNtqN6mWfs+qYbhryGSXQjFYB1Dzf5w+LN5qbUTPhW7Y5g==} engines: {node: '>= 20'} cpu: [x64] os: [linux] + libc: [glibc] '@tailwindcss/oxide-linux-x64-musl@4.2.1': resolution: {integrity: sha512-5r1X2FKnCMUPlXTWRYpHdPYUY6a1Ar/t7P24OuiEdEOmms5lyqjDRvVY1yy9Rmioh+AunQ0rWiOTPE8F9A3v5g==} engines: {node: '>= 20'} cpu: [x64] os: [linux] + libc: [musl] '@tailwindcss/oxide-wasm32-wasi@4.2.1': resolution: {integrity: sha512-MGFB5cVPvshR85MTJkEvqDUnuNoysrsRxd6vnk1Lf2tbiqNlXpHYZqkqOQalydienEWOHHFyyuTSYRsLfxFJ2Q==} @@ -4915,41 +4982,49 @@ packages: resolution: {integrity: sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==} cpu: [arm64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-arm64-musl@1.11.1': resolution: {integrity: sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==} cpu: [arm64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-ppc64-gnu@1.11.1': resolution: {integrity: sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-gnu@1.11.1': resolution: {integrity: sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==} cpu: [riscv64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-musl@1.11.1': resolution: {integrity: sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==} cpu: [riscv64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-s390x-gnu@1.11.1': resolution: {integrity: sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==} cpu: [s390x] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-gnu@1.11.1': resolution: {integrity: sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==} cpu: [x64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-musl@1.11.1': resolution: {integrity: sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==} cpu: [x64] os: [linux] + libc: [musl] '@unrs/resolver-binding-wasm32-wasi@1.11.1': resolution: {integrity: sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==} @@ -6838,24 +6913,28 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [glibc] lightningcss-linux-arm64-musl@1.31.1: resolution: {integrity: sha512-mVZ7Pg2zIbe3XlNbZJdjs86YViQFoJSpc41CbVmKBPiGmC4YrfeOyz65ms2qpAobVd7WQsbW4PdsSJEMymyIMg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [musl] lightningcss-linux-x64-gnu@1.31.1: resolution: {integrity: sha512-xGlFWRMl+0KvUhgySdIaReQdB4FNudfUTARn7q0hh/V67PVGCs3ADFjw+6++kG1RNd0zdGRlEKa+T13/tQjPMA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [glibc] lightningcss-linux-x64-musl@1.31.1: resolution: {integrity: sha512-eowF8PrKHw9LpoZii5tdZwnBcYDxRw2rRCyvAXLi34iyeYfqCQNA9rmUM0ce62NlPhCvof1+9ivRaTY6pSKDaA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [musl] lightningcss-win32-arm64-msvc@1.31.1: resolution: {integrity: sha512-aJReEbSEQzx1uBlQizAOBSjcmr9dCdL3XuC/6HLXAxmtErsj2ICo5yYggg1qOODQMtnjNQv2UHb9NpOuFtYe4w==} @@ -7641,6 +7720,11 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + qrcode.react@4.2.0: + resolution: {integrity: sha512-QpgqWi8rD9DsS9EP3z7BT+5lY5SFhsqGjpgW5DY/i3mK4M9DTBNz3ErMi8BWYEfI3L0d8GIbGmcdFAS1uIRGjA==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + query-selector-shadow-dom@1.0.1: resolution: {integrity: sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==} @@ -16759,6 +16843,10 @@ snapshots: punycode@2.3.1: {} + qrcode.react@4.2.0(react@19.2.4): + dependencies: + react: 19.2.4 + query-selector-shadow-dom@1.0.1: {} queue-microtask@1.2.3: {} From b0b0f3517b2cf76dce7aefc4aa5690e3b85dcb11 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:35:52 +0530 Subject: [PATCH 37/63] feat(gateway): add Slack external chat platform --- .../145_add_slack_gateway_platform.py | 102 ++++++++++++++++++ surfsense_backend/app/db.py | 14 ++- 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 surfsense_backend/alembic/versions/145_add_slack_gateway_platform.py diff --git a/surfsense_backend/alembic/versions/145_add_slack_gateway_platform.py b/surfsense_backend/alembic/versions/145_add_slack_gateway_platform.py new file mode 100644 index 000000000..f4ab18e72 --- /dev/null +++ b/surfsense_backend/alembic/versions/145_add_slack_gateway_platform.py @@ -0,0 +1,102 @@ +"""add slack gateway platform + +Revision ID: 145 +Revises: 144 +Create Date: 2026-05-31 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "145" +down_revision: str | None = "144" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _enum_value_exists(enum_name: str, value: str) -> bool: + conn = op.get_bind() + return ( + conn.execute( + sa.text( + "SELECT 1 FROM pg_enum e " + "JOIN pg_type t ON t.oid = e.enumtypid " + "WHERE t.typname = :enum_name AND e.enumlabel = :value" + ), + {"enum_name": enum_name, "value": value}, + ).fetchone() + is not None + ) + + +def _index_exists(index_name: str) -> bool: + conn = op.get_bind() + return ( + conn.execute( + sa.text( + "SELECT 1 FROM pg_indexes " + "WHERE schemaname = current_schema() AND indexname = :index_name" + ), + {"index_name": index_name}, + ).fetchone() + is not None + ) + + +def upgrade() -> None: + if not _enum_value_exists("external_chat_platform", "slack"): + op.execute("ALTER TYPE external_chat_platform ADD VALUE 'slack'") + + if _index_exists("uq_external_chat_accounts_system_platform"): + op.drop_index( + "uq_external_chat_accounts_system_platform", + table_name="external_chat_accounts", + ) + + op.create_index( + "uq_external_chat_accounts_system_platform", + "external_chat_accounts", + ["platform"], + unique=True, + postgresql_where=sa.text( + "is_system_account = true AND NOT (cursor_state ? 'team_id')" + ), + if_not_exists=True, + ) + op.create_index( + "uq_external_chat_accounts_slack_team", + "external_chat_accounts", + ["platform", sa.text("(cursor_state ->> 'team_id')")], + unique=True, + postgresql_where=sa.text( + "is_system_account = true AND cursor_state ? 'team_id'" + ), + if_not_exists=True, + ) + + +def downgrade() -> None: + if _index_exists("uq_external_chat_accounts_slack_team"): + op.drop_index( + "uq_external_chat_accounts_slack_team", + table_name="external_chat_accounts", + ) + if _index_exists("uq_external_chat_accounts_system_platform"): + op.drop_index( + "uq_external_chat_accounts_system_platform", + table_name="external_chat_accounts", + ) + op.create_index( + "uq_external_chat_accounts_system_platform", + "external_chat_accounts", + ["platform"], + unique=True, + postgresql_where=sa.text("is_system_account = true"), + if_not_exists=True, + ) + # PostgreSQL enum values are intentionally not removed on downgrade. diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index e9b301ece..14ba8cdec 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -577,6 +577,7 @@ class ChatVisibility(StrEnum): class ExternalChatPlatform(StrEnum): TELEGRAM = "telegram" WHATSAPP = "whatsapp" + SLACK = "slack" SIGNAL = "signal" @@ -888,7 +889,18 @@ class ExternalChatAccount(Base, TimestampMixin): "uq_external_chat_accounts_system_platform", "platform", unique=True, - postgresql_where=text("is_system_account = true"), + postgresql_where=text( + "is_system_account = true AND NOT (cursor_state ? 'team_id')" + ), + ), + Index( + "uq_external_chat_accounts_slack_team", + "platform", + text("(cursor_state ->> 'team_id')"), + unique=True, + postgresql_where=text( + "is_system_account = true AND cursor_state ? 'team_id'" + ), ), Index( "uq_external_chat_accounts_webhook_secret", From 5b71685dad89f87e9dd27ac07d3cf935be3fdb20 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:36:07 +0530 Subject: [PATCH 38/63] feat(gateway): add Slack gateway configuration --- surfsense_backend/.env.example | 5 +++- surfsense_backend/app/config/__init__.py | 4 +++ surfsense_backend/app/gateway/accounts.py | 30 +++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 6ddf18ebb..2d6c27e26 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -141,10 +141,13 @@ NOTION_CLIENT_ID=your_notion_client_id_here NOTION_CLIENT_SECRET=your_notion_client_secret_here NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback -# Slack OAuth Configuration +# Slack OAuth / Gateway Configuration +# The Slack connector and Slack gateway can use the same Slack app client ID/secret. SLACK_CLIENT_ID=your_slack_client_id_here SLACK_CLIENT_SECRET=your_slack_client_secret_here SLACK_REDIRECT_URI=http://localhost:8000/api/v1/auth/slack/connector/callback +GATEWAY_SLACK_SIGNING_SECRET=your_slack_signing_secret_here +GATEWAY_SLACK_REDIRECT_URI=http://localhost:8000/api/v1/gateway/slack/callback # Microsoft OAuth (Teams & OneDrive) MICROSOFT_CLIENT_ID=your_microsoft_client_id_here diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index c8ffa802e..fdf8834c1 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -570,6 +570,10 @@ class Config: raise ValueError( "GATEWAY_WHATSAPP_INTAKE_MODE must be one of: cloud, baileys, disabled" ) + GATEWAY_SLACK_CLIENT_ID = os.getenv("SLACK_CLIENT_ID") + GATEWAY_SLACK_CLIENT_SECRET = os.getenv("SLACK_CLIENT_SECRET") + GATEWAY_SLACK_SIGNING_SECRET = os.getenv("GATEWAY_SLACK_SIGNING_SECRET") + GATEWAY_SLACK_REDIRECT_URI = os.getenv("GATEWAY_SLACK_REDIRECT_URI") # Stripe checkout for pay-as-you-go page packs STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY") diff --git a/surfsense_backend/app/gateway/accounts.py b/surfsense_backend/app/gateway/accounts.py index 7379336a7..5daf75c69 100644 --- a/surfsense_backend/app/gateway/accounts.py +++ b/surfsense_backend/app/gateway/accounts.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -25,6 +27,19 @@ def account_token(account: ExternalChatAccount) -> str | None: ) +def slack_account_credentials(account: ExternalChatAccount) -> dict: + """Decrypt Slack gateway credentials stored as encrypted JSON.""" + if not account.encrypted_credentials: + return {} + raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token(account.encrypted_credentials) + try: + data = json.loads(raw) + except json.JSONDecodeError: + # Backward-compatible fallback if a token string was stored directly. + return {"bot_token": raw} + return data if isinstance(data, dict) else {} + + async def get_or_create_system_telegram_account( session: AsyncSession, ) -> ExternalChatAccount: @@ -78,3 +93,18 @@ async def get_or_create_system_whatsapp_account( await session.flush() return account + +async def get_slack_account_by_team( + session: AsyncSession, + *, + team_id: str, +) -> ExternalChatAccount | None: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.SLACK, + ExternalChatAccount.is_system_account.is_(True), + ExternalChatAccount.cursor_state["team_id"].astext == team_id, + ) + ) + return result.scalars().first() + From 78315eb55b09cea5e28323069d0bdd1611821ec7 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:36:27 +0530 Subject: [PATCH 39/63] feat(gateway): add Slack platform adapter --- .../app/gateway/slack/__init__.py | 1 + .../app/gateway/slack/adapter.py | 120 ++++++++++++++++++ surfsense_backend/app/gateway/slack/client.py | 72 +++++++++++ .../tests/unit/gateway/test_slack_adapter.py | 47 +++++++ 4 files changed, 240 insertions(+) create mode 100644 surfsense_backend/app/gateway/slack/__init__.py create mode 100644 surfsense_backend/app/gateway/slack/adapter.py create mode 100644 surfsense_backend/app/gateway/slack/client.py create mode 100644 surfsense_backend/tests/unit/gateway/test_slack_adapter.py diff --git a/surfsense_backend/app/gateway/slack/__init__.py b/surfsense_backend/app/gateway/slack/__init__.py new file mode 100644 index 000000000..7f7aaf2fc --- /dev/null +++ b/surfsense_backend/app/gateway/slack/__init__.py @@ -0,0 +1 @@ +"""Slack gateway integration.""" diff --git a/surfsense_backend/app/gateway/slack/adapter.py b/surfsense_backend/app/gateway/slack/adapter.py new file mode 100644 index 000000000..e49ca6b9c --- /dev/null +++ b/surfsense_backend/app/gateway/slack/adapter.py @@ -0,0 +1,120 @@ +"""Slack platform adapter for app mentions and threaded replies.""" + +from __future__ import annotations + +import re +from typing import Any + +from app.gateway.base.adapter import ( + BasePlatformAdapter, + ParsedInboundEvent, + PlatformSendResult, +) +from app.gateway.slack.client import SlackGatewayClient + +MENTION_RE = re.compile(r"<@[^>]+>\s*") + + +def slack_user_peer_id(team_id: str, slack_user_id: str) -> str: + return f"slack_user:{team_id}:{slack_user_id}" + + +def slack_thread_peer_id(team_id: str, channel_id: str, thread_ts: str) -> str: + return f"slack_thread:{team_id}:{channel_id}:{thread_ts}" + + +class SlackAdapter(BasePlatformAdapter): + platform = "slack" + + def __init__(self, bot_token: str, *, bot_user_id: str | None = None) -> None: + self.bot_user_id = bot_user_id + self.client = SlackGatewayClient(bot_token) + + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + event = raw_payload.get("event") or {} + event_type = str(event.get("type") or "other") + team_id = str(raw_payload.get("team_id") or event.get("team") or "") + channel_id = str(event.get("channel") or "") + slack_user_id = str(event.get("user") or "") + message_ts = str(event.get("ts") or "") + thread_ts = str(event.get("thread_ts") or message_ts) + bot_user_id = self.bot_user_id or str(raw_payload.get("authorizations", [{}])[0].get("user_id") or "") + + if not channel_id or not slack_user_id or not message_ts: + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_type, + external_peer_id=None, + external_peer_kind="unknown", + external_message_id=message_ts or None, + external_user_id=slack_user_id or None, + text=None, + raw_payload=raw_payload, + metadata={"team_id": team_id, "bot_user_id": bot_user_id}, + ) + + text = str(event.get("text") or "") + if bot_user_id: + text = text.replace(f"<@{bot_user_id}>", "") + text = MENTION_RE.sub("", text).strip() + + peer_kind = "direct" if str(event.get("channel_type")) == "im" else "channel" + thread_key = slack_thread_peer_id(team_id, channel_id, thread_ts) + user_key = slack_user_peer_id(team_id, slack_user_id) + + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_type, + external_peer_id=thread_key, + external_peer_kind=peer_kind, + external_message_id=message_ts, + external_user_id=slack_user_id, + text=text, + raw_payload=raw_payload, + display_name=None, + username=slack_user_id, + metadata={ + "team_id": team_id, + "channel_id": channel_id, + "slack_user_id": slack_user_id, + "message_ts": message_ts, + "thread_ts": thread_ts, + "bot_user_id": bot_user_id, + "slack_user_peer_id": user_key, + "slack_thread_peer_id": thread_key, + "channel_type": event.get("channel_type"), + }, + ) + + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + del parse_mode + return await self.client.send_message( + channel=external_peer_id, + text=text, + thread_ts=reply_to_message_id, + ) + + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + del parse_mode + return await self.client.update_message( + channel=external_peer_id, + ts=external_message_id, + text=text, + ) + + async def validate_credentials(self) -> dict[str, Any]: + return await self.client.validate() diff --git a/surfsense_backend/app/gateway/slack/client.py b/surfsense_backend/app/gateway/slack/client.py new file mode 100644 index 000000000..37ccda3bd --- /dev/null +++ b/surfsense_backend/app/gateway/slack/client.py @@ -0,0 +1,72 @@ +"""Slack Web API client for gateway bot operations.""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from app.gateway.base.adapter import PlatformSendResult + +SLACK_API = "https://slack.com/api" + + +class SlackGatewayClient: + def __init__(self, bot_token: str) -> None: + self.bot_token = bot_token + + async def api_call(self, method: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=20.0) as client: + response = await client.post( + f"{SLACK_API}/{method}", + json=payload or {}, + headers={ + "Authorization": f"Bearer {self.bot_token}", + "Content-Type": "application/json; charset=utf-8", + }, + ) + response.raise_for_status() + data = response.json() + if not data.get("ok", False): + error = data.get("error", "unknown_error") + raise RuntimeError(f"Slack API {method} failed: {error}") + return data + + async def send_message( + self, + *, + channel: str, + text: str, + thread_ts: str | None = None, + ) -> PlatformSendResult: + payload: dict[str, Any] = {"channel": channel, "text": text} + if thread_ts: + payload["thread_ts"] = thread_ts + data = await self.api_call("chat.postMessage", payload) + return PlatformSendResult( + external_message_id=str(data.get("ts", "")), + raw_response=data, + ) + + async def update_message( + self, + *, + channel: str, + ts: str, + text: str, + ) -> PlatformSendResult: + data = await self.api_call("chat.update", {"channel": channel, "ts": ts, "text": text}) + return PlatformSendResult( + external_message_id=str(data.get("ts") or ts), + raw_response=data, + ) + + async def validate(self) -> dict[str, Any]: + data = await self.api_call("auth.test") + return { + "ok": True, + "team_id": data.get("team_id"), + "team": data.get("team"), + "bot_user_id": data.get("user_id"), + "bot_username": data.get("user"), + } diff --git a/surfsense_backend/tests/unit/gateway/test_slack_adapter.py b/surfsense_backend/tests/unit/gateway/test_slack_adapter.py new file mode 100644 index 000000000..8742a6bf4 --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_slack_adapter.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from app.gateway.slack.adapter import SlackAdapter + + +def test_slack_adapter_parses_app_mention_and_strips_bot_mention(): + adapter = SlackAdapter("xoxb-test", bot_user_id="U_BOT") + + parsed = adapter.parse_inbound( + { + "team_id": "T123", + "event": { + "type": "app_mention", + "channel": "C123", + "user": "U123", + "text": "<@U_BOT> summarize this thread", + "ts": "1717000000.000100", + }, + } + ) + + assert parsed.platform == "slack" + assert parsed.text == "summarize this thread" + assert parsed.external_peer_id == "slack_thread:T123:C123:1717000000.000100" + assert parsed.metadata["slack_user_peer_id"] == "slack_user:T123:U123" + assert parsed.metadata["thread_ts"] == "1717000000.000100" + + +def test_slack_adapter_uses_existing_thread_ts(): + adapter = SlackAdapter("xoxb-test", bot_user_id="U_BOT") + + parsed = adapter.parse_inbound( + { + "team_id": "T123", + "event": { + "type": "app_mention", + "channel": "C123", + "user": "U123", + "text": "<@U_BOT> continue", + "ts": "1717000001.000200", + "thread_ts": "1717000000.000100", + }, + } + ) + + assert parsed.external_peer_id == "slack_thread:T123:C123:1717000000.000100" + assert parsed.metadata["message_ts"] == "1717000001.000200" From 61a3586caf0729f76a10ae5c40281c2f5b4bda50 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:36:39 +0530 Subject: [PATCH 40/63] feat(gateway): handle Slack thread replies --- .../app/gateway/slack/commands.py | 64 ++++++++++++++ .../app/gateway/slack/translator.py | 86 +++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 surfsense_backend/app/gateway/slack/commands.py create mode 100644 surfsense_backend/app/gateway/slack/translator.py diff --git a/surfsense_backend/app/gateway/slack/commands.py b/surfsense_backend/app/gateway/slack/commands.py new file mode 100644 index 000000000..ffbd5863b --- /dev/null +++ b/surfsense_backend/app/gateway/slack/commands.py @@ -0,0 +1,64 @@ +"""Slack command/onboarding handlers.""" + +from __future__ import annotations + +from app.gateway.base.adapter import ParsedInboundEvent +from app.gateway.base.commands import BaseGatewayCommands +from app.gateway.ratelimit import acquire_token +from app.gateway.slack.adapter import SlackAdapter + +HELP_TEXT = ( + "SurfSense Slack commands:\n" + "`/new` - start a fresh SurfSense conversation in this thread\n" + "`/help` - show this help\n\n" + "Mention the SurfSense bot in a channel thread to ask your agent a question." +) + + +class SlackGatewayCommands(BaseGatewayCommands): + async def handle_help_command( + self, + *, + adapter: SlackAdapter, + event: ParsedInboundEvent, + ) -> bool: + channel_id = event.metadata.get("channel_id") + thread_ts = event.metadata.get("thread_ts") + if not channel_id or not thread_ts: + return True + await adapter.send_message( + external_peer_id=channel_id, + text=HELP_TEXT, + reply_to_message_id=thread_ts, + ) + return True + + async def send_unbound_onboarding( + self, + *, + adapter: SlackAdapter, + event: ParsedInboundEvent, + dashboard_url: str, + ) -> None: + channel_id = event.metadata.get("channel_id") + thread_ts = event.metadata.get("thread_ts") + slack_user_id = event.metadata.get("slack_user_id") + if not channel_id or not thread_ts: + return + + wait_ms = await acquire_token( + f"slack:onboarded:{event.metadata.get('team_id')}:{slack_user_id}", + capacity=1, + refill_per_sec=1 / 3600, + ) + if wait_ms > 0: + return + + await adapter.send_message( + external_peer_id=channel_id, + reply_to_message_id=thread_ts, + text=( + "Hi! Connect your Slack user to SurfSense before using the bot here: " + f"{dashboard_url}" + ), + ) diff --git a/surfsense_backend/app/gateway/slack/translator.py b/surfsense_backend/app/gateway/slack/translator.py new file mode 100644 index 000000000..658b0cac7 --- /dev/null +++ b/surfsense_backend/app/gateway/slack/translator.py @@ -0,0 +1,86 @@ +"""Translate agent stream events into Slack thread replies.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator + +from app.gateway.base.adapter import PlatformSendResult +from app.gateway.base.formatting import split_text_message +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.ratelimit import wait_for_token +from app.gateway.slack.adapter import SlackAdapter +from app.observability.metrics import ( + record_gateway_hitl_aborted, + record_gateway_outbound, + record_gateway_rate_limit_hit, +) + +logger = logging.getLogger(__name__) + +SLACK_MAX_MESSAGE_CHARS = 35000 +HITL_UNSUPPORTED_MESSAGE = ( + "This action requires approval and is not yet supported from Slack. " + "Try again with a different request." +) + + +class SlackStreamTranslator(BaseStreamTranslator): + def __init__( + self, + *, + adapter: SlackAdapter, + channel_id: str, + thread_ts: str, + ) -> None: + self.adapter = adapter + self.channel_id = channel_id + self.thread_ts = thread_ts + self._buffer = "" + + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + async for event in events: + if event.type in {"text-delta", "text_delta", "text"}: + self._buffer += str(event.data.get("text") or event.data.get("delta") or "") + elif event.type in {"data-interrupt-request", "interrupt"}: + await self._handle_hitl_interrupt() + return + elif event.type in {"finish", "done"}: + break + + await self._flush_final() + + async def _flush_final(self) -> None: + if not self._buffer: + return + for chunk in split_text_message(self._buffer, max_chars=SLACK_MAX_MESSAGE_CHARS): + await self._send_text(chunk) + + async def _send_text(self, text: str) -> PlatformSendResult: + await self._throttle() + try: + result = await self.adapter.send_message( + external_peer_id=self.channel_id, + text=text, + reply_to_message_id=self.thread_ts, + ) + except Exception: + record_gateway_outbound(platform="slack", kind="send", status="failed") + raise + record_gateway_outbound(platform="slack", kind="send", status="sent") + return result + + async def _throttle(self) -> None: + chat_wait = await wait_for_token( + f"slack:channel:{self.channel_id}", + capacity=1, + refill_per_sec=1.0, + ) + if chat_wait: + record_gateway_rate_limit_hit(bucket="slack:channel") + + async def _handle_hitl_interrupt(self) -> None: + if self._buffer: + await self._flush_final() + await self._send_text(HITL_UNSUPPORTED_MESSAGE) + record_gateway_hitl_aborted(platform="slack") From f305a2e67dce8f154dae1111df9ba102595e30fe Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:36:53 +0530 Subject: [PATCH 41/63] feat(gateway): route Slack events through external chat --- surfsense_backend/app/gateway/bindings.py | 11 ++- surfsense_backend/app/gateway/inbox.py | 4 + .../app/gateway/inbox_processor.py | 97 ++++++++++++++++--- surfsense_backend/app/gateway/registry.py | 39 +++++++- 4 files changed, 136 insertions(+), 15 deletions(-) diff --git a/surfsense_backend/app/gateway/bindings.py b/surfsense_backend/app/gateway/bindings.py index e7205c5f1..971633571 100644 --- a/surfsense_backend/app/gateway/bindings.py +++ b/surfsense_backend/app/gateway/bindings.py @@ -9,8 +9,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.db import ( ChatVisibility, - ExternalChatBindingState, ExternalChatBinding, + ExternalChatBindingState, NewChatThread, ) @@ -27,12 +27,17 @@ async def get_or_create_thread_for_binding( if thread is not None and not thread.archived: return thread + source = str((binding.external_metadata or {}).get("platform") or "").strip() + if not source: + kind = str((binding.external_metadata or {}).get("kind") or "") + source = "slack" if kind.startswith("slack_") else "telegram" + thread = NewChatThread( - title="Telegram chat", + title=f"{source.title()} chat", search_space_id=binding.search_space_id, created_by_id=binding.user_id, visibility=ChatVisibility.PRIVATE, - source="telegram", + source=source, external_chat_binding_id=binding.id, ) session.add(thread) diff --git a/surfsense_backend/app/gateway/inbox.py b/surfsense_backend/app/gateway/inbox.py index 9bc660b9d..5769c8cc4 100644 --- a/surfsense_backend/app/gateway/inbox.py +++ b/surfsense_backend/app/gateway/inbox.py @@ -12,6 +12,10 @@ def telegram_event_dedupe_key(update_id: int | str) -> str: return f"update:{update_id}" +def slack_event_dedupe_key(event_id: int | str) -> str: + return f"slack_event:{event_id}" + + async def persist_inbound_event( session: AsyncSession, *, diff --git a/surfsense_backend/app/gateway/inbox_processor.py b/surfsense_backend/app/gateway/inbox_processor.py index bdf768d61..3e87b582d 100644 --- a/surfsense_backend/app/gateway/inbox_processor.py +++ b/surfsense_backend/app/gateway/inbox_processor.py @@ -21,6 +21,7 @@ from app.db import ( ExternalChatEventStatus, ExternalChatInboundEvent, ExternalChatPeerKind, + ExternalChatPlatform, NewChatThread, async_session_maker, ) @@ -128,6 +129,86 @@ async def _mark_failed( await session.commit() +async def _resolve_binding_for_event( + session: AsyncSession, + account: ExternalChatAccount, + parsed, +) -> ExternalChatBinding | None: + if account.platform == ExternalChatPlatform.SLACK: + return await _resolve_slack_thread_binding(session, account, parsed) + + result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == parsed.external_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + return result.scalars().first() + + +async def _resolve_slack_thread_binding( + session: AsyncSession, + account: ExternalChatAccount, + parsed, +) -> ExternalChatBinding | None: + user_peer_id = parsed.metadata.get("slack_user_peer_id") + thread_peer_id = parsed.metadata.get("slack_thread_peer_id") or parsed.external_peer_id + if not user_peer_id or not thread_peer_id: + return None + + user_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == user_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + user_binding = user_result.scalars().first() + if user_binding is None: + return None + + thread_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == thread_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + thread_binding = thread_result.scalars().first() + if thread_binding is not None: + return thread_binding + + thread_binding = ExternalChatBinding( + account_id=account.id, + user_id=user_binding.user_id, + search_space_id=user_binding.search_space_id, + state=ExternalChatBindingState.BOUND, + external_peer_id=thread_peer_id, + external_peer_kind=ExternalChatPeerKind.CHANNEL, + external_thread_id=parsed.metadata.get("thread_ts"), + external_display_name=parsed.metadata.get("channel_id"), + external_username=parsed.external_user_id, + external_metadata={ + "kind": "slack_thread", + "team_id": parsed.metadata.get("team_id"), + "channel_id": parsed.metadata.get("channel_id"), + "thread_ts": parsed.metadata.get("thread_ts"), + "slack_user_id": parsed.metadata.get("slack_user_id"), + "user_binding_id": user_binding.id, + }, + ) + session.add(thread_binding) + await session.flush() + return thread_binding + + async def _dispatch_inbound_event( inbox_id: int, session_maker: SessionMaker, @@ -161,18 +242,12 @@ async def _dispatch_inbound_event( _update_account_cursor(account, parsed.metadata.get("update_id")) - result = await session.execute( - select(ExternalChatBinding).where( - ExternalChatBinding.account_id == account.id, - ExternalChatBinding.external_peer_id == parsed.external_peer_id, - ExternalChatBinding.state.in_( - [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] - ), - ) - ) - binding = result.scalars().first() + binding = await _resolve_binding_for_event(session, account, parsed) - if parsed.external_peer_kind != ExternalChatPeerKind.DIRECT.value: + if ( + account.platform != ExternalChatPlatform.SLACK + and parsed.external_peer_kind != ExternalChatPeerKind.DIRECT.value + ): if hasattr(adapter, "leave_chat"): await adapter.leave_chat(external_peer_id=parsed.external_peer_id) event.status = ExternalChatEventStatus.IGNORED diff --git a/surfsense_backend/app/gateway/registry.py b/surfsense_backend/app/gateway/registry.py index db334b7f1..fc9cb37e5 100644 --- a/surfsense_backend/app/gateway/registry.py +++ b/surfsense_backend/app/gateway/registry.py @@ -6,7 +6,7 @@ from collections.abc import Callable from dataclasses import dataclass from app.db import ExternalChatAccount, ExternalChatAccountMode, ExternalChatPlatform -from app.gateway.accounts import account_token +from app.gateway.accounts import account_token, slack_account_credentials from app.gateway.base.adapter import BasePlatformAdapter, ParsedInboundEvent from app.gateway.base.commands import BaseGatewayCommands from app.gateway.base.translator import BaseStreamTranslator @@ -70,6 +70,23 @@ def _whatsapp_baileys_translator_factory( ) +def _slack_translator_factory( + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> BaseStreamTranslator: + channel_id = event.metadata.get("channel_id") + thread_ts = event.metadata.get("thread_ts") + if not channel_id or not thread_ts: + raise RuntimeError("missing_slack_thread_metadata") + from app.gateway.slack.translator import SlackStreamTranslator + + return SlackStreamTranslator( + adapter=adapter, # type: ignore[arg-type] + channel_id=channel_id, + thread_ts=thread_ts, + ) + + def resolve_platform_bundle(account: ExternalChatAccount) -> PlatformBundle: if account.platform == ExternalChatPlatform.TELEGRAM: token = account_token(account) @@ -108,4 +125,24 @@ def resolve_platform_bundle(account: ExternalChatAccount) -> PlatformBundle: auto_bind_owner=True, ) + if account.platform == ExternalChatPlatform.SLACK: + from app.gateway.slack.adapter import SlackAdapter + from app.gateway.slack.commands import SlackGatewayCommands + + credentials = slack_account_credentials(account) + bot_token = credentials.get("bot_token") + if not bot_token: + raise RuntimeError("missing_slack_bot_token") + cursor_state = account.cursor_state or {} + return PlatformBundle( + adapter=SlackAdapter( + bot_token, + bot_user_id=cursor_state.get("bot_user_id"), + ), + translator_factory=_slack_translator_factory, + platform_label="slack", + commands=SlackGatewayCommands(), + auto_bind_owner=False, + ) + raise RuntimeError(f"unsupported_gateway_platform:{account.platform.value}:{account.mode.value}") From 9c7e093db41481e1fbce924b9fb2094a8040119d Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:37:04 +0530 Subject: [PATCH 42/63] feat(gateway): add Slack gateway webhook flow --- .../app/routes/gateway_webhook_routes.py | 295 +++++++++++++++++- .../tests/unit/gateway/test_webhook_routes.py | 127 +++++++- 2 files changed, 418 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py index 5508e534c..ffadd19d7 100644 --- a/surfsense_backend/app/routes/gateway_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -2,24 +2,32 @@ from __future__ import annotations +import hashlib import hmac +import json import logging +import time import uuid from datetime import UTC, datetime from typing import Any -from urllib.parse import quote +from urllib.parse import quote, urlencode +from uuid import UUID +import httpx from fastapi import APIRouter, Depends, HTTPException, Request from pydantic import BaseModel from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from starlette.responses import Response +from starlette.responses import JSONResponse, RedirectResponse, Response from app.config import config from app.db import ( ExternalChatAccount, + ExternalChatAccountMode, ExternalChatBinding, ExternalChatBindingState, + ExternalChatHealthStatus, + ExternalChatPeerKind, ExternalChatPlatform, User, get_async_session, @@ -27,19 +35,92 @@ from app.db import ( from app.gateway.accounts import ( get_or_create_system_telegram_account, get_or_create_system_whatsapp_account, + get_slack_account_by_team, ) from app.gateway.bindings import resume_binding, revoke_binding -from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key +from app.gateway.inbox import ( + persist_inbound_event, + slack_event_dedupe_key, + telegram_event_dedupe_key, +) from app.gateway.pairing import generate_pairing_code, pairing_expires_at +from app.gateway.slack.adapter import slack_user_peer_id from app.observability.metrics import ( record_gateway_inbox_write, record_gateway_webhook_parse_error, ) from app.users import current_active_user +from app.utils.oauth_security import OAuthStateManager, TokenEncryption router = APIRouter(prefix="/gateway", tags=["gateway"]) logger = logging.getLogger(__name__) +SLACK_AUTHORIZATION_URL = "https://slack.com/oauth/v2/authorize" +SLACK_TOKEN_URL = "https://slack.com/api/oauth.v2.access" +SLACK_BOT_SCOPES = [ + "app_mentions:read", + "chat:write", + "channels:read", + "groups:read", + "im:write", + "users:read", + "team:read", +] +_state_manager: OAuthStateManager | None = None +_token_encryption: TokenEncryption | None = None + + +def _get_state_manager() -> OAuthStateManager: + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise HTTPException(status_code=500, detail="SECRET_KEY is not configured") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def _get_token_encryption() -> TokenEncryption: + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise HTTPException(status_code=500, detail="SECRET_KEY is not configured") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +def _slack_redirect_uri() -> str: + if config.GATEWAY_SLACK_REDIRECT_URI: + return config.GATEWAY_SLACK_REDIRECT_URI + base = config.BACKEND_URL or "" + return f"{base.rstrip('/')}/api/v1/gateway/slack/callback" + + +def _slack_frontend_redirect(space_id: int, *, success: bool = False, error: str | None = None) -> RedirectResponse: + qs = "slack_gateway=connected" if success else f"error={error or 'slack_gateway_failed'}" + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/user-settings?{qs}" + ) + + +def verify_slack_signature(*, signing_secret: str, timestamp: str | None, signature: str | None, body: bytes) -> bool: + if not signing_secret or not timestamp or not signature: + return False + try: + ts = int(timestamp) + except ValueError: + return False + if abs(time.time() - ts) > 60 * 5: + return False + base = b"v0:" + timestamp.encode() + b":" + body + digest = hmac.new(signing_secret.encode(), base, hashlib.sha256).hexdigest() + expected = f"v0={digest}" + return hmac.compare_digest(expected, signature) + + +def _slack_event_kind(payload: dict[str, Any]) -> str: + event_type = str((payload.get("event") or {}).get("type") or "") + return "message" if event_type in {"app_mention", "message"} else "other" + class StartBindingRequest(BaseModel): platform: ExternalChatPlatform = ExternalChatPlatform.TELEGRAM @@ -67,6 +148,213 @@ def _telegram_message(payload: dict[str, Any]) -> dict[str, Any] | None: return payload.get("message") or payload.get("edited_message") +@router.get("/slack/install") +async def install_slack_gateway( + search_space_id: int, + user: User = Depends(current_active_user), +) -> dict[str, str]: + if not config.GATEWAY_SLACK_CLIENT_ID: + raise HTTPException(status_code=500, detail="Slack gateway OAuth is not configured") + state = _get_state_manager().generate_secure_state(search_space_id, user.id) + auth_params = { + "client_id": config.GATEWAY_SLACK_CLIENT_ID, + "scope": ",".join(SLACK_BOT_SCOPES), + "redirect_uri": _slack_redirect_uri(), + "state": state, + } + return {"auth_url": f"{SLACK_AUTHORIZATION_URL}?{urlencode(auth_params)}"} + + +@router.get("/slack/callback") +async def slack_gateway_callback( + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +) -> RedirectResponse: + space_id = None + if state: + try: + state_data = _get_state_manager().validate_state(state) + space_id = int(state_data["space_id"]) + except Exception: + state_data = None + else: + state_data = None + + if error: + return _slack_frontend_redirect(space_id or 0, error="slack_gateway_oauth_denied") + if not code or state_data is None: + raise HTTPException(status_code=400, detail="Invalid Slack gateway OAuth callback") + if not config.GATEWAY_SLACK_CLIENT_ID or not config.GATEWAY_SLACK_CLIENT_SECRET: + raise HTTPException(status_code=500, detail="Slack gateway OAuth is not configured") + + user_id = UUID(state_data["user_id"]) + token_payload = { + "client_id": config.GATEWAY_SLACK_CLIENT_ID, + "client_secret": config.GATEWAY_SLACK_CLIENT_SECRET, + "code": code, + "redirect_uri": _slack_redirect_uri(), + } + async with httpx.AsyncClient(timeout=30.0) as client: + token_response = await client.post( + SLACK_TOKEN_URL, + data=token_payload, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + token_response.raise_for_status() + token_json = token_response.json() + if not token_json.get("ok", False): + raise HTTPException( + status_code=400, + detail=f"Slack gateway OAuth failed: {token_json.get('error', 'unknown_error')}", + ) + + bot_token = token_json.get("access_token") + team = token_json.get("team") or {} + team_id = team.get("id") + if not bot_token or not team_id: + raise HTTPException(status_code=400, detail="Slack gateway OAuth returned incomplete data") + + bot_user_id = token_json.get("bot_user_id") + app_id = token_json.get("app_id") + authed_user = token_json.get("authed_user") or {} + authed_slack_user_id = authed_user.get("id") + enc = _get_token_encryption() + credentials = { + "bot_token": bot_token, + "token_type": token_json.get("token_type", "bot"), + "scope": token_json.get("scope"), + } + cursor_state = { + "team_id": team_id, + "team_name": team.get("name"), + "enterprise_id": (token_json.get("enterprise") or {}).get("id"), + "app_id": app_id, + "bot_user_id": bot_user_id, + "scope": token_json.get("scope"), + } + + account = await get_slack_account_by_team(session, team_id=team_id) + if account is None: + account = ExternalChatAccount( + platform=ExternalChatPlatform.SLACK, + mode=ExternalChatAccountMode.CLOUD_SHARED, + is_system_account=True, + encrypted_credentials=enc.encrypt_token(json.dumps(credentials)), + bot_username="SurfSense", + cursor_state=cursor_state, + health_status=ExternalChatHealthStatus.UNKNOWN, + ) + session.add(account) + await session.flush() + else: + account.encrypted_credentials = enc.encrypt_token(json.dumps(credentials)) + account.cursor_state = {**(account.cursor_state or {}), **cursor_state} + account.health_status = ExternalChatHealthStatus.UNKNOWN + + if authed_slack_user_id: + peer_id = slack_user_peer_id(team_id, authed_slack_user_id) + existing_binding_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + binding = existing_binding_result.scalars().first() + if binding is None: + session.add( + ExternalChatBinding( + account_id=account.id, + user_id=user_id, + search_space_id=space_id, + state=ExternalChatBindingState.BOUND, + external_peer_id=peer_id, + external_peer_kind=ExternalChatPeerKind.DIRECT, + external_username=authed_slack_user_id, + external_metadata={ + "kind": "slack_user", + "team_id": team_id, + "slack_user_id": authed_slack_user_id, + }, + ) + ) + elif binding.user_id == user_id: + binding.search_space_id = space_id + binding.external_metadata = { + **(binding.external_metadata or {}), + "kind": "slack_user", + "team_id": team_id, + "slack_user_id": authed_slack_user_id, + } + + await session.commit() + return _slack_frontend_redirect(space_id, success=True) + + +@router.post("/webhooks/slack") +async def slack_webhook( + request: Request, + session: AsyncSession = Depends(get_async_session), +) -> Response: + body = await request.body() + if not verify_slack_signature( + signing_secret=config.GATEWAY_SLACK_SIGNING_SECRET or "", + timestamp=request.headers.get("X-Slack-Request-Timestamp"), + signature=request.headers.get("X-Slack-Signature"), + body=body, + ): + raise HTTPException(status_code=403, detail="Invalid Slack signature") + + try: + payload = json.loads(body.decode()) + except ValueError: + record_gateway_webhook_parse_error() + return Response(status_code=200) + + if payload.get("type") == "url_verification": + return JSONResponse({"challenge": payload.get("challenge", "")}) + if payload.get("type") != "event_callback": + return Response(status_code=200) + + event = payload.get("event") or {} + event_id = payload.get("event_id") + team_id = payload.get("team_id") or event.get("team") + if not event_id or not team_id: + return Response(status_code=200) + + account = await get_slack_account_by_team(session, team_id=str(team_id)) + if account is None: + logger.warning("Ignoring Slack event for uninstalled team_id=%s", team_id) + return Response(status_code=200) + + bot_user_id = (account.cursor_state or {}).get("bot_user_id") + if event.get("bot_id") or (bot_user_id and event.get("user") == bot_user_id): + return Response(status_code=200) + + try: + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=ExternalChatPlatform.SLACK, + event_dedupe_key=slack_event_dedupe_key(event_id), + external_event_id=str(event_id), + external_message_id=str(event.get("ts")) if event.get("ts") else None, + event_kind=_slack_event_kind(payload), + raw_payload=payload, + request_id=f"gateway_{uuid.uuid4().hex[:16]}", + ) + await session.commit() + record_gateway_inbox_write(platform="slack", dedup_skipped=inbox_id is None) + except Exception: + await session.rollback() + logger.exception("Slack webhook persistence failed team_id=%s", team_id) + return Response(status_code=200) + + async def _resolve_webhook_account( session: AsyncSession, *, @@ -207,6 +495,7 @@ async def list_bindings( "search_space_id": binding.search_space_id, "external_display_name": binding.external_display_name, "external_username": binding.external_username, + "external_metadata": binding.external_metadata, "suspended_reason": binding.suspended_reason, } for binding, account in result.all() diff --git a/surfsense_backend/tests/unit/gateway/test_webhook_routes.py b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py index 9a62a3cce..a624cbde1 100644 --- a/surfsense_backend/tests/unit/gateway/test_webhook_routes.py +++ b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py @@ -1,10 +1,14 @@ from __future__ import annotations +import hashlib +import hmac import inspect +import json +import time import pytest -from app.db import ExternalChatPlatform, ExternalChatAccount +from app.db import ExternalChatAccount, ExternalChatAccountMode, ExternalChatPlatform from app.routes import gateway_webhook_routes as routes @@ -19,6 +23,9 @@ class RequestStub: raise self._json_exc return self._payload + async def body(self): + return json.dumps(self._payload).encode() + def _account(secret: str = "secret") -> ExternalChatAccount: return ExternalChatAccount( @@ -29,6 +36,38 @@ def _account(secret: str = "secret") -> ExternalChatAccount: ) +def _slack_account() -> ExternalChatAccount: + return ExternalChatAccount( + id=456, + platform=ExternalChatPlatform.SLACK, + mode=ExternalChatAccountMode.CLOUD_SHARED, + is_system_account=True, + cursor_state={"team_id": "T123", "bot_user_id": "U_BOT"}, + ) + + +def _signed_slack_request(payload: dict, *, secret: str = "signing-secret") -> RequestStub: + body = json.dumps(payload).encode() + timestamp = str(int(time.time())) + digest = hmac.new( + secret.encode(), + b"v0:" + timestamp.encode() + b":" + body, + hashlib.sha256, + ).hexdigest() + + class SlackRequestStub(RequestStub): + async def body(self): + return body + + return SlackRequestStub( + payload, + headers={ + "X-Slack-Request-Timestamp": timestamp, + "X-Slack-Signature": f"v0={digest}", + }, + ) + + async def _call_webhook(*, request: RequestStub, account_id: int, session): return await routes.telegram_webhook( request=request, @@ -147,3 +186,89 @@ def test_telegram_webhook_does_not_use_slowapi_limiter(): assert "@limiter.limit" not in route_source + +def test_verify_slack_signature_accepts_valid_signature(): + payload = b'{"type":"event_callback"}' + timestamp = str(int(time.time())) + digest = hmac.new( + b"secret", + b"v0:" + timestamp.encode() + b":" + payload, + hashlib.sha256, + ).hexdigest() + + assert routes.verify_slack_signature( + signing_secret="secret", + timestamp=timestamp, + signature=f"v0={digest}", + body=payload, + ) + + +@pytest.mark.asyncio +async def test_slack_webhook_url_verification(monkeypatch, mocker): + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_SIGNING_SECRET", "signing-secret") + request = _signed_slack_request({"type": "url_verification", "challenge": "abc123"}) + + response = await routes.slack_webhook(request=request, session=mocker.AsyncMock()) + + assert response.status_code == 200 + assert json.loads(response.body)["challenge"] == "abc123" + + +@pytest.mark.asyncio +async def test_slack_webhook_persists_event(monkeypatch, mocker): + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_SIGNING_SECRET", "signing-secret") + session = mocker.AsyncMock() + monkeypatch.setattr(routes, "get_slack_account_by_team", mocker.AsyncMock(return_value=_slack_account())) + persist = mocker.AsyncMock(return_value=100) + monkeypatch.setattr(routes, "persist_inbound_event", persist) + payload = { + "type": "event_callback", + "team_id": "T123", + "event_id": "Ev123", + "event": { + "type": "app_mention", + "channel": "C123", + "user": "U123", + "text": "<@U_BOT> hello", + "ts": "1717000000.000100", + }, + } + request = _signed_slack_request(payload) + + response = await routes.slack_webhook(request=request, session=session) + + assert response.status_code == 200 + persist.assert_awaited_once() + assert persist.await_args.kwargs["event_dedupe_key"] == "slack_event:Ev123" + assert persist.await_args.kwargs["platform"] == ExternalChatPlatform.SLACK + session.commit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_slack_webhook_ignores_self_event(monkeypatch, mocker): + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_SIGNING_SECRET", "signing-secret") + session = mocker.AsyncMock() + monkeypatch.setattr(routes, "get_slack_account_by_team", mocker.AsyncMock(return_value=_slack_account())) + persist = mocker.AsyncMock(return_value=100) + monkeypatch.setattr(routes, "persist_inbound_event", persist) + request = _signed_slack_request( + { + "type": "event_callback", + "team_id": "T123", + "event_id": "Ev123", + "event": { + "type": "app_mention", + "channel": "C123", + "user": "U_BOT", + "text": "loop", + "ts": "1717000000.000100", + }, + } + ) + + response = await routes.slack_webhook(request=request, session=session) + + assert response.status_code == 200 + persist.assert_not_awaited() + From b5cc19843c3610934aa576ed4ace22b9fe4866c8 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:37:16 +0530 Subject: [PATCH 43/63] feat(gateway): include Slack in gateway maintenance --- surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py index 1c2bb166f..e227db71d 100644 --- a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py @@ -85,6 +85,13 @@ def gateway_health_check_task() -> None: if key in metadata: cursor_state[key] = metadata[key] account.cursor_state = cursor_state + elif account.platform == ExternalChatPlatform.SLACK: + cursor_state = dict(account.cursor_state or {}) + for key in ("team_id", "team", "bot_user_id", "bot_username"): + if key in metadata: + cursor_state[key] = metadata[key] + account.cursor_state = cursor_state + account.bot_username = metadata.get("bot_username") except Exception: logger.warning( "External chat health check failed platform=%s account_id=%s", From ad24c3a36996cf45ce6daf4e3cb095bb19b867ef Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:37:35 +0530 Subject: [PATCH 44/63] feat(web): add Slack messaging channel setup --- surfsense_web/.env.example | 5 ++ .../components/MessagingChannelsContent.tsx | 54 ++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/surfsense_web/.env.example b/surfsense_web/.env.example index 12d81ee3f..032bb48ea 100644 --- a/surfsense_web/.env.example +++ b/surfsense_web/.env.example @@ -6,7 +6,12 @@ FASTAPI_BACKEND_INTERNAL_URL=https://your-internal-backend.example.com NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL or GOOGLE NEXT_PUBLIC_ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848 + +# Messaging gateway options +# WhatsApp UI toggle: disabled, cloud, or baileys NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE=disabled +# Slack gateway UI toggle: true or false +NEXT_PUBLIC_GATEWAY_SLACK_ENABLED=false # Contact Form Vars (optional) DATABASE_URL=postgresql://postgres:[YOUR-PASSWORD]@db.sdsf.supabase.co:5432/postgres diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx index b44f3ecbb..57dfe321e 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx @@ -18,6 +18,7 @@ type Binding = { external_display_name?: string | null; external_username?: string | null; suspended_reason?: string | null; + external_metadata?: Record | null; }; type Platform = { @@ -50,6 +51,7 @@ export function MessagingChannelsContent() { const params = useParams<{ search_space_id: string }>(); const searchSpaceId = Number(params.search_space_id); const whatsappMode = process.env.NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE ?? "disabled"; + const slackGatewayEnabled = process.env.NEXT_PUBLIC_GATEWAY_SLACK_ENABLED === "true"; const [bindings, setBindings] = useState([]); const [platforms, setPlatforms] = useState([]); const [pairing, setPairing] = useState(null); @@ -96,6 +98,17 @@ export function MessagingChannelsContent() { await refresh(); } + async function installSlackGateway() { + const res = await authenticatedFetch( + `${BACKEND_URL}/api/v1/gateway/slack/install?search_space_id=${searchSpaceId}` + ); + if (!res.ok) return; + const data = (await res.json()) as { auth_url?: string }; + if (data.auth_url) { + window.location.href = data.auth_url; + } + } + function refreshBaileys() { startTransition(async () => { await refreshBaileysHealth(); @@ -119,8 +132,13 @@ export function MessagingChannelsContent() { const telegram = platforms.find((p) => p.platform === "telegram"); const whatsapp = platforms.find((p) => p.platform === "whatsapp"); + const slack = platforms.find((p) => p.platform === "slack"); const baileysQr = baileysHealth?.qr || null; - const activeBindings = bindings.filter((binding) => binding.search_space_id === searchSpaceId); + const activeBindings = bindings.filter( + (binding) => + binding.search_space_id === searchSpaceId && + binding.external_metadata?.kind !== "slack_thread" + ); const renderPairingPanel = (platform: PairingPlatform) => { if (!pairing || pairingPlatform !== platform) return null; @@ -170,6 +188,40 @@ export function MessagingChannelsContent() { + {slackGatewayEnabled ? ( + + +
+ + + Slack Bot + + + {slack ? "enabled" : "not enabled"} + +
+

+ Enable the SurfSense Slack bot so teammates can mention it in Slack. This is separate + from the Slack search connector. +

+ + +
+ + +
+

+ Slack search remains controlled by the Slack connector in the connector popup. +

+ + + ) : null} + {whatsappMode !== "disabled" ? ( From 78fbca19676fc63eca7880fb4ed2eb39572edded Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:38:01 +0530 Subject: [PATCH 45/63] refactor(assistant-ui): remove unused ShieldCheck icon from MCPTrustedTools component --- .../connector-configs/components/mcp-trusted-tools.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx index ed01511ca..08d20f2ab 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx @@ -1,6 +1,6 @@ "use client"; -import { ShieldCheck, Trash2 } from "lucide-react"; +import { Trash2 } from "lucide-react"; import type { FC } from "react"; import { useState } from "react"; import { toast } from "sonner"; @@ -36,7 +36,6 @@ export const MCPTrustedTools: FC = ({ connector }) => { return (

- Trusted Tools

From 4b8ca29f9eae6574ccbfe9ebe801edc9daf476bb Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:58:17 +0530 Subject: [PATCH 46/63] feat(gateway): add Discord external chat platform --- .../146_add_discord_gateway_platform.py | 106 ++++++++++++++++++ surfsense_backend/app/db.py | 14 ++- 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 surfsense_backend/alembic/versions/146_add_discord_gateway_platform.py diff --git a/surfsense_backend/alembic/versions/146_add_discord_gateway_platform.py b/surfsense_backend/alembic/versions/146_add_discord_gateway_platform.py new file mode 100644 index 000000000..32b02d059 --- /dev/null +++ b/surfsense_backend/alembic/versions/146_add_discord_gateway_platform.py @@ -0,0 +1,106 @@ +"""add discord gateway platform + +Revision ID: 146 +Revises: 145 +Create Date: 2026-06-01 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "146" +down_revision: str | None = "145" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _enum_value_exists(enum_name: str, value: str) -> bool: + conn = op.get_bind() + return ( + conn.execute( + sa.text( + "SELECT 1 FROM pg_enum e " + "JOIN pg_type t ON t.oid = e.enumtypid " + "WHERE t.typname = :enum_name AND e.enumlabel = :value" + ), + {"enum_name": enum_name, "value": value}, + ).fetchone() + is not None + ) + + +def _index_exists(index_name: str) -> bool: + conn = op.get_bind() + return ( + conn.execute( + sa.text( + "SELECT 1 FROM pg_indexes " + "WHERE schemaname = current_schema() AND indexname = :index_name" + ), + {"index_name": index_name}, + ).fetchone() + is not None + ) + + +def upgrade() -> None: + if not _enum_value_exists("external_chat_platform", "discord"): + op.execute("ALTER TYPE external_chat_platform ADD VALUE 'discord'") + + if _index_exists("uq_external_chat_accounts_system_platform"): + op.drop_index( + "uq_external_chat_accounts_system_platform", + table_name="external_chat_accounts", + ) + + op.create_index( + "uq_external_chat_accounts_system_platform", + "external_chat_accounts", + ["platform"], + unique=True, + postgresql_where=sa.text( + "is_system_account = true " + "AND NOT (cursor_state ? 'team_id') " + "AND NOT (cursor_state ? 'guild_id')" + ), + if_not_exists=True, + ) + op.create_index( + "uq_external_chat_accounts_discord_guild", + "external_chat_accounts", + ["platform", sa.text("(cursor_state ->> 'guild_id')")], + unique=True, + postgresql_where=sa.text( + "is_system_account = true AND cursor_state ? 'guild_id'" + ), + if_not_exists=True, + ) + + +def downgrade() -> None: + if _index_exists("uq_external_chat_accounts_discord_guild"): + op.drop_index( + "uq_external_chat_accounts_discord_guild", + table_name="external_chat_accounts", + ) + if _index_exists("uq_external_chat_accounts_system_platform"): + op.drop_index( + "uq_external_chat_accounts_system_platform", + table_name="external_chat_accounts", + ) + op.create_index( + "uq_external_chat_accounts_system_platform", + "external_chat_accounts", + ["platform"], + unique=True, + postgresql_where=sa.text( + "is_system_account = true AND NOT (cursor_state ? 'team_id')" + ), + if_not_exists=True, + ) + # PostgreSQL enum values are intentionally not removed on downgrade. diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 14ba8cdec..b10dc51a8 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -578,6 +578,7 @@ class ExternalChatPlatform(StrEnum): TELEGRAM = "telegram" WHATSAPP = "whatsapp" SLACK = "slack" + DISCORD = "discord" SIGNAL = "signal" @@ -890,7 +891,9 @@ class ExternalChatAccount(Base, TimestampMixin): "platform", unique=True, postgresql_where=text( - "is_system_account = true AND NOT (cursor_state ? 'team_id')" + "is_system_account = true " + "AND NOT (cursor_state ? 'team_id') " + "AND NOT (cursor_state ? 'guild_id')" ), ), Index( @@ -902,6 +905,15 @@ class ExternalChatAccount(Base, TimestampMixin): "is_system_account = true AND cursor_state ? 'team_id'" ), ), + Index( + "uq_external_chat_accounts_discord_guild", + "platform", + text("(cursor_state ->> 'guild_id')"), + unique=True, + postgresql_where=text( + "is_system_account = true AND cursor_state ? 'guild_id'" + ), + ), Index( "uq_external_chat_accounts_webhook_secret", "webhook_secret", From 68da295b5da7af3eecd0f90ebcc09f913f9f4e25 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:58:31 +0530 Subject: [PATCH 47/63] feat(gateway): add Discord gateway configuration --- surfsense_backend/.env.example | 5 +++- surfsense_backend/app/config/__init__.py | 4 ++++ surfsense_backend/app/gateway/accounts.py | 28 +++++++++++++++++++++++ surfsense_backend/app/gateway/inbox.py | 4 ++++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 2d6c27e26..170bece32 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -119,11 +119,14 @@ CLICKUP_CLIENT_ID=your_clickup_client_id_here CLICKUP_CLIENT_SECRET=your_clickup_client_secret_here CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback -# Discord OAuth Configuration +# Discord OAuth / Gateway Configuration +# The Discord connector and Discord gateway use the same Discord application/bot. DISCORD_CLIENT_ID=your_discord_client_id_here DISCORD_CLIENT_SECRET=your_discord_client_secret_here DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback DISCORD_BOT_TOKEN=your_bot_token_from_developer_portal +GATEWAY_DISCORD_ENABLED=FALSE +GATEWAY_DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/gateway/discord/callback # Atlassian OAuth Configuration (Jira & Confluence) ATLASSIAN_CLIENT_ID=your_atlassian_client_id_here diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index fdf8834c1..98c1d5dec 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -574,6 +574,10 @@ class Config: GATEWAY_SLACK_CLIENT_SECRET = os.getenv("SLACK_CLIENT_SECRET") GATEWAY_SLACK_SIGNING_SECRET = os.getenv("GATEWAY_SLACK_SIGNING_SECRET") GATEWAY_SLACK_REDIRECT_URI = os.getenv("GATEWAY_SLACK_REDIRECT_URI") + GATEWAY_DISCORD_ENABLED = ( + os.getenv("GATEWAY_DISCORD_ENABLED", "FALSE").upper() == "TRUE" + ) + GATEWAY_DISCORD_REDIRECT_URI = os.getenv("GATEWAY_DISCORD_REDIRECT_URI") # Stripe checkout for pay-as-you-go page packs STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY") diff --git a/surfsense_backend/app/gateway/accounts.py b/surfsense_backend/app/gateway/accounts.py index 5daf75c69..2d924e200 100644 --- a/surfsense_backend/app/gateway/accounts.py +++ b/surfsense_backend/app/gateway/accounts.py @@ -40,6 +40,19 @@ def slack_account_credentials(account: ExternalChatAccount) -> dict: return data if isinstance(data, dict) else {} +def discord_account_credentials(account: ExternalChatAccount) -> dict: + """Decrypt Discord gateway credentials stored as encrypted JSON.""" + if not account.encrypted_credentials: + return {} + raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token(account.encrypted_credentials) + try: + data = json.loads(raw) + except json.JSONDecodeError: + # Backward-compatible fallback if a token string was stored directly. + return {"bot_token": raw} + return data if isinstance(data, dict) else {} + + async def get_or_create_system_telegram_account( session: AsyncSession, ) -> ExternalChatAccount: @@ -108,3 +121,18 @@ async def get_slack_account_by_team( ) return result.scalars().first() + +async def get_discord_account_by_guild( + session: AsyncSession, + *, + guild_id: str, +) -> ExternalChatAccount | None: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.DISCORD, + ExternalChatAccount.is_system_account.is_(True), + ExternalChatAccount.cursor_state["guild_id"].astext == guild_id, + ) + ) + return result.scalars().first() + diff --git a/surfsense_backend/app/gateway/inbox.py b/surfsense_backend/app/gateway/inbox.py index 5769c8cc4..cd0e2f9b7 100644 --- a/surfsense_backend/app/gateway/inbox.py +++ b/surfsense_backend/app/gateway/inbox.py @@ -16,6 +16,10 @@ def slack_event_dedupe_key(event_id: int | str) -> str: return f"slack_event:{event_id}" +def discord_message_dedupe_key(message_id: int | str) -> str: + return f"discord_message:{message_id}" + + async def persist_inbound_event( session: AsyncSession, *, From bc8a285187db97cf297a4b7fd3bb734eda28b532 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:58:50 +0530 Subject: [PATCH 48/63] feat(gateway): add Discord platform adapter --- .../app/gateway/discord/__init__.py | 1 + .../app/gateway/discord/adapter.py | 135 ++++++++++++++++++ .../app/gateway/discord/client.py | 109 ++++++++++++++ .../unit/gateway/test_discord_adapter.py | 92 ++++++++++++ 4 files changed, 337 insertions(+) create mode 100644 surfsense_backend/app/gateway/discord/__init__.py create mode 100644 surfsense_backend/app/gateway/discord/adapter.py create mode 100644 surfsense_backend/app/gateway/discord/client.py create mode 100644 surfsense_backend/tests/unit/gateway/test_discord_adapter.py diff --git a/surfsense_backend/app/gateway/discord/__init__.py b/surfsense_backend/app/gateway/discord/__init__.py new file mode 100644 index 000000000..1dd0edc96 --- /dev/null +++ b/surfsense_backend/app/gateway/discord/__init__.py @@ -0,0 +1 @@ +"""Discord gateway platform integration.""" diff --git a/surfsense_backend/app/gateway/discord/adapter.py b/surfsense_backend/app/gateway/discord/adapter.py new file mode 100644 index 000000000..60db895fe --- /dev/null +++ b/surfsense_backend/app/gateway/discord/adapter.py @@ -0,0 +1,135 @@ +"""Discord platform adapter for bot mentions and replies.""" + +from __future__ import annotations + +import re +from typing import Any + +from app.gateway.base.adapter import ( + BasePlatformAdapter, + ParsedInboundEvent, + PlatformSendResult, +) +from app.gateway.discord.client import DiscordGatewayClient + +MENTION_RE = re.compile(r"<@!?\d+>\s*") + + +def discord_user_peer_id(guild_id: str, discord_user_id: str) -> str: + return f"discord_user:{guild_id}:{discord_user_id}" + + +def discord_thread_peer_id(guild_id: str, channel_id: str, thread_key: str) -> str: + return f"discord_thread:{guild_id}:{channel_id}:{thread_key}" + + +class DiscordAdapter(BasePlatformAdapter): + platform = "discord" + + def __init__(self, bot_token: str, *, bot_user_id: str | None = None) -> None: + self.bot_user_id = bot_user_id + self.client = DiscordGatewayClient(bot_token) + + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + event = raw_payload.get("event") or raw_payload + event_kind = str(raw_payload.get("type") or event.get("type") or "message") + guild_id = str(event.get("guild_id") or "") + channel_id = str(event.get("channel_id") or "") + author = event.get("author") or {} + discord_user_id = str(author.get("id") or event.get("author_id") or "") + message_id = str(event.get("id") or event.get("message_id") or "") + bot_user_id = self.bot_user_id or str(raw_payload.get("bot_user_id") or "") + + if not guild_id or not channel_id or not discord_user_id or not message_id: + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_kind, + external_peer_id=None, + external_peer_kind="unknown", + external_message_id=message_id or None, + external_user_id=discord_user_id or None, + text=None, + raw_payload=raw_payload, + metadata={ + "guild_id": guild_id, + "channel_id": channel_id, + "bot_user_id": bot_user_id, + }, + ) + + text = str(event.get("content") or "") + if bot_user_id: + text = text.replace(f"<@{bot_user_id}>", "") + text = text.replace(f"<@!{bot_user_id}>", "") + text = MENTION_RE.sub("", text).strip() + + thread_key = str( + event.get("thread_id") + or (event.get("message_reference") or {}).get("message_id") + or message_id + ) + thread_peer_id = discord_thread_peer_id(guild_id, channel_id, thread_key) + user_peer_id = discord_user_peer_id(guild_id, discord_user_id) + mentions = event.get("mentions") or [] + mentions_bot = bool( + bot_user_id + and any(str(mention.get("id")) == bot_user_id for mention in mentions) + ) + + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_kind, + external_peer_id=thread_peer_id, + external_peer_kind="channel", + external_message_id=message_id, + external_user_id=discord_user_id, + text=text, + raw_payload=raw_payload, + display_name=event.get("channel_name"), + username=author.get("username") or discord_user_id, + metadata={ + "guild_id": guild_id, + "channel_id": channel_id, + "discord_user_id": discord_user_id, + "message_id": message_id, + "thread_key": thread_key, + "bot_user_id": bot_user_id, + "discord_user_peer_id": user_peer_id, + "discord_thread_peer_id": thread_peer_id, + "mentions_bot": mentions_bot, + "is_dm": False, + }, + ) + + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + del parse_mode + return await self.client.send_message( + channel_id=external_peer_id, + content=text, + reply_to_message_id=reply_to_message_id, + ) + + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + del parse_mode + return await self.client.update_message( + channel_id=external_peer_id, + message_id=external_message_id, + content=text, + ) + + async def validate_credentials(self) -> dict[str, Any]: + return await self.client.validate() diff --git a/surfsense_backend/app/gateway/discord/client.py b/surfsense_backend/app/gateway/discord/client.py new file mode 100644 index 000000000..206abaa5f --- /dev/null +++ b/surfsense_backend/app/gateway/discord/client.py @@ -0,0 +1,109 @@ +"""Discord REST API client for gateway bot operations.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import httpx + +from app.gateway.base.adapter import PlatformSendResult + +DISCORD_API = "https://discord.com/api/v10" + + +class DiscordGatewayClient: + def __init__(self, bot_token: str) -> None: + self.bot_token = bot_token + + async def api_call( + self, + method: str, + path: str, + *, + payload: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + retry_rate_limit: bool = True, + ) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=20.0) as client: + response = await client.request( + method, + f"{DISCORD_API}{path}", + json=payload, + params=params, + headers={ + "Authorization": f"Bot {self.bot_token}", + "Content-Type": "application/json", + }, + ) + if response.status_code == 429 and retry_rate_limit: + data = response.json() + retry_after = float(data.get("retry_after") or 1.0) + await asyncio.sleep(min(retry_after, 5.0)) + return await self.api_call( + method, + path, + payload=payload, + params=params, + retry_rate_limit=False, + ) + response.raise_for_status() + if not response.content: + return {} + return response.json() + + async def send_message( + self, + *, + channel_id: str, + content: str, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + payload: dict[str, Any] = { + "content": content, + "allowed_mentions": {"parse": []}, + } + if reply_to_message_id: + payload["message_reference"] = { + "message_id": reply_to_message_id, + "channel_id": channel_id, + "fail_if_not_exists": False, + } + data = await self.api_call( + "POST", + f"/channels/{channel_id}/messages", + payload=payload, + ) + return PlatformSendResult( + external_message_id=str(data.get("id", "")), + raw_response=data, + ) + + async def update_message( + self, + *, + channel_id: str, + message_id: str, + content: str, + ) -> PlatformSendResult: + data = await self.api_call( + "PATCH", + f"/channels/{channel_id}/messages/{message_id}", + payload={"content": content, "allowed_mentions": {"parse": []}}, + ) + return PlatformSendResult( + external_message_id=str(data.get("id") or message_id), + raw_response=data, + ) + + async def validate(self) -> dict[str, Any]: + data = await self.api_call("GET", "/users/@me") + return { + "ok": True, + "bot_user_id": data.get("id"), + "bot_username": data.get("username"), + "global_name": data.get("global_name"), + } + + async def get_guild(self, guild_id: str) -> dict[str, Any]: + return await self.api_call("GET", f"/guilds/{guild_id}") diff --git a/surfsense_backend/tests/unit/gateway/test_discord_adapter.py b/surfsense_backend/tests/unit/gateway/test_discord_adapter.py new file mode 100644 index 000000000..c6790f20b --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_discord_adapter.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import pytest + +from app.gateway.base.adapter import PlatformSendResult +from app.gateway.discord.adapter import DiscordAdapter + + +def _discord_payload(content: str = "<@999> summarize this channel"): + return { + "type": "message", + "bot_user_id": "999", + "event": { + "type": "message", + "id": "111", + "guild_id": "222", + "guild_name": "SurfSense Guild", + "channel_id": "333", + "channel_name": "general", + "content": content, + "author": {"id": "444", "username": "anish", "bot": False}, + "mentions": [{"id": "999", "username": "SurfSense"}], + }, + } + + +def test_discord_adapter_parses_mention_and_strips_bot_mention(): + adapter = DiscordAdapter("discord-token", bot_user_id="999") + + parsed = adapter.parse_inbound(_discord_payload()) + + assert parsed.platform == "discord" + assert parsed.text == "summarize this channel" + assert parsed.external_peer_id == "discord_thread:222:333:111" + assert parsed.metadata["discord_user_peer_id"] == "discord_user:222:444" + assert parsed.metadata["discord_thread_peer_id"] == "discord_thread:222:333:111" + assert parsed.metadata["mentions_bot"] is True + + +def test_discord_adapter_strips_nickname_mention(): + adapter = DiscordAdapter("discord-token", bot_user_id="999") + + parsed = adapter.parse_inbound(_discord_payload("<@!999> continue")) + + assert parsed.text == "continue" + + +def test_discord_adapter_uses_message_reference_as_thread_key(): + adapter = DiscordAdapter("discord-token", bot_user_id="999") + payload = _discord_payload("<@999> continue") + payload["event"]["id"] = "112" + payload["event"]["message_reference"] = { + "message_id": "111", + "channel_id": "333", + "guild_id": "222", + } + + parsed = adapter.parse_inbound(payload) + + assert parsed.external_peer_id == "discord_thread:222:333:111" + assert parsed.metadata["message_id"] == "112" + assert parsed.metadata["thread_key"] == "111" + + +def test_discord_adapter_returns_missing_peer_for_incomplete_payload(): + adapter = DiscordAdapter("discord-token", bot_user_id="999") + + parsed = adapter.parse_inbound({"event": {"id": "111"}}) + + assert parsed.external_peer_id is None + assert parsed.external_peer_kind == "unknown" + + +@pytest.mark.asyncio +async def test_discord_adapter_sends_message(mocker): + adapter = DiscordAdapter("discord-token", bot_user_id="999") + adapter.client.send_message = mocker.AsyncMock( + return_value=PlatformSendResult(external_message_id="555") + ) + + result = await adapter.send_message( + external_peer_id="333", + text="hello", + reply_to_message_id="111", + ) + + assert result.external_message_id == "555" + adapter.client.send_message.assert_awaited_once_with( + channel_id="333", + content="hello", + reply_to_message_id="111", + ) From 5024b69e6990a018d935b8cf9a140fa60260208a Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:59:04 +0530 Subject: [PATCH 49/63] feat(gateway): handle Discord channel replies --- .../app/gateway/discord/commands.py | 66 ++++++++++++++ .../app/gateway/discord/translator.py | 86 +++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 surfsense_backend/app/gateway/discord/commands.py create mode 100644 surfsense_backend/app/gateway/discord/translator.py diff --git a/surfsense_backend/app/gateway/discord/commands.py b/surfsense_backend/app/gateway/discord/commands.py new file mode 100644 index 000000000..2152e75c5 --- /dev/null +++ b/surfsense_backend/app/gateway/discord/commands.py @@ -0,0 +1,66 @@ +"""Discord command/onboarding handlers.""" + +from __future__ import annotations + +from app.gateway.base.adapter import ParsedInboundEvent +from app.gateway.base.commands import BaseGatewayCommands +from app.gateway.discord.adapter import DiscordAdapter +from app.gateway.ratelimit import acquire_token + +HELP_TEXT = ( + "SurfSense Discord commands:\n" + "`/new` - start a fresh SurfSense conversation for this Discord thread\n" + "`/help` - show this help\n\n" + "Mention the SurfSense bot in a Discord channel to ask your agent a question. " + "Discord search remains controlled by the Discord connector in SurfSense." +) + + +class DiscordGatewayCommands(BaseGatewayCommands): + async def handle_help_command( + self, + *, + adapter: DiscordAdapter, + event: ParsedInboundEvent, + ) -> bool: + channel_id = event.metadata.get("channel_id") + message_id = event.metadata.get("message_id") + if not channel_id: + return True + await adapter.send_message( + external_peer_id=channel_id, + text=HELP_TEXT, + reply_to_message_id=message_id, + ) + return True + + async def send_unbound_onboarding( + self, + *, + adapter: DiscordAdapter, + event: ParsedInboundEvent, + dashboard_url: str, + ) -> None: + channel_id = event.metadata.get("channel_id") + message_id = event.metadata.get("message_id") + guild_id = event.metadata.get("guild_id") + discord_user_id = event.metadata.get("discord_user_id") + if not channel_id or not message_id: + return + + wait_ms = await acquire_token( + f"discord:onboarded:{guild_id}:{discord_user_id}", + capacity=1, + refill_per_sec=1 / 3600, + ) + if wait_ms > 0: + return + + await adapter.send_message( + external_peer_id=channel_id, + reply_to_message_id=message_id, + text=( + "Hi! Connect your Discord user to SurfSense before using the bot here: " + f"{dashboard_url}" + ), + ) diff --git a/surfsense_backend/app/gateway/discord/translator.py b/surfsense_backend/app/gateway/discord/translator.py new file mode 100644 index 000000000..2bd843e3d --- /dev/null +++ b/surfsense_backend/app/gateway/discord/translator.py @@ -0,0 +1,86 @@ +"""Translate agent stream events into Discord replies.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator + +from app.gateway.base.adapter import PlatformSendResult +from app.gateway.base.formatting import split_text_message +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.discord.adapter import DiscordAdapter +from app.gateway.ratelimit import wait_for_token +from app.observability.metrics import ( + record_gateway_hitl_aborted, + record_gateway_outbound, + record_gateway_rate_limit_hit, +) + +logger = logging.getLogger(__name__) + +DISCORD_MAX_MESSAGE_CHARS = 1900 +HITL_UNSUPPORTED_MESSAGE = ( + "This action requires approval and is not yet supported from Discord. " + "Try again with a different request." +) + + +class DiscordStreamTranslator(BaseStreamTranslator): + def __init__( + self, + *, + adapter: DiscordAdapter, + channel_id: str, + reply_to_message_id: str | None, + ) -> None: + self.adapter = adapter + self.channel_id = channel_id + self.reply_to_message_id = reply_to_message_id + self._buffer = "" + + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + async for event in events: + if event.type in {"text-delta", "text_delta", "text"}: + self._buffer += str(event.data.get("text") or event.data.get("delta") or "") + elif event.type in {"data-interrupt-request", "interrupt"}: + await self._handle_hitl_interrupt() + return + elif event.type in {"finish", "done"}: + break + + await self._flush_final() + + async def _flush_final(self) -> None: + if not self._buffer: + return + for chunk in split_text_message(self._buffer, max_chars=DISCORD_MAX_MESSAGE_CHARS): + await self._send_text(chunk) + + async def _send_text(self, text: str) -> PlatformSendResult: + await self._throttle() + try: + result = await self.adapter.send_message( + external_peer_id=self.channel_id, + text=text, + reply_to_message_id=self.reply_to_message_id, + ) + except Exception: + record_gateway_outbound(platform="discord", kind="send", status="failed") + raise + record_gateway_outbound(platform="discord", kind="send", status="sent") + return result + + async def _throttle(self) -> None: + chat_wait = await wait_for_token( + f"discord:channel:{self.channel_id}", + capacity=5, + refill_per_sec=1.0, + ) + if chat_wait: + record_gateway_rate_limit_hit(bucket="discord:channel") + + async def _handle_hitl_interrupt(self) -> None: + if self._buffer: + await self._flush_final() + await self._send_text(HITL_UNSUPPORTED_MESSAGE) + record_gateway_hitl_aborted(platform="discord") From f8ff58bdcef5a01c2a5ad6a8aa7f0366f246672b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:59:13 +0530 Subject: [PATCH 50/63] feat(gateway): route Discord events through external chat --- .../app/gateway/inbox_processor.py | 84 +++++++++++++++++-- surfsense_backend/app/gateway/registry.py | 43 +++++++++- 2 files changed, 121 insertions(+), 6 deletions(-) diff --git a/surfsense_backend/app/gateway/inbox_processor.py b/surfsense_backend/app/gateway/inbox_processor.py index 3e87b582d..d47206443 100644 --- a/surfsense_backend/app/gateway/inbox_processor.py +++ b/surfsense_backend/app/gateway/inbox_processor.py @@ -136,6 +136,8 @@ async def _resolve_binding_for_event( ) -> ExternalChatBinding | None: if account.platform == ExternalChatPlatform.SLACK: return await _resolve_slack_thread_binding(session, account, parsed) + if account.platform == ExternalChatPlatform.DISCORD: + return await _resolve_discord_thread_binding(session, account, parsed) result = await session.execute( select(ExternalChatBinding).where( @@ -209,6 +211,74 @@ async def _resolve_slack_thread_binding( return thread_binding +async def _resolve_discord_thread_binding( + session: AsyncSession, + account: ExternalChatAccount, + parsed, +) -> ExternalChatBinding | None: + user_peer_id = parsed.metadata.get("discord_user_peer_id") + thread_peer_id = parsed.metadata.get("discord_thread_peer_id") or parsed.external_peer_id + if not user_peer_id or not thread_peer_id: + return None + + user_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == user_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + user_binding = user_result.scalars().first() + if user_binding is None: + return None + + thread_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == thread_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + thread_binding = thread_result.scalars().first() + if thread_binding is not None: + return thread_binding + + thread_binding = ExternalChatBinding( + account_id=account.id, + user_id=user_binding.user_id, + search_space_id=user_binding.search_space_id, + state=ExternalChatBindingState.BOUND, + external_peer_id=thread_peer_id, + external_peer_kind=ExternalChatPeerKind.CHANNEL, + external_thread_id=parsed.metadata.get("thread_key"), + external_display_name=parsed.metadata.get("channel_id"), + external_username=parsed.external_user_id, + external_metadata={ + "kind": "discord_thread", + "guild_id": parsed.metadata.get("guild_id"), + "channel_id": parsed.metadata.get("channel_id"), + "thread_key": parsed.metadata.get("thread_key"), + "discord_user_id": parsed.metadata.get("discord_user_id"), + "user_binding_id": user_binding.id, + }, + ) + session.add(thread_binding) + await session.flush() + return thread_binding + + +def _reply_target(parsed) -> tuple[str | None, str | None]: + if parsed.platform == "slack": + return parsed.metadata.get("channel_id"), parsed.metadata.get("thread_ts") + if parsed.platform == "discord": + return parsed.metadata.get("channel_id"), parsed.metadata.get("message_id") + return parsed.external_peer_id, None + + async def _dispatch_inbound_event( inbox_id: int, session_maker: SessionMaker, @@ -245,7 +315,8 @@ async def _dispatch_inbound_event( binding = await _resolve_binding_for_event(session, account, parsed) if ( - account.platform != ExternalChatPlatform.SLACK + account.platform + not in {ExternalChatPlatform.SLACK, ExternalChatPlatform.DISCORD} and parsed.external_peer_kind != ExternalChatPeerKind.DIRECT.value ): if hasattr(adapter, "leave_chat"): @@ -300,10 +371,13 @@ async def _dispatch_inbound_event( return if cmd == "/new": binding.new_chat_thread_id = None - await adapter.send_message( - external_peer_id=parsed.external_peer_id, - text="Started a new SurfSense conversation.", - ) + reply_peer_id, reply_message_id = _reply_target(parsed) + if reply_peer_id: + await adapter.send_message( + external_peer_id=reply_peer_id, + text="Started a new SurfSense conversation.", + reply_to_message_id=reply_message_id, + ) event.status = ExternalChatEventStatus.PROCESSED await session.commit() return diff --git a/surfsense_backend/app/gateway/registry.py b/surfsense_backend/app/gateway/registry.py index fc9cb37e5..3aa9e607a 100644 --- a/surfsense_backend/app/gateway/registry.py +++ b/surfsense_backend/app/gateway/registry.py @@ -6,7 +6,11 @@ from collections.abc import Callable from dataclasses import dataclass from app.db import ExternalChatAccount, ExternalChatAccountMode, ExternalChatPlatform -from app.gateway.accounts import account_token, slack_account_credentials +from app.gateway.accounts import ( + account_token, + discord_account_credentials, + slack_account_credentials, +) from app.gateway.base.adapter import BasePlatformAdapter, ParsedInboundEvent from app.gateway.base.commands import BaseGatewayCommands from app.gateway.base.translator import BaseStreamTranslator @@ -87,6 +91,23 @@ def _slack_translator_factory( ) +def _discord_translator_factory( + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> BaseStreamTranslator: + channel_id = event.metadata.get("channel_id") + message_id = event.metadata.get("message_id") + if not channel_id: + raise RuntimeError("missing_discord_channel_metadata") + from app.gateway.discord.translator import DiscordStreamTranslator + + return DiscordStreamTranslator( + adapter=adapter, # type: ignore[arg-type] + channel_id=channel_id, + reply_to_message_id=message_id, + ) + + def resolve_platform_bundle(account: ExternalChatAccount) -> PlatformBundle: if account.platform == ExternalChatPlatform.TELEGRAM: token = account_token(account) @@ -145,4 +166,24 @@ def resolve_platform_bundle(account: ExternalChatAccount) -> PlatformBundle: auto_bind_owner=False, ) + if account.platform == ExternalChatPlatform.DISCORD: + from app.gateway.discord.adapter import DiscordAdapter + from app.gateway.discord.commands import DiscordGatewayCommands + + credentials = discord_account_credentials(account) + bot_token = credentials.get("bot_token") + if not bot_token: + raise RuntimeError("missing_discord_bot_token") + cursor_state = account.cursor_state or {} + return PlatformBundle( + adapter=DiscordAdapter( + bot_token, + bot_user_id=cursor_state.get("bot_user_id"), + ), + translator_factory=_discord_translator_factory, + platform_label="discord", + commands=DiscordGatewayCommands(), + auto_bind_owner=False, + ) + raise RuntimeError(f"unsupported_gateway_platform:{account.platform.value}:{account.mode.value}") From 05eaa46c3a874043ae6c0e13a7f4c148b9327bcf Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:59:21 +0530 Subject: [PATCH 51/63] feat(gateway): add Discord mention intake supervisor --- surfsense_backend/app/app.py | 6 + .../app/gateway/discord/intake.py | 201 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 surfsense_backend/app/gateway/discord/intake.py diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index 17f4e093e..a91f86ef5 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -41,6 +41,10 @@ from app.gateway.byo_long_poll import ( start_byo_long_poll_supervisors, stop_byo_long_poll_supervisors, ) +from app.gateway.discord.intake import ( + start_discord_gateway_supervisor, + stop_discord_gateway_supervisor, +) from app.gateway.inbox_worker import ( start_gateway_inbox_worker, stop_gateway_inbox_worker, @@ -607,10 +611,12 @@ async def lifespan(app: FastAPI): log_system_snapshot("startup_complete") await start_gateway_inbox_worker() await start_byo_long_poll_supervisors() + await start_discord_gateway_supervisor() try: yield finally: + await stop_discord_gateway_supervisor() await stop_byo_long_poll_supervisors() await stop_gateway_inbox_worker() _stop_openrouter_background_refresh() diff --git a/surfsense_backend/app/gateway/discord/intake.py b/surfsense_backend/app/gateway/discord/intake.py new file mode 100644 index 000000000..4c89de821 --- /dev/null +++ b/surfsense_backend/app/gateway/discord/intake.py @@ -0,0 +1,201 @@ +"""FastAPI lifespan supervisor for Discord Gateway WebSocket intake.""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from contextlib import suppress +from typing import Any + +import discord + +from app.config import config +from app.db import ExternalChatPlatform, async_session_maker +from app.gateway.accounts import get_discord_account_by_guild +from app.gateway.inbox import discord_message_dedupe_key, persist_inbound_event +from app.observability.metrics import record_gateway_inbox_write + +logger = logging.getLogger(__name__) + +_task: asyncio.Task[None] | None = None +_client: discord.Client | None = None +_shutdown_event: asyncio.Event | None = None + + +def _message_reference_payload(message: discord.Message) -> dict[str, Any] | None: + if message.reference is None: + return None + return { + "message_id": str(message.reference.message_id) + if message.reference.message_id + else None, + "channel_id": str(message.reference.channel_id) + if message.reference.channel_id + else None, + "guild_id": str(message.reference.guild_id) + if message.reference.guild_id + else None, + } + + +def _serialize_message(message: discord.Message, *, bot_user_id: str | None) -> dict[str, Any]: + guild = message.guild + channel = message.channel + thread_id = str(channel.id) if isinstance(channel, discord.Thread) else None + parent_id = str(channel.parent_id) if isinstance(channel, discord.Thread) else None + return { + "type": "message", + "bot_user_id": bot_user_id, + "event": { + "type": "message", + "id": str(message.id), + "guild_id": str(guild.id) if guild else None, + "guild_name": guild.name if guild else None, + "channel_id": parent_id or str(message.channel.id), + "thread_id": thread_id, + "channel_name": getattr(channel, "name", None), + "content": message.content, + "author": { + "id": str(message.author.id), + "username": message.author.name, + "bot": message.author.bot, + }, + "mentions": [ + {"id": str(user.id), "username": user.name} + for user in message.mentions + ], + "message_reference": _message_reference_payload(message), + "created_at": message.created_at.isoformat() + if message.created_at + else None, + }, + } + + +async def _persist_message(message: discord.Message, *, bot_user_id: str | None) -> None: + if message.guild is None: + return + guild_id = str(message.guild.id) + raw_payload = _serialize_message(message, bot_user_id=bot_user_id) + + async with async_session_maker() as session: + account = await get_discord_account_by_guild(session, guild_id=guild_id) + if account is None: + logger.info("Ignoring Discord message for uninstalled guild_id=%s", guild_id) + return + + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=ExternalChatPlatform.DISCORD, + event_dedupe_key=discord_message_dedupe_key(message.id), + external_event_id=str(message.id), + external_message_id=str(message.id), + event_kind="message", + raw_payload=raw_payload, + request_id=f"gateway_{uuid.uuid4().hex[:16]}", + ) + await session.commit() + record_gateway_inbox_write(platform="discord", dedup_skipped=inbox_id is None) + logger.info( + "Persisted Discord gateway message_id=%s guild_id=%s inbox_id=%s", + message.id, + guild_id, + inbox_id, + ) + + +def _build_client() -> discord.Client: + intents = discord.Intents.default() + intents.guilds = True + intents.messages = True + intents.message_content = True + client = discord.Client(intents=intents) + + @client.event + async def on_ready() -> None: + logger.info( + "Discord gateway connected as %s (%s)", + client.user, + getattr(client.user, "id", None), + ) + + @client.event + async def on_message(message: discord.Message) -> None: + if message.author.bot: + return + bot_user = client.user + if bot_user is None: + return + if message.author.id == bot_user.id: + return + bot_user_id = str(bot_user.id) + mention_ids = {str(user.id) for user in message.mentions} + if bot_user_id not in mention_ids: + return + logger.info( + "Received Discord gateway mention message_id=%s guild_id=%s channel_id=%s content_present=%s", + message.id, + getattr(message.guild, "id", None), + getattr(message.channel, "id", None), + bool(message.content), + ) + try: + await _persist_message(message, bot_user_id=bot_user_id) + except Exception: + logger.exception("Discord gateway failed to persist message_id=%s", message.id) + + return client + + +async def _run_discord_gateway() -> None: + global _client + token = config.DISCORD_BOT_TOKEN + if not token: + logger.warning("Discord gateway enabled but DISCORD_BOT_TOKEN is not set") + return + + while _shutdown_event is None or not _shutdown_event.is_set(): + _client = _build_client() + try: + await _client.start(token) + except asyncio.CancelledError: + raise + except Exception: + logger.exception("Discord gateway WebSocket failed; retrying in 30s") + finally: + if _client is not None and not _client.is_closed(): + await _client.close() + if _shutdown_event is not None and _shutdown_event.is_set(): + break + try: + await asyncio.wait_for(_shutdown_event.wait(), timeout=30.0) + except (TimeoutError, AttributeError): + continue + + +async def start_discord_gateway_supervisor() -> None: + global _shutdown_event, _task + if not config.GATEWAY_DISCORD_ENABLED: + return + if _task is not None and not _task.done(): + return + _shutdown_event = asyncio.Event() + _task = asyncio.create_task(_run_discord_gateway(), name="gateway-discord-intake") + logger.info("Started Discord gateway intake supervisor") + + +async def stop_discord_gateway_supervisor() -> None: + global _client, _shutdown_event, _task + if _shutdown_event is not None: + _shutdown_event.set() + if _client is not None and not _client.is_closed(): + await _client.close() + if _task is not None: + _task.cancel() + with suppress(TimeoutError, asyncio.CancelledError): + await asyncio.wait_for(_task, timeout=10) + _client = None + _task = None + _shutdown_event = None From 7860714f741c083f657291a1039f0baf9fd634dd Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:59:31 +0530 Subject: [PATCH 52/63] feat(gateway): add Discord gateway install flow --- .../app/routes/gateway_webhook_routes.py | 190 ++++++++++++++++++ .../tests/unit/gateway/test_webhook_routes.py | 28 +++ 2 files changed, 218 insertions(+) diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py index ffadd19d7..adfcd56c6 100644 --- a/surfsense_backend/app/routes/gateway_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -33,11 +33,13 @@ from app.db import ( get_async_session, ) from app.gateway.accounts import ( + get_discord_account_by_guild, get_or_create_system_telegram_account, get_or_create_system_whatsapp_account, get_slack_account_by_team, ) from app.gateway.bindings import resume_binding, revoke_binding +from app.gateway.discord.adapter import discord_user_peer_id from app.gateway.inbox import ( persist_inbound_event, slack_event_dedupe_key, @@ -57,6 +59,9 @@ logger = logging.getLogger(__name__) SLACK_AUTHORIZATION_URL = "https://slack.com/oauth/v2/authorize" SLACK_TOKEN_URL = "https://slack.com/api/oauth.v2.access" +DISCORD_AUTHORIZATION_URL = "https://discord.com/api/oauth2/authorize" +DISCORD_TOKEN_URL = "https://discord.com/api/oauth2/token" +DISCORD_API = "https://discord.com/api/v10" SLACK_BOT_SCOPES = [ "app_mentions:read", "chat:write", @@ -66,6 +71,17 @@ SLACK_BOT_SCOPES = [ "users:read", "team:read", ] +DISCORD_GATEWAY_SCOPES = ["identify", "guilds", "bot"] +DISCORD_VIEW_CHANNEL = 1 << 10 +DISCORD_SEND_MESSAGES = 1 << 11 +DISCORD_READ_MESSAGE_HISTORY = 1 << 16 +DISCORD_SEND_MESSAGES_IN_THREADS = 1 << 38 +DISCORD_GATEWAY_PERMISSIONS = ( + DISCORD_VIEW_CHANNEL + | DISCORD_SEND_MESSAGES + | DISCORD_READ_MESSAGE_HISTORY + | DISCORD_SEND_MESSAGES_IN_THREADS +) _state_manager: OAuthStateManager | None = None _token_encryption: TokenEncryption | None = None @@ -95,6 +111,13 @@ def _slack_redirect_uri() -> str: return f"{base.rstrip('/')}/api/v1/gateway/slack/callback" +def _discord_redirect_uri() -> str: + if config.GATEWAY_DISCORD_REDIRECT_URI: + return config.GATEWAY_DISCORD_REDIRECT_URI + base = config.BACKEND_URL or "" + return f"{base.rstrip('/')}/api/v1/gateway/discord/callback" + + def _slack_frontend_redirect(space_id: int, *, success: bool = False, error: str | None = None) -> RedirectResponse: qs = "slack_gateway=connected" if success else f"error={error or 'slack_gateway_failed'}" return RedirectResponse( @@ -102,6 +125,13 @@ def _slack_frontend_redirect(space_id: int, *, success: bool = False, error: str ) +def _discord_frontend_redirect(space_id: int, *, success: bool = False, error: str | None = None) -> RedirectResponse: + qs = "discord_gateway=connected" if success else f"error={error or 'discord_gateway_failed'}" + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/user-settings?{qs}" + ) + + def verify_slack_signature(*, signing_secret: str, timestamp: str | None, signature: str | None, body: bytes) -> bool: if not signing_secret or not timestamp or not signature: return False @@ -295,6 +325,166 @@ async def slack_gateway_callback( return _slack_frontend_redirect(space_id, success=True) +@router.get("/discord/install") +async def install_discord_gateway( + search_space_id: int, + user: User = Depends(current_active_user), +) -> dict[str, str]: + if not config.DISCORD_CLIENT_ID: + raise HTTPException(status_code=500, detail="Discord gateway OAuth is not configured") + state = _get_state_manager().generate_secure_state(search_space_id, user.id) + auth_params = { + "client_id": config.DISCORD_CLIENT_ID, + "scope": " ".join(DISCORD_GATEWAY_SCOPES), + "redirect_uri": _discord_redirect_uri(), + "response_type": "code", + "state": state, + "permissions": str(DISCORD_GATEWAY_PERMISSIONS), + } + return {"auth_url": f"{DISCORD_AUTHORIZATION_URL}?{urlencode(auth_params)}"} + + +@router.get("/discord/callback") +async def discord_gateway_callback( + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +) -> RedirectResponse: + space_id = None + if state: + try: + state_data = _get_state_manager().validate_state(state) + space_id = int(state_data["space_id"]) + except Exception: + state_data = None + else: + state_data = None + + if error: + return _discord_frontend_redirect(space_id or 0, error="discord_gateway_oauth_denied") + if not code or state_data is None: + raise HTTPException(status_code=400, detail="Invalid Discord gateway OAuth callback") + if not config.DISCORD_CLIENT_ID or not config.DISCORD_CLIENT_SECRET: + raise HTTPException(status_code=500, detail="Discord gateway OAuth is not configured") + if not config.DISCORD_BOT_TOKEN: + raise HTTPException(status_code=500, detail="Discord gateway bot token is not configured") + + user_id = UUID(state_data["user_id"]) + token_payload = { + "client_id": config.DISCORD_CLIENT_ID, + "client_secret": config.DISCORD_CLIENT_SECRET, + "grant_type": "authorization_code", + "code": code, + "redirect_uri": _discord_redirect_uri(), + } + async with httpx.AsyncClient(timeout=30.0) as client: + token_response = await client.post( + DISCORD_TOKEN_URL, + data=token_payload, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + token_response.raise_for_status() + token_json = token_response.json() + + oauth_access_token = token_json.get("access_token") + guild = token_json.get("guild") or {} + guild_id = guild.get("id") + guild_name = guild.get("name") + discord_user_id = None + discord_username = None + if oauth_access_token: + async with httpx.AsyncClient(timeout=30.0) as client: + user_response = await client.get( + f"{DISCORD_API}/users/@me", + headers={"Authorization": f"Bearer {oauth_access_token}"}, + ) + user_response.raise_for_status() + user_json = user_response.json() + discord_user_id = user_json.get("id") + discord_username = user_json.get("username") + + if not guild_id: + raise HTTPException( + status_code=400, + detail=( + "Discord gateway OAuth did not return a guild. " + "Choose a server during bot installation and try again." + ), + ) + + enc = _get_token_encryption() + credentials = { + "bot_token": config.DISCORD_BOT_TOKEN, + "token_type": "bot", + "scope": token_json.get("scope"), + } + cursor_state = { + "guild_id": guild_id, + "guild_name": guild_name, + "application_id": config.DISCORD_CLIENT_ID, + "scope": token_json.get("scope"), + "permissions": str(DISCORD_GATEWAY_PERMISSIONS), + } + + account = await get_discord_account_by_guild(session, guild_id=str(guild_id)) + if account is None: + account = ExternalChatAccount( + platform=ExternalChatPlatform.DISCORD, + mode=ExternalChatAccountMode.CLOUD_SHARED, + is_system_account=True, + encrypted_credentials=enc.encrypt_token(json.dumps(credentials)), + bot_username="SurfSense", + cursor_state=cursor_state, + health_status=ExternalChatHealthStatus.UNKNOWN, + ) + session.add(account) + await session.flush() + else: + account.encrypted_credentials = enc.encrypt_token(json.dumps(credentials)) + account.cursor_state = {**(account.cursor_state or {}), **cursor_state} + account.health_status = ExternalChatHealthStatus.UNKNOWN + + if discord_user_id: + peer_id = discord_user_peer_id(str(guild_id), str(discord_user_id)) + existing_binding_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + binding = existing_binding_result.scalars().first() + metadata = { + "kind": "discord_user", + "guild_id": guild_id, + "guild_name": guild_name, + "discord_user_id": discord_user_id, + } + if binding is None: + session.add( + ExternalChatBinding( + account_id=account.id, + user_id=user_id, + search_space_id=space_id, + state=ExternalChatBindingState.BOUND, + external_peer_id=peer_id, + external_peer_kind=ExternalChatPeerKind.DIRECT, + external_username=discord_username or discord_user_id, + external_metadata=metadata, + ) + ) + elif binding.user_id == user_id: + binding.search_space_id = space_id + binding.external_username = discord_username or binding.external_username + binding.external_metadata = {**(binding.external_metadata or {}), **metadata} + + await session.commit() + return _discord_frontend_redirect(space_id, success=True) + + @router.post("/webhooks/slack") async def slack_webhook( request: Request, diff --git a/surfsense_backend/tests/unit/gateway/test_webhook_routes.py b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py index a624cbde1..34d0651ab 100644 --- a/surfsense_backend/tests/unit/gateway/test_webhook_routes.py +++ b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py @@ -5,6 +5,7 @@ import hmac import inspect import json import time +from types import SimpleNamespace import pytest @@ -272,3 +273,30 @@ async def test_slack_webhook_ignores_self_event(monkeypatch, mocker): assert response.status_code == 200 persist.assert_not_awaited() + +@pytest.mark.asyncio +async def test_discord_gateway_install_returns_oauth_url(monkeypatch): + monkeypatch.setattr(routes.config, "DISCORD_CLIENT_ID", "discord-client") + monkeypatch.setattr( + routes.config, + "GATEWAY_DISCORD_REDIRECT_URI", + "http://localhost:8000/api/v1/gateway/discord/callback", + ) + monkeypatch.setattr(routes.config, "SECRET_KEY", "test-secret") + + response = await routes.install_discord_gateway( + search_space_id=123, + user=SimpleNamespace(id="00000000-0000-0000-0000-000000000001"), + ) + + assert response["auth_url"].startswith("https://discord.com/api/oauth2/authorize?") + assert "client_id=discord-client" in response["auth_url"] + assert "gateway%2Fdiscord%2Fcallback" in response["auth_url"] + assert "scope=identify+guilds+bot" in response["auth_url"] + + +def test_discord_gateway_callback_does_not_create_search_source_connector(): + callback_source = inspect.getsource(routes.discord_gateway_callback) + + assert "SearchSourceConnector" not in callback_source + From e85710dd7b1ab5196c4a721f5c64efd7d7588349 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:59:41 +0530 Subject: [PATCH 53/63] feat(gateway): include Discord in gateway maintenance --- surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py index e227db71d..898d8c8af 100644 --- a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py @@ -92,6 +92,13 @@ def gateway_health_check_task() -> None: cursor_state[key] = metadata[key] account.cursor_state = cursor_state account.bot_username = metadata.get("bot_username") + elif account.platform == ExternalChatPlatform.DISCORD: + cursor_state = dict(account.cursor_state or {}) + for key in ("bot_user_id", "bot_username", "global_name"): + if key in metadata: + cursor_state[key] = metadata[key] + account.cursor_state = cursor_state + account.bot_username = metadata.get("bot_username") except Exception: logger.warning( "External chat health check failed platform=%s account_id=%s", From 455a3ee021c86729f85cf6329774ec2613518c1c Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:59:52 +0530 Subject: [PATCH 54/63] feat(web): add Discord messaging channel setup --- surfsense_web/.env.example | 2 + .../components/MessagingChannelsContent.tsx | 50 ++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/surfsense_web/.env.example b/surfsense_web/.env.example index 032bb48ea..e4aaf91d7 100644 --- a/surfsense_web/.env.example +++ b/surfsense_web/.env.example @@ -12,6 +12,8 @@ NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848 NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE=disabled # Slack gateway UI toggle: true or false NEXT_PUBLIC_GATEWAY_SLACK_ENABLED=false +# Discord gateway UI toggle: true or false +NEXT_PUBLIC_GATEWAY_DISCORD_ENABLED=false # Contact Form Vars (optional) DATABASE_URL=postgresql://postgres:[YOUR-PASSWORD]@db.sdsf.supabase.co:5432/postgres diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx index 57dfe321e..7d6f7cba6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx @@ -52,6 +52,7 @@ export function MessagingChannelsContent() { const searchSpaceId = Number(params.search_space_id); const whatsappMode = process.env.NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE ?? "disabled"; const slackGatewayEnabled = process.env.NEXT_PUBLIC_GATEWAY_SLACK_ENABLED === "true"; + const discordGatewayEnabled = process.env.NEXT_PUBLIC_GATEWAY_DISCORD_ENABLED === "true"; const [bindings, setBindings] = useState([]); const [platforms, setPlatforms] = useState([]); const [pairing, setPairing] = useState(null); @@ -109,6 +110,17 @@ export function MessagingChannelsContent() { } } + async function installDiscordGateway() { + const res = await authenticatedFetch( + `${BACKEND_URL}/api/v1/gateway/discord/install?search_space_id=${searchSpaceId}` + ); + if (!res.ok) return; + const data = (await res.json()) as { auth_url?: string }; + if (data.auth_url) { + window.location.href = data.auth_url; + } + } + function refreshBaileys() { startTransition(async () => { await refreshBaileysHealth(); @@ -133,11 +145,13 @@ export function MessagingChannelsContent() { const telegram = platforms.find((p) => p.platform === "telegram"); const whatsapp = platforms.find((p) => p.platform === "whatsapp"); const slack = platforms.find((p) => p.platform === "slack"); + const discord = platforms.find((p) => p.platform === "discord"); const baileysQr = baileysHealth?.qr || null; const activeBindings = bindings.filter( (binding) => binding.search_space_id === searchSpaceId && - binding.external_metadata?.kind !== "slack_thread" + binding.external_metadata?.kind !== "slack_thread" && + binding.external_metadata?.kind !== "discord_thread" ); const renderPairingPanel = (platform: PairingPlatform) => { if (!pairing || pairingPlatform !== platform) return null; @@ -222,6 +236,40 @@ export function MessagingChannelsContent() { ) : null} + {discordGatewayEnabled ? ( + + +
+ + + Discord Bot + + + {discord ? "enabled" : "not enabled"} + +
+

+ Enable the SurfSense Discord bot so teammates can mention it in Discord. This is + separate from the Discord connector. +

+ + +
+ + +
+

+ Discord search remains controlled by the Discord connector in the connector popup. +

+ + + ) : null} + {whatsappMode !== "disabled" ? ( From 2d1a6be776c323051d3f2ce223c23c9eb686e3ff Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 21:39:09 +0530 Subject: [PATCH 55/63] feat(gateway): implement search space management for messaging channels --- .../app/routes/gateway_webhook_routes.py | 91 +++++++++++ .../routes/gateway_whatsapp_baileys_routes.py | 2 + .../components/MessagingChannelsContent.tsx | 152 +++++++++++++----- 3 files changed, 205 insertions(+), 40 deletions(-) diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py index adfcd56c6..d4b574c26 100644 --- a/surfsense_backend/app/routes/gateway_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -53,6 +53,7 @@ from app.observability.metrics import ( ) from app.users import current_active_user from app.utils.oauth_security import OAuthStateManager, TokenEncryption +from app.utils.rbac import check_search_space_access router = APIRouter(prefix="/gateway", tags=["gateway"]) logger = logging.getLogger(__name__) @@ -164,6 +165,10 @@ class StartBindingResponse(BaseModel): expires_at: datetime +class UpdateBindingSearchSpaceRequest(BaseModel): + search_space_id: int + + def _classify_telegram_event(payload: dict[str, Any]) -> str: if "message" in payload: return "message" @@ -182,9 +187,11 @@ def _telegram_message(payload: dict[str, Any]) -> dict[str, Any] | None: async def install_slack_gateway( search_space_id: int, user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), ) -> dict[str, str]: if not config.GATEWAY_SLACK_CLIENT_ID: raise HTTPException(status_code=500, detail="Slack gateway OAuth is not configured") + await check_search_space_access(session, user, search_space_id) state = _get_state_manager().generate_secure_state(search_space_id, user.id) auth_params = { "client_id": config.GATEWAY_SLACK_CLIENT_ID, @@ -329,9 +336,11 @@ async def slack_gateway_callback( async def install_discord_gateway( search_space_id: int, user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), ) -> dict[str, str]: if not config.DISCORD_CLIENT_ID: raise HTTPException(status_code=500, detail="Discord gateway OAuth is not configured") + await check_search_space_access(session, user, search_space_id) state = _get_state_manager().generate_secure_state(search_space_id, user.id) auth_params = { "client_id": config.DISCORD_CLIENT_ID, @@ -613,6 +622,7 @@ async def start_binding( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> StartBindingResponse: + await check_search_space_access(session, user, body.search_space_id) code = generate_pairing_code() if body.platform == ExternalChatPlatform.TELEGRAM: account = await get_or_create_system_telegram_account(session) @@ -692,6 +702,62 @@ async def list_bindings( ] +@router.get("/connections") +async def list_connections( + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> list[dict[str, Any]]: + result = await session.execute( + select(ExternalChatBinding, ExternalChatAccount) + .join(ExternalChatAccount, ExternalChatBinding.account_id == ExternalChatAccount.id) + .where( + ExternalChatBinding.user_id == user.id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + + connections: list[dict[str, Any]] = [] + for binding, account in result.all(): + binding_metadata = binding.external_metadata or {} + kind = str(binding_metadata.get("kind") or "") + if kind in {"slack_thread", "discord_thread"}: + continue + + account_state = account.cursor_state or {} + workspace_name = None + workspace_id = None + if account.platform == ExternalChatPlatform.SLACK: + workspace_name = account_state.get("team_name") + workspace_id = account_state.get("team_id") + elif account.platform == ExternalChatPlatform.DISCORD: + workspace_name = account_state.get("guild_name") + workspace_id = account_state.get("guild_id") + elif account.platform == ExternalChatPlatform.WHATSAPP: + workspace_name = account_state.get("display_phone_number") + workspace_id = account_state.get("phone_number_id") + + connections.append( + { + "id": binding.id, + "platform": account.platform.value, + "state": binding.state.value, + "search_space_id": binding.search_space_id, + "display_name": binding.external_display_name + or binding.external_username + or workspace_name, + "external_username": binding.external_username, + "workspace_name": workspace_name, + "workspace_id": workspace_id, + "health_status": account.health_status.value, + "suspended_reason": binding.suspended_reason, + } + ) + + return connections + + @router.get("/platforms") async def list_platforms( user: User = Depends(current_active_user), @@ -716,6 +782,31 @@ async def list_platforms( ] +@router.patch("/bindings/{binding_id}/search-space") +async def update_binding_search_space( + binding_id: int, + body: UpdateBindingSearchSpaceRequest, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, bool]: + binding = await session.get(ExternalChatBinding, binding_id) + if binding is None or binding.user_id != user.id: + raise HTTPException(status_code=404, detail="Binding not found") + if binding.state not in { + ExternalChatBindingState.BOUND, + ExternalChatBindingState.SUSPENDED, + }: + raise HTTPException(status_code=400, detail="Only active bindings can be routed") + + await check_search_space_access(session, user, body.search_space_id) + if binding.search_space_id != body.search_space_id: + binding.search_space_id = body.search_space_id + binding.new_chat_thread_id = None + binding.updated_at = datetime.now(UTC) + await session.commit() + return {"ok": True} + + @router.delete("/bindings/{binding_id}") async def delete_binding( binding_id: int, diff --git a/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py index 24209f86f..fa49d0558 100644 --- a/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py +++ b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py @@ -21,6 +21,7 @@ from app.db import ( ) from app.gateway.whatsapp.adapter_baileys import WhatsAppBaileysAdapter from app.users import current_active_user +from app.utils.rbac import check_search_space_access router = APIRouter(prefix="/gateway/whatsapp/baileys", tags=["gateway"]) @@ -61,6 +62,7 @@ async def request_pairing_code( session: AsyncSession = Depends(get_async_session), ) -> dict[str, Any]: _ensure_baileys_enabled() + await check_search_space_access(session, user, body.search_space_id) adapter = WhatsAppBaileysAdapter() try: pairing = await adapter.request_pairing_code(phone_number=body.phone_number) diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx index 7d6f7cba6..ff6365499 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx @@ -4,21 +4,33 @@ import { MessageCircle, RefreshCw, ShieldAlert } from "lucide-react"; import { useParams } from "next/navigation"; import { QRCodeSVG } from "qrcode.react"; import { useCallback, useEffect, useState, useTransition } from "react"; +import { toast } from "sonner"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import type { SearchSpace } from "@/contracts/types/search-space.types"; +import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service"; import { authenticatedFetch } from "@/lib/auth-utils"; import { BACKEND_URL } from "@/lib/env-config"; -type Binding = { +type GatewayConnection = { id: number; - platform?: string; + platform: string; state: string; search_space_id: number; - external_display_name?: string | null; + display_name?: string | null; external_username?: string | null; + workspace_name?: string | null; + workspace_id?: string | null; + health_status: string; suspended_reason?: string | null; - external_metadata?: Record | null; }; type Platform = { @@ -53,8 +65,9 @@ export function MessagingChannelsContent() { const whatsappMode = process.env.NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE ?? "disabled"; const slackGatewayEnabled = process.env.NEXT_PUBLIC_GATEWAY_SLACK_ENABLED === "true"; const discordGatewayEnabled = process.env.NEXT_PUBLIC_GATEWAY_DISCORD_ENABLED === "true"; - const [bindings, setBindings] = useState([]); + const [connections, setConnections] = useState([]); const [platforms, setPlatforms] = useState([]); + const [searchSpaces, setSearchSpaces] = useState([]); const [pairing, setPairing] = useState(null); const [pairingPlatform, setPairingPlatform] = useState(null); const [baileysHealth, setBaileysHealth] = useState(null); @@ -63,12 +76,14 @@ export function MessagingChannelsContent() { const refresh = useCallback(async () => { setLoading(true); - const [bindingsRes, platformsRes] = await Promise.all([ - authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings`), + const [connectionsRes, platformsRes, spaces] = await Promise.all([ + authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/connections`), authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/platforms`), + searchSpacesApiService.getSearchSpaces(), ]); - setBindings(await bindingsRes.json()); + setConnections(await connectionsRes.json()); setPlatforms(await platformsRes.json()); + setSearchSpaces(spaces); setLoading(false); }, []); @@ -135,6 +150,31 @@ export function MessagingChannelsContent() { await refresh(); } + async function updateConnectionSearchSpace(id: number, nextSearchSpaceId: string) { + const previousConnections = connections; + const parsedSearchSpaceId = Number(nextSearchSpaceId); + setConnections((current) => + current.map((connection) => + connection.id === id ? { ...connection, search_space_id: parsedSearchSpaceId } : connection + ) + ); + const res = await authenticatedFetch( + `${BACKEND_URL}/api/v1/gateway/bindings/${id}/search-space`, + { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ search_space_id: parsedSearchSpaceId }), + } + ); + if (!res.ok) { + setConnections(previousConnections); + toast.error("Failed to update messaging route"); + return; + } + toast.success("Messaging route updated"); + await refresh(); + } + async function resume(id: number) { await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings/${id}/resume`, { method: "POST", @@ -147,12 +187,27 @@ export function MessagingChannelsContent() { const slack = platforms.find((p) => p.platform === "slack"); const discord = platforms.find((p) => p.platform === "discord"); const baileysQr = baileysHealth?.qr || null; - const activeBindings = bindings.filter( - (binding) => - binding.search_space_id === searchSpaceId && - binding.external_metadata?.kind !== "slack_thread" && - binding.external_metadata?.kind !== "discord_thread" - ); + const currentSearchSpaceName = + searchSpaces.find((space) => space.id === searchSpaceId)?.name || "this search space"; + const platformLabel = (platform: string) => { + switch (platform) { + case "discord": + return "Discord"; + case "slack": + return "Slack"; + case "telegram": + return "Telegram"; + case "whatsapp": + return "WhatsApp"; + default: + return platform; + } + }; + const connectionTitle = (connection: GatewayConnection) => + connection.workspace_name || + connection.display_name || + connection.external_username || + `${platformLabel(connection.platform)} connection`; const renderPairingPanel = (platform: PairingPlatform) => { if (!pairing || pairingPlatform !== platform) return null; @@ -221,16 +276,15 @@ export function MessagingChannelsContent() {
- +

- Slack search remains controlled by the Slack connector in the connector popup. + New Slack workspace connections will route to {currentSearchSpaceName} first. You can + change each connection's search space below.

@@ -255,16 +309,15 @@ export function MessagingChannelsContent() {
- +

- Discord search remains controlled by the Discord connector in the connector popup. + New Discord server connections will route to {currentSearchSpaceName} first. You can + change each connection's search space below.

@@ -329,39 +382,58 @@ export function MessagingChannelsContent() { - Active Chats + Connected Messaging Channels +

+ Choose which search space each external channel should use when messages arrive. +

- {activeBindings.length === 0 ? ( -

No external chats connected yet.

+ {connections.length === 0 ? ( +

No messaging channels connected yet.

) : ( - activeBindings.map((binding) => ( + connections.map((connection) => (
-
-

- {binding.external_display_name || - binding.external_username || - `Binding ${binding.id}`} +

+

{connectionTitle(connection)}

+

+ {platformLabel(connection.platform)} + {connection.external_username ? ` · ${connection.external_username}` : ""} + {connection.state ? ` · ${connection.state}` : ""}

-

{binding.state}

- {binding.suspended_reason ? ( + {connection.suspended_reason ? (

- {binding.suspended_reason} + {connection.suspended_reason}

) : null}
-
- {binding.state === "suspended" ? ( - ) : null} -
From a151e8f72998d7ff93601c997c1ec2d2cbcd28b9 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 22:49:46 +0530 Subject: [PATCH 56/63] feat(gateway): enhance WhatsApp account management and connection handling --- .../app/routes/gateway_webhook_routes.py | 155 +++++- .../routes/gateway_whatsapp_baileys_routes.py | 2 + .../components/MessagingChannelsContent.tsx | 452 ++++++++++-------- 3 files changed, 399 insertions(+), 210 deletions(-) diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py index d4b574c26..a47bcd0c5 100644 --- a/surfsense_backend/app/routes/gateway_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -169,6 +169,10 @@ class UpdateBindingSearchSpaceRequest(BaseModel): search_space_id: int +class UpdateAccountSearchSpaceRequest(BaseModel): + search_space_id: int + + def _classify_telegram_event(payload: dict[str, Any]) -> str: if "message" in payload: return "message" @@ -704,21 +708,27 @@ async def list_bindings( @router.get("/connections") async def list_connections( + platform: ExternalChatPlatform | None = None, user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> list[dict[str, Any]]: + filters = [ + ExternalChatBinding.user_id == user.id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ] + if platform is not None: + filters.append(ExternalChatAccount.platform == platform) + result = await session.execute( select(ExternalChatBinding, ExternalChatAccount) .join(ExternalChatAccount, ExternalChatBinding.account_id == ExternalChatAccount.id) - .where( - ExternalChatBinding.user_id == user.id, - ExternalChatBinding.state.in_( - [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] - ), - ) + .where(*filters) ) connections: list[dict[str, Any]] = [] + baileys_account_ids: set[int] = set() for binding, account in result.all(): binding_metadata = binding.external_metadata or {} kind = str(binding_metadata.get("kind") or "") @@ -728,6 +738,10 @@ async def list_connections( account_state = account.cursor_state or {} workspace_name = None workspace_id = None + route_type = "binding" + connection_id = binding.id + search_space_id = binding.search_space_id + display_name = binding.external_display_name or binding.external_username if account.platform == ExternalChatPlatform.SLACK: workspace_name = account_state.get("team_name") workspace_id = account_state.get("team_id") @@ -737,17 +751,30 @@ async def list_connections( elif account.platform == ExternalChatPlatform.WHATSAPP: workspace_name = account_state.get("display_phone_number") workspace_id = account_state.get("phone_number_id") + if account.mode == ExternalChatAccountMode.SELF_HOST_BYO: + if int(account.id) in baileys_account_ids: + continue + baileys_account_ids.add(int(account.id)) + route_type = "account" + connection_id = account.id + search_space_id = account.owner_search_space_id or binding.search_space_id + display_name = "WhatsApp Bridge" connections.append( { - "id": binding.id, + "id": connection_id, + "account_id": account.id, + "route_type": route_type, "platform": account.platform.value, + "mode": account.mode.value, "state": binding.state.value, - "search_space_id": binding.search_space_id, - "display_name": binding.external_display_name - or binding.external_username - or workspace_name, - "external_username": binding.external_username, + "search_space_id": search_space_id, + "display_name": display_name or workspace_name, + "external_username": ( + None + if account.mode == ExternalChatAccountMode.SELF_HOST_BYO + else binding.external_username + ), "workspace_name": workspace_name, "workspace_id": workspace_id, "health_status": account.health_status.value, @@ -755,6 +782,37 @@ async def list_connections( } ) + if platform is None or platform == ExternalChatPlatform.WHATSAPP: + account_result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.owner_user_id == user.id, + ExternalChatAccount.platform == ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.mode == ExternalChatAccountMode.SELF_HOST_BYO, + ExternalChatAccount.owner_search_space_id.is_not(None), + ) + ) + for account in account_result.scalars(): + if int(account.id) in baileys_account_ids: + continue + account_state = account.cursor_state or {} + connections.append( + { + "id": account.id, + "account_id": account.id, + "route_type": "account", + "platform": account.platform.value, + "mode": account.mode.value, + "state": "bound", + "search_space_id": account.owner_search_space_id, + "display_name": "WhatsApp Bridge", + "external_username": None, + "workspace_name": account_state.get("display_phone_number"), + "workspace_id": account_state.get("phone_number_id"), + "health_status": account.health_status.value, + "suspended_reason": account.suspended_reason, + } + ) + return connections @@ -807,6 +865,44 @@ async def update_binding_search_space( return {"ok": True} +@router.patch("/accounts/{account_id}/search-space") +async def update_gateway_account_search_space( + account_id: int, + body: UpdateAccountSearchSpaceRequest, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, bool]: + account = await session.get(ExternalChatAccount, account_id) + if ( + account is None + or account.owner_user_id != user.id + or account.platform != ExternalChatPlatform.WHATSAPP + or account.mode != ExternalChatAccountMode.SELF_HOST_BYO + ): + raise HTTPException(status_code=404, detail="Gateway account not found") + + await check_search_space_access(session, user, body.search_space_id) + account.owner_search_space_id = body.search_space_id + account.updated_at = datetime.now(UTC) + + result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.user_id == user.id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + for binding in result.scalars(): + binding.search_space_id = body.search_space_id + binding.new_chat_thread_id = None + binding.updated_at = datetime.now(UTC) + + await session.commit() + return {"ok": True} + + @router.delete("/bindings/{binding_id}") async def delete_binding( binding_id: int, @@ -821,6 +917,41 @@ async def delete_binding( return {"ok": True} +@router.delete("/accounts/{account_id}") +async def delete_gateway_account( + account_id: int, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, bool]: + account = await session.get(ExternalChatAccount, account_id) + if ( + account is None + or account.owner_user_id != user.id + or account.platform != ExternalChatPlatform.WHATSAPP + or account.mode != ExternalChatAccountMode.SELF_HOST_BYO + ): + raise HTTPException(status_code=404, detail="Gateway account not found") + + result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.user_id == user.id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + for binding in result.scalars(): + revoke_binding(binding) + + account.owner_search_space_id = None + account.suspended_at = datetime.now(UTC) + account.suspended_reason = "disconnected" + account.updated_at = datetime.now(UTC) + await session.commit() + return {"ok": True} + + @router.post("/bindings/{binding_id}/resume") async def resume_external_chat_binding( binding_id: int, diff --git a/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py index fa49d0558..5ab669503 100644 --- a/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py +++ b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py @@ -85,6 +85,8 @@ async def request_pairing_code( account.mode = ExternalChatAccountMode.SELF_HOST_BYO account.owner_search_space_id = body.search_space_id account.health_status = ExternalChatHealthStatus.UNKNOWN + account.suspended_at = None + account.suspended_reason = None account.last_health_check_at = datetime.now(UTC) await session.commit() await session.refresh(account) diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx index ff6365499..0aa156980 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx @@ -1,11 +1,10 @@ "use client"; -import { MessageCircle, RefreshCw, ShieldAlert } from "lucide-react"; +import { RefreshCw, ShieldAlert } from "lucide-react"; import { useParams } from "next/navigation"; import { QRCodeSVG } from "qrcode.react"; -import { useCallback, useEffect, useState, useTransition } from "react"; +import { useCallback, useEffect, useState } from "react"; import { toast } from "sonner"; -import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { @@ -15,14 +14,19 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; +import { Separator } from "@/components/ui/separator"; import type { SearchSpace } from "@/contracts/types/search-space.types"; import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service"; import { authenticatedFetch } from "@/lib/auth-utils"; import { BACKEND_URL } from "@/lib/env-config"; +import { cn } from "@/lib/utils"; type GatewayConnection = { id: number; + account_id?: number | null; + route_type?: "account" | "binding"; platform: string; + mode?: string; state: string; search_space_id: number; display_name?: string | null; @@ -33,15 +37,6 @@ type GatewayConnection = { suspended_reason?: string | null; }; -type Platform = { - id: number; - platform: string; - mode: string; - bot_username?: string | null; - health_status: string; - last_health_check_at?: string | null; -}; - type Pairing = { binding_id: number; code: string; @@ -50,6 +45,7 @@ type Pairing = { }; type PairingPlatform = "telegram" | "whatsapp"; +type GatewayPlatform = PairingPlatform | "slack" | "discord"; type BaileysHealth = { status: string; @@ -66,31 +62,47 @@ export function MessagingChannelsContent() { const slackGatewayEnabled = process.env.NEXT_PUBLIC_GATEWAY_SLACK_ENABLED === "true"; const discordGatewayEnabled = process.env.NEXT_PUBLIC_GATEWAY_DISCORD_ENABLED === "true"; const [connections, setConnections] = useState([]); - const [platforms, setPlatforms] = useState([]); const [searchSpaces, setSearchSpaces] = useState([]); const [pairing, setPairing] = useState(null); const [pairingPlatform, setPairingPlatform] = useState(null); const [baileysHealth, setBaileysHealth] = useState(null); - const [loading, setLoading] = useState(true); - const [isPending, startTransition] = useTransition(); + const [refreshingPlatform, setRefreshingPlatform] = useState(null); + + const fetchConnections = useCallback(async (platform?: GatewayPlatform) => { + const query = platform ? `?platform=${encodeURIComponent(platform)}` : ""; + const res = await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/connections${query}`); + return (await res.json()) as GatewayConnection[]; + }, []); const refresh = useCallback(async () => { - setLoading(true); - const [connectionsRes, platformsRes, spaces] = await Promise.all([ - authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/connections`), - authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/platforms`), + const [nextConnections, spaces] = await Promise.all([ + fetchConnections(), searchSpacesApiService.getSearchSpaces(), ]); - setConnections(await connectionsRes.json()); - setPlatforms(await platformsRes.json()); + setConnections(nextConnections); setSearchSpaces(spaces); - setLoading(false); - }, []); + }, [fetchConnections]); useEffect(() => { void refresh(); }, [refresh]); + const refreshPlatform = useCallback( + async (platform: GatewayPlatform) => { + setRefreshingPlatform(platform); + try { + const nextConnections = await fetchConnections(platform); + setConnections((current) => [ + ...current.filter((connection) => connection.platform !== platform), + ...nextConnections, + ]); + } finally { + setRefreshingPlatform(null); + } + }, + [fetchConnections] + ); + const refreshBaileysHealth = useCallback(async () => { if (whatsappMode !== "baileys") return; const res = await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/whatsapp/baileys/health`); @@ -111,7 +123,7 @@ export function MessagingChannelsContent() { }); setPairing(await res.json()); setPairingPlatform(platform); - await refresh(); + await refreshPlatform(platform); } async function installSlackGateway() { @@ -136,59 +148,77 @@ export function MessagingChannelsContent() { } } - function refreshBaileys() { - startTransition(async () => { - await refreshBaileysHealth(); - await refresh(); - }); + async function refreshBaileys() { + await refreshBaileysHealth(); + await refreshPlatform("whatsapp"); } - async function revoke(id: number) { - await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings/${id}`, { + const connectionKey = (connection: GatewayConnection) => + connection.route_type === "account" && connection.account_id + ? `account:${connection.account_id}` + : `binding:${connection.id}`; + + async function revoke(connection: GatewayConnection) { + const url = + connection.route_type === "account" && connection.account_id + ? `${BACKEND_URL}/api/v1/gateway/accounts/${connection.account_id}` + : `${BACKEND_URL}/api/v1/gateway/bindings/${connection.id}`; + await authenticatedFetch(url, { method: "DELETE", }); - await refresh(); + await refreshPlatform(connection.platform as GatewayPlatform); } - async function updateConnectionSearchSpace(id: number, nextSearchSpaceId: string) { + async function updateConnectionSearchSpace( + connection: GatewayConnection, + nextSearchSpaceId: string + ) { const previousConnections = connections; const parsedSearchSpaceId = Number(nextSearchSpaceId); + const targetKey = connectionKey(connection); setConnections((current) => current.map((connection) => - connection.id === id ? { ...connection, search_space_id: parsedSearchSpaceId } : connection + connectionKey(connection) === targetKey + ? { ...connection, search_space_id: parsedSearchSpaceId } + : connection ) ); - const res = await authenticatedFetch( - `${BACKEND_URL}/api/v1/gateway/bindings/${id}/search-space`, - { - method: "PATCH", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ search_space_id: parsedSearchSpaceId }), - } - ); + const url = + connection.route_type === "account" && connection.account_id + ? `${BACKEND_URL}/api/v1/gateway/accounts/${connection.account_id}/search-space` + : `${BACKEND_URL}/api/v1/gateway/bindings/${connection.id}/search-space`; + const res = await authenticatedFetch(url, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ search_space_id: parsedSearchSpaceId }), + }); if (!res.ok) { setConnections(previousConnections); toast.error("Failed to update messaging route"); return; } toast.success("Messaging route updated"); - await refresh(); + await refreshPlatform(connection.platform as GatewayPlatform); } - async function resume(id: number) { - await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings/${id}/resume`, { + async function resume(connection: GatewayConnection) { + await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/bindings/${connection.id}/resume`, { method: "POST", }); - await refresh(); + await refreshPlatform(connection.platform as GatewayPlatform); } - const telegram = platforms.find((p) => p.platform === "telegram"); - const whatsapp = platforms.find((p) => p.platform === "whatsapp"); - const slack = platforms.find((p) => p.platform === "slack"); - const discord = platforms.find((p) => p.platform === "discord"); const baileysQr = baileysHealth?.qr || null; - const currentSearchSpaceName = - searchSpaces.find((space) => space.id === searchSpaceId)?.name || "this search space"; + const hasTelegramConnection = connections.some( + (connection) => connection.platform === "telegram" + ); + const hasWhatsAppConnection = connections.some( + (connection) => connection.platform === "whatsapp" + ); + const isRefreshing = (platform: GatewayPlatform) => refreshingPlatform === platform; + const refreshButtonClassName = "gap-2"; + const refreshIconClassName = (platform: GatewayPlatform) => + cn("mr-2 h-4 w-4", isRefreshing(platform) && "animate-spin"); const platformLabel = (platform: string) => { switch (platform) { case "discord": @@ -204,16 +234,85 @@ export function MessagingChannelsContent() { } }; const connectionTitle = (connection: GatewayConnection) => - connection.workspace_name || - connection.display_name || - connection.external_username || - `${platformLabel(connection.platform)} connection`; + connection.platform === "whatsapp" && connection.mode === "self_host_byo" + ? "WhatsApp Bridge" + : connection.workspace_name || + connection.display_name || + connection.external_username || + `${platformLabel(connection.platform)} connection`; + const renderConnectionRows = (platform: GatewayConnection["platform"], emptyText: string) => { + const platformConnections = connections.filter( + (connection) => connection.platform === platform + ); + + if (platformConnections.length === 0) { + return

{emptyText}

; + } + + return ( +
+

Connected accounts

+ {platformConnections.map((connection, index) => ( +
+ {index > 0 ? : null} +
+
+

{connectionTitle(connection)}

+ {connection.suspended_reason ? ( +

+ + {connection.suspended_reason} +

+ ) : null} +
+
+ + {connection.state === "suspended" ? ( + + ) : null} + +
+
+
+ ))} +
+ ); + }; const renderPairingPanel = (platform: PairingPlatform) => { if (!pairing || pairingPlatform !== platform) return null; return ( -
-

Pairing code

+
+

Pairing code

{pairing.code}

Open {platform === "whatsapp" ? "WhatsApp" : "Telegram"} pairing link @@ -227,136 +326,153 @@ export function MessagingChannelsContent() { }; return ( -
- - +
+ +
- - - Telegram - - - {telegram?.health_status ?? "not configured"} - + Telegram
-

- Pair a Telegram chat with this search space. Telegram conversations stay in Telegram and - are not mirrored in SurfSense chat history. -

+

Pair Telegram with this search space.

 - +
- - + )} +
- {renderPairingPanel("telegram")} + {hasTelegramConnection ? null : renderPairingPanel("telegram")} + + {renderConnectionRows("telegram", "No Telegram chats connected yet.")} {slackGatewayEnabled ? ( - - + +
- - - Slack Bot - - - {slack ? "enabled" : "not enabled"} - + Slack
-

- Enable the SurfSense Slack bot so teammates can mention it in Slack. This is separate - from the Slack search connector. +

+ Enable the SurfSense Slack bot so teammates can mention it in Slack.

- +
- - +
-

- New Slack workspace connections will route to {currentSearchSpaceName} first. You can - change each connection's search space below. -

+ + {renderConnectionRows("slack", "No Slack workspaces connected yet.")} ) : null} {discordGatewayEnabled ? ( - - + +
- - - Discord Bot - - - {discord ? "enabled" : "not enabled"} - + Discord
-

- Enable the SurfSense Discord bot so teammates can mention it in Discord. This is - separate from the Discord connector. +

+ Enable the SurfSense Discord bot so teammates can mention it in Discord.

- +
- - +
-

- New Discord server connections will route to {currentSearchSpaceName} first. You can - change each connection's search space below. -

+ + {renderConnectionRows("discord", "No Discord servers connected yet.")} ) : null} {whatsappMode !== "disabled" ? ( - - + +
- - - WhatsApp - - - {whatsapp?.health_status ?? "not configured"} - + WhatsApp
-

+

Pair this search space with WhatsApp using the configured gateway mode. + {whatsappMode === "baileys" + ? " Send messages to your own WhatsApp chat. Other chats are ignored." + : ""}

- + {whatsappMode === "cloud" ? (
- - {renderPairingPanel("whatsapp")} +
+ {hasWhatsAppConnection ? null : ( + + )} + +
+ {hasWhatsAppConnection ? null : renderPairingPanel("whatsapp")}
) : null} {whatsappMode === "baileys" ? (
-

- Self-hosted WhatsApp uses Message Yourself mode. After pairing, send messages in - your own WhatsApp chat with yourself; messages from other chats are ignored. -

- {baileysQr ? ( -
+

WhatsApp QR pairing

Scan this QR from WhatsApp > Linked Devices > Link a Device. @@ -376,71 +492,11 @@ export function MessagingChannelsContent() { ) : null}

) : null} + + {renderConnectionRows("whatsapp", "No WhatsApp chats connected yet.")} ) : null} - - - - Connected Messaging Channels -

- Choose which search space each external channel should use when messages arrive. -

- - - {connections.length === 0 ? ( -

No messaging channels connected yet.

- ) : ( - connections.map((connection) => ( -
-
-

{connectionTitle(connection)}

-

- {platformLabel(connection.platform)} - {connection.external_username ? ` · ${connection.external_username}` : ""} - {connection.state ? ` · ${connection.state}` : ""} -

- {connection.suspended_reason ? ( -

- - {connection.suspended_reason} -

- ) : null} -
-
- - {connection.state === "suspended" ? ( - - ) : null} - -
-
- )) - )} - -
); } From fc2467be3dfc1b02115220c602a72572303bd8f2 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:08:56 +0530 Subject: [PATCH 57/63] feat(gateway): improve WhatsApp account mode handling and connection filtering --- .../app/gateway/inbox_processor.py | 21 ++++++++ .../app/routes/gateway_webhook_routes.py | 48 ++++++++++++++++++- .../components/MessagingChannelsContent.tsx | 25 ++++++---- 3 files changed, 82 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/gateway/inbox_processor.py b/surfsense_backend/app/gateway/inbox_processor.py index d47206443..478c42a5e 100644 --- a/surfsense_backend/app/gateway/inbox_processor.py +++ b/surfsense_backend/app/gateway/inbox_processor.py @@ -16,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from app.config import config from app.db import ( ExternalChatAccount, + ExternalChatAccountMode, ExternalChatBinding, ExternalChatBindingState, ExternalChatEventStatus, @@ -40,6 +41,21 @@ def _dashboard_url() -> str: return config.NEXT_FRONTEND_URL or "/dashboard" +def _active_whatsapp_account_mode() -> ExternalChatAccountMode | None: + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "cloud": + return ExternalChatAccountMode.CLOUD_SHARED + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "baileys": + return ExternalChatAccountMode.SELF_HOST_BYO + return None + + +def _is_inactive_whatsapp_account(account: ExternalChatAccount) -> bool: + return ( + account.platform == ExternalChatPlatform.WHATSAPP + and account.mode != _active_whatsapp_account_mode() + ) + + async def claim_next_inbound_event( session_maker: SessionMaker = async_session_maker, ) -> int | None: @@ -293,6 +309,11 @@ async def _dispatch_inbound_event( event.last_error = "account_missing" await session.commit() return + if _is_inactive_whatsapp_account(account): + event.status = ExternalChatEventStatus.IGNORED + event.last_error = "inactive_whatsapp_mode" + await session.commit() + return try: bundle = resolve_platform_bundle(account) diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py index a47bcd0c5..5b0f57ed3 100644 --- a/surfsense_backend/app/routes/gateway_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -16,7 +16,7 @@ from uuid import UUID import httpx from fastapi import APIRouter, Depends, HTTPException, Request from pydantic import BaseModel -from sqlalchemy import select +from sqlalchemy import or_, select from sqlalchemy.ext.asyncio import AsyncSession from starlette.responses import JSONResponse, RedirectResponse, Response @@ -173,6 +173,21 @@ class UpdateAccountSearchSpaceRequest(BaseModel): search_space_id: int +def _active_whatsapp_account_mode() -> ExternalChatAccountMode | None: + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "cloud": + return ExternalChatAccountMode.CLOUD_SHARED + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "baileys": + return ExternalChatAccountMode.SELF_HOST_BYO + return None + + +def _is_inactive_whatsapp_account(account: ExternalChatAccount) -> bool: + return ( + account.platform == ExternalChatPlatform.WHATSAPP + and account.mode != _active_whatsapp_account_mode() + ) + + def _classify_telegram_event(payload: dict[str, Any]) -> str: if "message" in payload: return "message" @@ -712,6 +727,10 @@ async def list_connections( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> list[dict[str, Any]]: + active_whatsapp_mode = _active_whatsapp_account_mode() + if platform == ExternalChatPlatform.WHATSAPP and active_whatsapp_mode is None: + return [] + filters = [ ExternalChatBinding.user_id == user.id, ExternalChatBinding.state.in_( @@ -720,6 +739,17 @@ async def list_connections( ] if platform is not None: filters.append(ExternalChatAccount.platform == platform) + if platform == ExternalChatPlatform.WHATSAPP and active_whatsapp_mode is not None: + filters.append(ExternalChatAccount.mode == active_whatsapp_mode) + elif active_whatsapp_mode is None: + filters.append(ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP) + else: + filters.append( + or_( + ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.mode == active_whatsapp_mode, + ) + ) result = await session.execute( select(ExternalChatBinding, ExternalChatAccount) @@ -782,7 +812,10 @@ async def list_connections( } ) - if platform is None or platform == ExternalChatPlatform.WHATSAPP: + if ( + active_whatsapp_mode == ExternalChatAccountMode.SELF_HOST_BYO + and (platform is None or platform == ExternalChatPlatform.WHATSAPP) + ): account_result = await session.execute( select(ExternalChatAccount).where( ExternalChatAccount.owner_user_id == user.id, @@ -855,6 +888,9 @@ async def update_binding_search_space( ExternalChatBindingState.SUSPENDED, }: raise HTTPException(status_code=400, detail="Only active bindings can be routed") + account = await session.get(ExternalChatAccount, binding.account_id) + if account is None or _is_inactive_whatsapp_account(account): + raise HTTPException(status_code=404, detail="Binding not found") await check_search_space_access(session, user, body.search_space_id) if binding.search_space_id != body.search_space_id: @@ -878,6 +914,7 @@ async def update_gateway_account_search_space( or account.owner_user_id != user.id or account.platform != ExternalChatPlatform.WHATSAPP or account.mode != ExternalChatAccountMode.SELF_HOST_BYO + or _is_inactive_whatsapp_account(account) ): raise HTTPException(status_code=404, detail="Gateway account not found") @@ -912,6 +949,9 @@ async def delete_binding( binding = await session.get(ExternalChatBinding, binding_id) if binding is None or binding.user_id != user.id: raise HTTPException(status_code=404, detail="Binding not found") + account = await session.get(ExternalChatAccount, binding.account_id) + if account is None or _is_inactive_whatsapp_account(account): + raise HTTPException(status_code=404, detail="Binding not found") revoke_binding(binding) await session.commit() return {"ok": True} @@ -929,6 +969,7 @@ async def delete_gateway_account( or account.owner_user_id != user.id or account.platform != ExternalChatPlatform.WHATSAPP or account.mode != ExternalChatAccountMode.SELF_HOST_BYO + or _is_inactive_whatsapp_account(account) ): raise HTTPException(status_code=404, detail="Gateway account not found") @@ -961,6 +1002,9 @@ async def resume_external_chat_binding( binding = await session.get(ExternalChatBinding, binding_id) if binding is None or binding.user_id != user.id: raise HTTPException(status_code=404, detail="Binding not found") + account = await session.get(ExternalChatAccount, binding.account_id) + if account is None or _is_inactive_whatsapp_account(account): + raise HTTPException(status_code=404, detail="Binding not found") resume_binding(binding) binding.updated_at = datetime.now(UTC) await session.commit() diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx index 0aa156980..9aa97c816 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx @@ -208,12 +208,18 @@ export function MessagingChannelsContent() { await refreshPlatform(connection.platform as GatewayPlatform); } + const isConnectionInActiveMode = (connection: GatewayConnection) => { + if (connection.platform !== "whatsapp") return true; + if (whatsappMode === "baileys") return connection.mode === "self_host_byo"; + if (whatsappMode === "cloud") return connection.mode !== "self_host_byo"; + return false; + }; const baileysQr = baileysHealth?.qr || null; const hasTelegramConnection = connections.some( (connection) => connection.platform === "telegram" ); const hasWhatsAppConnection = connections.some( - (connection) => connection.platform === "whatsapp" + (connection) => connection.platform === "whatsapp" && isConnectionInActiveMode(connection) ); const isRefreshing = (platform: GatewayPlatform) => refreshingPlatform === platform; const refreshButtonClassName = "gap-2"; @@ -242,7 +248,7 @@ export function MessagingChannelsContent() { `${platformLabel(connection.platform)} connection`; const renderConnectionRows = (platform: GatewayConnection["platform"], emptyText: string) => { const platformConnections = connections.filter( - (connection) => connection.platform === platform + (connection) => connection.platform === platform && isConnectionInActiveMode(connection) ); if (platformConnections.length === 0) { @@ -327,7 +333,7 @@ export function MessagingChannelsContent() { return (
- +
Telegram @@ -360,7 +366,7 @@ export function MessagingChannelsContent() { {slackGatewayEnabled ? ( - +
Slack @@ -392,7 +398,7 @@ export function MessagingChannelsContent() { ) : null} {discordGatewayEnabled ? ( - +
Discord @@ -424,16 +430,15 @@ export function MessagingChannelsContent() { ) : null} {whatsappMode !== "disabled" ? ( - +
WhatsApp

- Pair this search space with WhatsApp using the configured gateway mode. {whatsappMode === "baileys" - ? " Send messages to your own WhatsApp chat. Other chats are ignored." - : ""} + ? "Use the WhatsApp bridge for your own WhatsApp chat. Other chats are ignored." + : "Pair this search space with WhatsApp Cloud API."}

@@ -469,7 +474,7 @@ export function MessagingChannelsContent() { disabled={isRefreshing("whatsapp")} > - Refresh WhatsApp Bridge + Refresh {baileysQr ? (
From 799a83239f40ded5d731bb417b986078e18833d3 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:58:28 +0530 Subject: [PATCH 58/63] feat(gateway): add Slack and Telegram gateway configuration and enablement checks --- docker/docker-compose.yml | 1 - surfsense_backend/.env.example | 5 +- surfsense_backend/app/config/__init__.py | 1 + .../app/routes/gateway_webhook_routes.py | 79 ++++++++-- surfsense_web/.env.example | 8 -- .../components/MessagingChannelsContent.tsx | 136 +++++++++++++----- 6 files changed, 168 insertions(+), 62 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 66ad55b77..11f4fdb5c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -285,7 +285,6 @@ services: NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${AUTH_TYPE:-LOCAL} NEXT_PUBLIC_ETL_SERVICE: ${ETL_SERVICE:-DOCLING} NEXT_PUBLIC_DEPLOYMENT_MODE: ${DEPLOYMENT_MODE:-self-hosted} - NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE: ${GATEWAY_WHATSAPP_INTAKE_MODE:-disabled} NEXT_PUBLIC_WHATSAPP_DISPLAY_PHONE_NUMBER: ${WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER:-} FASTAPI_BACKEND_INTERNAL_URL: ${FASTAPI_BACKEND_INTERNAL_URL:-http://backend:8000} labels: diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 170bece32..808d29051 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -17,7 +17,7 @@ REDIS_APP_URL=redis://localhost:6379/0 # Telegram Gateway # TELEGRAM_WEBHOOK_SECRET must be 1-256 chars and contain only A-Z, a-z, 0-9, _ or - -# GATEWAY_TELEGRAM_INTAKE_MODE: webhook for production, longpoll for single-replica self-host fallback, disabled to skip Telegram intake +# GATEWAY_TELEGRAM_INTAKE_MODE: `webhook` for production, `longpoll` for single-replica self-host fallback, `disabled` to skip Telegram intake TELEGRAM_SHARED_BOT_TOKEN= TELEGRAM_SHARED_BOT_USERNAME= TELEGRAM_WEBHOOK_SECRET= @@ -25,7 +25,7 @@ GATEWAY_BASE_URL=http://localhost:8000 GATEWAY_TELEGRAM_INTAKE_MODE=webhook # WhatsApp Gateway -# GATEWAY_WHATSAPP_INTAKE_MODE: cloud for Meta Cloud API, baileys for self-hosted bridge, disabled to skip WhatsApp intake +# GATEWAY_WHATSAPP_INTAKE_MODE: `cloud` for Meta Cloud API, `baileys` for self-hosted bridge, `disabled` to skip WhatsApp intake GATEWAY_WHATSAPP_INTAKE_MODE=disabled WHATSAPP_SHARED_BUSINESS_TOKEN= WHATSAPP_SHARED_PHONE_NUMBER_ID= @@ -149,6 +149,7 @@ NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback SLACK_CLIENT_ID=your_slack_client_id_here SLACK_CLIENT_SECRET=your_slack_client_secret_here SLACK_REDIRECT_URI=http://localhost:8000/api/v1/auth/slack/connector/callback +GATEWAY_SLACK_ENABLED=FALSE GATEWAY_SLACK_SIGNING_SECRET=your_slack_signing_secret_here GATEWAY_SLACK_REDIRECT_URI=http://localhost:8000/api/v1/gateway/slack/callback diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 98c1d5dec..f3c05f2d6 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -572,6 +572,7 @@ class Config: ) GATEWAY_SLACK_CLIENT_ID = os.getenv("SLACK_CLIENT_ID") GATEWAY_SLACK_CLIENT_SECRET = os.getenv("SLACK_CLIENT_SECRET") + GATEWAY_SLACK_ENABLED = os.getenv("GATEWAY_SLACK_ENABLED", "FALSE").upper() == "TRUE" GATEWAY_SLACK_SIGNING_SECRET = os.getenv("GATEWAY_SLACK_SIGNING_SECRET") GATEWAY_SLACK_REDIRECT_URI = os.getenv("GATEWAY_SLACK_REDIRECT_URI") GATEWAY_DISCORD_ENABLED = ( diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py index 5b0f57ed3..9c890b610 100644 --- a/surfsense_backend/app/routes/gateway_webhook_routes.py +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -188,6 +188,36 @@ def _is_inactive_whatsapp_account(account: ExternalChatAccount) -> bool: ) +def _telegram_gateway_enabled() -> bool: + return ( + config.GATEWAY_TELEGRAM_INTAKE_MODE != "disabled" + and bool(config.TELEGRAM_SHARED_BOT_TOKEN) + and bool(config.TELEGRAM_SHARED_BOT_USERNAME) + and ( + config.GATEWAY_TELEGRAM_INTAKE_MODE != "webhook" + or bool(config.TELEGRAM_WEBHOOK_SECRET) + ) + ) + + +def _slack_gateway_enabled() -> bool: + return bool( + config.GATEWAY_SLACK_ENABLED + and config.GATEWAY_SLACK_CLIENT_ID + and config.GATEWAY_SLACK_CLIENT_SECRET + and config.GATEWAY_SLACK_SIGNING_SECRET + ) + + +def _discord_gateway_enabled() -> bool: + return bool( + config.GATEWAY_DISCORD_ENABLED + and config.DISCORD_CLIENT_ID + and config.DISCORD_CLIENT_SECRET + and config.DISCORD_BOT_TOKEN + ) + + def _classify_telegram_event(payload: dict[str, Any]) -> str: if "message" in payload: return "message" @@ -208,7 +238,7 @@ async def install_slack_gateway( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> dict[str, str]: - if not config.GATEWAY_SLACK_CLIENT_ID: + if not _slack_gateway_enabled(): raise HTTPException(status_code=500, detail="Slack gateway OAuth is not configured") await check_search_space_access(session, user, search_space_id) state = _get_state_manager().generate_secure_state(search_space_id, user.id) @@ -242,7 +272,7 @@ async def slack_gateway_callback( return _slack_frontend_redirect(space_id or 0, error="slack_gateway_oauth_denied") if not code or state_data is None: raise HTTPException(status_code=400, detail="Invalid Slack gateway OAuth callback") - if not config.GATEWAY_SLACK_CLIENT_ID or not config.GATEWAY_SLACK_CLIENT_SECRET: + if not _slack_gateway_enabled(): raise HTTPException(status_code=500, detail="Slack gateway OAuth is not configured") user_id = UUID(state_data["user_id"]) @@ -357,7 +387,7 @@ async def install_discord_gateway( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> dict[str, str]: - if not config.DISCORD_CLIENT_ID: + if not _discord_gateway_enabled(): raise HTTPException(status_code=500, detail="Discord gateway OAuth is not configured") await check_search_space_access(session, user, search_space_id) state = _get_state_manager().generate_secure_state(search_space_id, user.id) @@ -393,10 +423,8 @@ async def discord_gateway_callback( return _discord_frontend_redirect(space_id or 0, error="discord_gateway_oauth_denied") if not code or state_data is None: raise HTTPException(status_code=400, detail="Invalid Discord gateway OAuth callback") - if not config.DISCORD_CLIENT_ID or not config.DISCORD_CLIENT_SECRET: + if not _discord_gateway_enabled(): raise HTTPException(status_code=500, detail="Discord gateway OAuth is not configured") - if not config.DISCORD_BOT_TOKEN: - raise HTTPException(status_code=500, detail="Discord gateway bot token is not configured") user_id = UUID(state_data["user_id"]) token_payload = { @@ -518,6 +546,9 @@ async def slack_webhook( request: Request, session: AsyncSession = Depends(get_async_session), ) -> Response: + if not _slack_gateway_enabled(): + return Response(status_code=200) + body = await request.body() if not verify_slack_signature( signing_secret=config.GATEWAY_SLACK_SIGNING_SECRET or "", @@ -594,6 +625,9 @@ async def telegram_webhook( account_id: int, session: AsyncSession = Depends(get_async_session), ) -> Response: + if not _telegram_gateway_enabled(): + return Response(status_code=200) + request_id = f"gateway_{uuid.uuid4().hex[:16]}" try: payload = await request.json() @@ -644,6 +678,8 @@ async def start_binding( await check_search_space_access(session, user, body.search_space_id) code = generate_pairing_code() if body.platform == ExternalChatPlatform.TELEGRAM: + if not _telegram_gateway_enabled(): + raise HTTPException(status_code=400, detail="Telegram gateway is disabled") account = await get_or_create_system_telegram_account(session) username = account.bot_username or config.TELEGRAM_SHARED_BOT_USERNAME if not username: @@ -730,6 +766,8 @@ async def list_connections( active_whatsapp_mode = _active_whatsapp_account_mode() if platform == ExternalChatPlatform.WHATSAPP and active_whatsapp_mode is None: return [] + if platform == ExternalChatPlatform.TELEGRAM and not _telegram_gateway_enabled(): + return [] filters = [ ExternalChatBinding.user_id == user.id, @@ -741,15 +779,18 @@ async def list_connections( filters.append(ExternalChatAccount.platform == platform) if platform == ExternalChatPlatform.WHATSAPP and active_whatsapp_mode is not None: filters.append(ExternalChatAccount.mode == active_whatsapp_mode) - elif active_whatsapp_mode is None: - filters.append(ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP) else: - filters.append( - or_( - ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP, - ExternalChatAccount.mode == active_whatsapp_mode, + if not _telegram_gateway_enabled(): + filters.append(ExternalChatAccount.platform != ExternalChatPlatform.TELEGRAM) + if active_whatsapp_mode is None: + filters.append(ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP) + else: + filters.append( + or_( + ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.mode == active_whatsapp_mode, + ) ) - ) result = await session.execute( select(ExternalChatBinding, ExternalChatAccount) @@ -873,6 +914,18 @@ async def list_platforms( ] +@router.get("/config") +async def get_gateway_config( + user: User = Depends(current_active_user), +) -> dict[str, bool | str]: + return { + "telegram_enabled": _telegram_gateway_enabled(), + "whatsapp_intake_mode": config.GATEWAY_WHATSAPP_INTAKE_MODE, + "slack_enabled": _slack_gateway_enabled(), + "discord_enabled": _discord_gateway_enabled(), + } + + @router.patch("/bindings/{binding_id}/search-space") async def update_binding_search_space( binding_id: int, diff --git a/surfsense_web/.env.example b/surfsense_web/.env.example index e4aaf91d7..5fb9d07d1 100644 --- a/surfsense_web/.env.example +++ b/surfsense_web/.env.example @@ -7,14 +7,6 @@ NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL or GOOGLE NEXT_PUBLIC_ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848 -# Messaging gateway options -# WhatsApp UI toggle: disabled, cloud, or baileys -NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE=disabled -# Slack gateway UI toggle: true or false -NEXT_PUBLIC_GATEWAY_SLACK_ENABLED=false -# Discord gateway UI toggle: true or false -NEXT_PUBLIC_GATEWAY_DISCORD_ENABLED=false - # Contact Form Vars (optional) DATABASE_URL=postgresql://postgres:[YOUR-PASSWORD]@db.sdsf.supabase.co:5432/postgres diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx index 9aa97c816..b0cb6699c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MessagingChannelsContent.tsx @@ -15,6 +15,7 @@ import { SelectValue, } from "@/components/ui/select"; import { Separator } from "@/components/ui/separator"; +import { Skeleton } from "@/components/ui/skeleton"; import type { SearchSpace } from "@/contracts/types/search-space.types"; import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service"; import { authenticatedFetch } from "@/lib/auth-utils"; @@ -37,6 +38,15 @@ type GatewayConnection = { suspended_reason?: string | null; }; +type GatewayConfig = { + telegram_enabled: boolean; + whatsapp_intake_mode: "disabled" | "cloud" | "baileys"; + slack_enabled: boolean; + discord_enabled: boolean; +}; + +type GatewayConfigState = GatewayConfig | null; + type Pairing = { binding_id: number; code: string; @@ -58,15 +68,18 @@ type BaileysHealth = { export function MessagingChannelsContent() { const params = useParams<{ search_space_id: string }>(); const searchSpaceId = Number(params.search_space_id); - const whatsappMode = process.env.NEXT_PUBLIC_GATEWAY_WHATSAPP_INTAKE_MODE ?? "disabled"; - const slackGatewayEnabled = process.env.NEXT_PUBLIC_GATEWAY_SLACK_ENABLED === "true"; - const discordGatewayEnabled = process.env.NEXT_PUBLIC_GATEWAY_DISCORD_ENABLED === "true"; + const [gatewayConfig, setGatewayConfig] = useState(null); const [connections, setConnections] = useState([]); const [searchSpaces, setSearchSpaces] = useState([]); const [pairing, setPairing] = useState(null); const [pairingPlatform, setPairingPlatform] = useState(null); const [baileysHealth, setBaileysHealth] = useState(null); const [refreshingPlatform, setRefreshingPlatform] = useState(null); + const isGatewayConfigLoading = gatewayConfig === null; + const telegramGatewayEnabled = gatewayConfig?.telegram_enabled ?? false; + const whatsappMode = gatewayConfig?.whatsapp_intake_mode ?? "disabled"; + const slackGatewayEnabled = gatewayConfig?.slack_enabled ?? false; + const discordGatewayEnabled = gatewayConfig?.discord_enabled ?? false; const fetchConnections = useCallback(async (platform?: GatewayPlatform) => { const query = platform ? `?platform=${encodeURIComponent(platform)}` : ""; @@ -74,14 +87,21 @@ export function MessagingChannelsContent() { return (await res.json()) as GatewayConnection[]; }, []); + const fetchGatewayConfig = useCallback(async () => { + const res = await authenticatedFetch(`${BACKEND_URL}/api/v1/gateway/config`); + return (await res.json()) as GatewayConfig; + }, []); + const refresh = useCallback(async () => { - const [nextConnections, spaces] = await Promise.all([ + const [nextConnections, spaces, nextGatewayConfig] = await Promise.all([ fetchConnections(), searchSpacesApiService.getSearchSpaces(), + fetchGatewayConfig(), ]); setConnections(nextConnections); setSearchSpaces(spaces); - }, [fetchConnections]); + setGatewayConfig(nextGatewayConfig); + }, [fetchConnections, fetchGatewayConfig]); useEffect(() => { void refresh(); @@ -221,6 +241,11 @@ export function MessagingChannelsContent() { const hasWhatsAppConnection = connections.some( (connection) => connection.platform === "whatsapp" && isConnectionInActiveMode(connection) ); + const hasEnabledGateway = + telegramGatewayEnabled || + whatsappMode !== "disabled" || + slackGatewayEnabled || + discordGatewayEnabled; const isRefreshing = (platform: GatewayPlatform) => refreshingPlatform === platform; const refreshButtonClassName = "gap-2"; const refreshIconClassName = (platform: GatewayPlatform) => @@ -252,7 +277,11 @@ export function MessagingChannelsContent() { ); if (platformConnections.length === 0) { - return

{emptyText}

; + return ( +
+

{emptyText}

+
+ ); } return ( @@ -330,40 +359,71 @@ export function MessagingChannelsContent() {
); }; + const renderGatewaySkeletons = () => ( + <> + {[0, 1].map((index) => ( + + + + + + + + + + + + ))} + + ); return (
- - -
- Telegram -
-

Pair Telegram with this search space.

- - -
- {hasTelegramConnection ? null : ( - - )} - -
+ {isGatewayConfigLoading ? renderGatewaySkeletons() : null} - {hasTelegramConnection ? null : renderPairingPanel("telegram")} - - {renderConnectionRows("telegram", "No Telegram chats connected yet.")} - - + {!isGatewayConfigLoading && !hasEnabledGateway ? ( + + + No messaging gateways enabled + + + ) : null} + + {telegramGatewayEnabled ? ( + + +
+ Telegram +
+

+ Connect Telegram to chat with SurfSense. +

+ + +
+ {hasTelegramConnection ? null : ( + + )} + +
+ + {hasTelegramConnection ? null : renderPairingPanel("telegram")} + + {renderConnectionRows("telegram", "No Telegram chats connected yet.")} + + + ) : null} {slackGatewayEnabled ? ( @@ -437,8 +497,8 @@ export function MessagingChannelsContent() {

{whatsappMode === "baileys" - ? "Use the WhatsApp bridge for your own WhatsApp chat. Other chats are ignored." - : "Pair this search space with WhatsApp Cloud API."} + ? 'Use "Message Yourself". Other chats are ignored.' + : "Connect WhatsApp to chat with Surfsense."}

From 20994671bc63978cec102b03507057d21209b3a2 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 2 Jun 2026 00:03:26 +0530 Subject: [PATCH 59/63] feat(gateway): add configuration options for Telegram, WhatsApp, Slack, and Discord gateways --- docker/.env.example | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docker/.env.example b/docker/.env.example index 39fd8989b..96152c129 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -265,6 +265,43 @@ STT_SERVICE=local/base # COMPOSIO_ENABLED=TRUE # COMPOSIO_REDIRECT_URI=http://localhost:8000/api/v1/auth/composio/connector/callback +# ------------------------------------------------------------------------------ +# Messaging Gateways (optional) +# ------------------------------------------------------------------------------ +# Configure only the gateways you want to use. + +# -- Telegram Gateway -- +# TELEGRAM_SHARED_BOT_TOKEN= +# TELEGRAM_SHARED_BOT_USERNAME= +# TELEGRAM_WEBHOOK_SECRET= +# GATEWAY_BASE_URL=http://localhost:8929 +# GATEWAY_TELEGRAM_INTAKE_MODE=webhook + +# -- WhatsApp Gateway -- +# GATEWAY_WHATSAPP_INTAKE_MODE=disabled +# WHATSAPP_SHARED_BUSINESS_TOKEN= +# WHATSAPP_SHARED_PHONE_NUMBER_ID= +# WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER= +# WHATSAPP_SHARED_WABA_ID= +# WHATSAPP_GRAPH_API_VERSION=v25.0 +# WHATSAPP_WEBHOOK_VERIFY_TOKEN= +# WHATSAPP_WEBHOOK_APP_SECRET= +# WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:9929 + +# -- Slack Gateway -- +# Uses SLACK_CLIENT_ID and SLACK_CLIENT_SECRET from the Slack connector section. +# +# GATEWAY_SLACK_ENABLED=FALSE +# GATEWAY_SLACK_SIGNING_SECRET= +# GATEWAY_SLACK_REDIRECT_URI=http://localhost:8929/api/v1/gateway/slack/callback + +# -- Discord Gateway -- +# Uses DISCORD_CLIENT_ID, DISCORD_CLIENT_SECRET, and DISCORD_BOT_TOKEN from the +# Discord connector section. +# +# GATEWAY_DISCORD_ENABLED=FALSE +# GATEWAY_DISCORD_REDIRECT_URI=http://localhost:8929/api/v1/gateway/discord/callback + # ------------------------------------------------------------------------------ # SearXNG (bundled web search — works out of the box, no config needed) # ------------------------------------------------------------------------------ From ebddf4506aa19f58266afb73cab4cb3a859f0734 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 2 Jun 2026 00:25:49 +0530 Subject: [PATCH 60/63] feat(messaging): introduce comprehensive setup docs for Telegram, WhatsApp, Slack, and Discord messaging channels --- docker/.env.example | 32 +++---- .../docker-installation/docker-compose.mdx | 21 ++++- surfsense_web/content/docs/index.mdx | 8 +- .../content/docs/manual-installation.mdx | 42 ++-------- .../docs/messaging-channels/discord.mdx | 76 +++++++++++++++++ .../docs/messaging-channels/docker.mdx | 60 +++++++++++++ .../content/docs/messaging-channels/index.mdx | 42 ++++++++++ .../content/docs/messaging-channels/meta.json | 13 +++ .../content/docs/messaging-channels/slack.mdx | 84 +++++++++++++++++++ .../docs/messaging-channels/telegram.mdx | 62 ++++++++++++++ .../messaging-channels/troubleshooting.mdx | 66 +++++++++++++++ .../docs/messaging-channels/whatsapp.mdx | 75 +++++++++++++++++ surfsense_web/content/docs/meta.json | 1 + surfsense_web/lib/source.ts | 2 + 14 files changed, 530 insertions(+), 54 deletions(-) create mode 100644 surfsense_web/content/docs/messaging-channels/discord.mdx create mode 100644 surfsense_web/content/docs/messaging-channels/docker.mdx create mode 100644 surfsense_web/content/docs/messaging-channels/index.mdx create mode 100644 surfsense_web/content/docs/messaging-channels/meta.json create mode 100644 surfsense_web/content/docs/messaging-channels/slack.mdx create mode 100644 surfsense_web/content/docs/messaging-channels/telegram.mdx create mode 100644 surfsense_web/content/docs/messaging-channels/troubleshooting.mdx create mode 100644 surfsense_web/content/docs/messaging-channels/whatsapp.mdx diff --git a/docker/.env.example b/docker/.env.example index 96152c129..12c5dcc55 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -70,7 +70,7 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # ------------------------------------------------------------------------------ # ONLY set these if you are serving SurfSense on a real domain via a reverse # proxy (e.g. Caddy, Nginx, Cloudflare Tunnel). -# For standard localhost deployments, leave all of these commented out — +# For standard localhost deployments, leave all of these commented out. # they are automatically derived from the port settings above. # # NEXT_FRONTEND_URL=https://app.yourdomain.com @@ -92,7 +92,7 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # Only change this if you manage publications manually. # ZERO_APP_PUBLICATIONS=zero_publication -# Sync worker tuning — zero-cache defaults ZERO_NUM_SYNC_WORKERS to the number +# Sync worker tuning. zero-cache defaults ZERO_NUM_SYNC_WORKERS to the number # of CPU cores, which can exceed the connection pool limits on high-core machines. # Each sync worker needs at least 1 connection from both the UPSTREAM and CVR # pools, so these constraints must hold: @@ -137,7 +137,7 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # SSL mode for database connections: disable, require, verify-ca, verify-full # DB_SSLMODE=disable -# Full DATABASE_URL override — when set, takes precedence over the individual +# Full DATABASE_URL override. When set, this takes precedence over the individual # DB_USER / DB_PASSWORD / DB_NAME / DB_HOST / DB_PORT settings above. # Use this for managed databases (AWS RDS, GCP Cloud SQL, Supabase, etc.) # DATABASE_URL=postgresql+asyncpg://user:password@your-rds-host:5432/surfsense?sslmode=require @@ -152,7 +152,7 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # REDIS_URL=redis://redis:6379/0 # ------------------------------------------------------------------------------ -# Stripe (pay-as-you-go page packs — disabled by default) +# Stripe (pay-as-you-go page packs, disabled by default) # ------------------------------------------------------------------------------ # Set TRUE to allow users to buy additional page packs via Stripe Checkout @@ -171,7 +171,7 @@ STRIPE_PAGE_BUYING_ENABLED=FALSE # STRIPE_TOKEN_BUYING_ENABLED=FALSE # STRIPE_PREMIUM_TOKEN_PRICE_ID=price_... # STRIPE_CREDIT_MICROS_PER_UNIT=1000000 -# DEPRECATED — STRIPE_TOKENS_PER_UNIT=1000000 +# DEPRECATED: STRIPE_TOKENS_PER_UNIT=1000000 # ------------------------------------------------------------------------------ # TTS & STT (Text-to-Speech / Speech-to-Text) @@ -266,18 +266,18 @@ STT_SERVICE=local/base # COMPOSIO_REDIRECT_URI=http://localhost:8000/api/v1/auth/composio/connector/callback # ------------------------------------------------------------------------------ -# Messaging Gateways (optional) +# Messaging Channels (optional) # ------------------------------------------------------------------------------ -# Configure only the gateways you want to use. +# Configure only the external chat channels you want to use. -# -- Telegram Gateway -- +# -- Telegram -- # TELEGRAM_SHARED_BOT_TOKEN= # TELEGRAM_SHARED_BOT_USERNAME= # TELEGRAM_WEBHOOK_SECRET= # GATEWAY_BASE_URL=http://localhost:8929 # GATEWAY_TELEGRAM_INTAKE_MODE=webhook -# -- WhatsApp Gateway -- +# -- WhatsApp -- # GATEWAY_WHATSAPP_INTAKE_MODE=disabled # WHATSAPP_SHARED_BUSINESS_TOKEN= # WHATSAPP_SHARED_PHONE_NUMBER_ID= @@ -288,14 +288,14 @@ STT_SERVICE=local/base # WHATSAPP_WEBHOOK_APP_SECRET= # WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:9929 -# -- Slack Gateway -- +# -- Slack -- # Uses SLACK_CLIENT_ID and SLACK_CLIENT_SECRET from the Slack connector section. # # GATEWAY_SLACK_ENABLED=FALSE # GATEWAY_SLACK_SIGNING_SECRET= # GATEWAY_SLACK_REDIRECT_URI=http://localhost:8929/api/v1/gateway/slack/callback -# -- Discord Gateway -- +# -- Discord -- # Uses DISCORD_CLIENT_ID, DISCORD_CLIENT_SECRET, and DISCORD_BOT_TOKEN from the # Discord connector section. # @@ -303,7 +303,7 @@ STT_SERVICE=local/base # GATEWAY_DISCORD_REDIRECT_URI=http://localhost:8929/api/v1/gateway/discord/callback # ------------------------------------------------------------------------------ -# SearXNG (bundled web search — works out of the box, no config needed) +# SearXNG (bundled web search, works out of the box with no config needed) # ------------------------------------------------------------------------------ # SearXNG provides web search to all search spaces automatically. # To access the SearXNG UI directly: http://localhost:8888 @@ -313,7 +313,7 @@ STT_SERVICE=local/base # SEARXNG_SECRET=surfsense-searxng-secret # ------------------------------------------------------------------------------ -# Daytona Sandbox (optional — cloud code execution for the deep agent) +# Daytona Sandbox (optional cloud code execution for the deep agent) # ------------------------------------------------------------------------------ # Set DAYTONA_SANDBOX_ENABLED=TRUE and provide credentials to give the agent # an isolated code execution environment via the Daytona cloud API. @@ -404,7 +404,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # Premium turns are debited at the actual per-call provider cost reported # by LiteLLM. Only applies to models with billing_tier=premium. # PREMIUM_CREDIT_MICROS_LIMIT=5000000 -# DEPRECATED — PREMIUM_TOKEN_LIMIT=5000000 +# DEPRECATED: PREMIUM_TOKEN_LIMIT=5000000 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 @@ -416,10 +416,10 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # QUOTA_DEFAULT_PODCAST_RESERVE_MICROS=200000 # Per-video-presentation reservation for the video Celery task ($1.00 default). -# Override path bypasses QUOTA_MAX_RESERVE_MICROS clamp — raise with care. +# Override path bypasses QUOTA_MAX_RESERVE_MICROS clamp. Raise with care. # QUOTA_DEFAULT_VIDEO_PRESENTATION_RESERVE_MICROS=1000000 -# No-login (anonymous) mode — public users can chat without an account +# No-login (anonymous) mode. Public users can chat without an account # Set TRUE to enable /free pages and anonymous chat API NOLOGIN_MODE_ENABLED=FALSE # ANON_TOKEN_LIMIT=1000000 diff --git a/surfsense_web/content/docs/docker-installation/docker-compose.mdx b/surfsense_web/content/docs/docker-installation/docker-compose.mdx index 0155969cd..488f4e24a 100644 --- a/surfsense_web/content/docs/docker-installation/docker-compose.mdx +++ b/surfsense_web/content/docs/docker-installation/docker-compose.mdx @@ -124,6 +124,19 @@ Uncomment the connectors you want to use. Redirect URIs follow the pattern `http | Microsoft (Teams & OneDrive) | `MICROSOFT_CLIENT_ID`, `MICROSOFT_CLIENT_SECRET`, `TEAMS_REDIRECT_URI`, `ONEDRIVE_REDIRECT_URI` | | Dropbox | `DROPBOX_APP_KEY`, `DROPBOX_APP_SECRET`, `DROPBOX_REDIRECT_URI` | +### Messaging Channels + +Configure these in the same `docker/.env` file when you want users to chat with +SurfSense from external apps. See [Messaging Channels](/docs/messaging-channels) +for full setup. + +| Channel | Variables | +|---------|-----------| +| Telegram | `TELEGRAM_SHARED_BOT_TOKEN`, `TELEGRAM_SHARED_BOT_USERNAME`, `TELEGRAM_WEBHOOK_SECRET`, `GATEWAY_BASE_URL`, `GATEWAY_TELEGRAM_INTAKE_MODE` | +| WhatsApp | `GATEWAY_WHATSAPP_INTAKE_MODE`, `WHATSAPP_SHARED_BUSINESS_TOKEN`, `WHATSAPP_SHARED_PHONE_NUMBER_ID`, `WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER`, `WHATSAPP_SHARED_WABA_ID`, `WHATSAPP_WEBHOOK_VERIFY_TOKEN`, `WHATSAPP_WEBHOOK_APP_SECRET` | +| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `GATEWAY_SLACK_ENABLED`, `GATEWAY_SLACK_SIGNING_SECRET`, `GATEWAY_SLACK_REDIRECT_URI` | +| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `GATEWAY_DISCORD_ENABLED`, `GATEWAY_DISCORD_REDIRECT_URI` | + ### Observability (optional) | Variable | Description | @@ -187,9 +200,9 @@ Postgres. Before this design, a silent migration failure would leave The backend exposes two endpoints: -- `GET /health` — lightweight liveness probe (always returns 200 if the +- `GET /health`: lightweight liveness probe (always returns 200 if the process is up). -- `GET /ready` — readiness probe that confirms `zero_publication` exists. +- `GET /ready`: readiness probe that confirms `zero_publication` exists. Returns 503 if not. The compose `backend.healthcheck` uses `/ready` so the container only reports `healthy` once the schema is actually usable by zero-cache. @@ -247,7 +260,7 @@ docker compose exec db psql -U surfsense -d surfsense \ ``` The default migration timeout is 900 seconds. Slow disks (Windows / WSL2) -may need more — set `MIGRATION_TIMEOUT` in `.env` to increase it. +may need more. Set `MIGRATION_TIMEOUT` in `.env` to increase it. ### Zero-cache stuck on `Unknown or invalid publications` @@ -258,7 +271,7 @@ Error: Unknown or invalid publications. Specified: [zero_publication]. Found: [] ``` This means `zero-cache` started before `zero_publication` was created. With -the current compose files this should be impossible — the `migrations` +the current compose files this should be impossible. The `migrations` service blocks `zero-cache` from starting. If you see it, your stack predates the fix or you brought up `zero-cache` manually with `docker compose up zero-cache` before the migrations service ran. diff --git a/surfsense_web/content/docs/index.mdx b/surfsense_web/content/docs/index.mdx index 2204e4e34..4a321b376 100644 --- a/surfsense_web/content/docs/index.mdx +++ b/surfsense_web/content/docs/index.mdx @@ -5,7 +5,7 @@ icon: BookOpen --- import { Card, Cards } from 'fumadocs-ui/components/card'; -import { ClipboardCheck, Download, Container, Wrench, Cable, BookOpen, FlaskConical, Heart } from 'lucide-react'; +import { ClipboardCheck, Download, Container, Wrench, Cable, BookOpen, FlaskConical, Heart, MessageCircle } from 'lucide-react'; Welcome to **SurfSense's Documentation!** Here, you'll find everything you need to get the most out of SurfSense. Dive in to explore how SurfSense can be your AI-powered research companion. @@ -40,6 +40,12 @@ Welcome to **SurfSense's Documentation!** Here, you'll find everything you need description="Integrate with third-party services" href="/docs/connectors" /> + } + title="Messaging Channels" + description="Chat with SurfSense from Telegram, WhatsApp, Slack, and Discord" + href="/docs/messaging-channels" + /> } title="How-To Guides" diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index 602c9a30b..203c244c0 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -39,38 +39,14 @@ Complete all the [setup steps](/docs), including: The backend is the core of SurfSense. Follow these steps to set it up: -### Optional: Telegram External Chat Surface +### Optional: Messaging Channels -SurfSense can expose the same canonical chat agent through Telegram. The `external_chat_*` tables store adapter identity, delivery configuration, and durable inbox rows. The actual chat thread and messages remain in `new_chat_threads` and `new_chat_messages`, with first-party web/desktop chats marked as `source="surfsense"` and external surfaces marked by platform, such as `source="telegram"`. All chat-message sources are eligible for Zero replication so a future SurfSense UI layer can render external chat surfaces. The web app initially shows pairing, health, revoke, and resume controls under **User Settings > Messaging Channels**. +SurfSense can expose the same backend agent through Telegram, WhatsApp, Slack, +and Discord. For manual installs, configure the relevant channel variables in +`surfsense_backend/.env`. -Add these variables to `surfsense_backend/.env` when enabling the Telegram surface: - -```bash -TELEGRAM_SHARED_BOT_TOKEN=123456:bot-token-from-botfather -TELEGRAM_SHARED_BOT_USERNAME=your_bot_username -TELEGRAM_WEBHOOK_SECRET=generate-a-long-random-secret -GATEWAY_BASE_URL=https://api.example.com -GATEWAY_TELEGRAM_INTAKE_MODE=webhook -``` - -`GATEWAY_TELEGRAM_INTAKE_MODE` must be `webhook`, `longpoll`, or `disabled`. Use `webhook` for production/SaaS deployments, `longpoll` only for single-replica self-host installs that cannot expose a public HTTPS webhook, and `disabled` to skip Telegram intake. `TELEGRAM_WEBHOOK_SECRET` must use only `A-Z`, `a-z`, `0-9`, `_`, and `-` characters. `REDIS_APP_URL` is reused for external chat rate limits and per-thread locks. The webhook URL shape is: - -```text -${GATEWAY_BASE_URL}/api/v1/gateway/webhooks/telegram/{account_id} -``` - -After deploying the backend, register the webhook: - -```bash -cd surfsense_backend -uv run python scripts/register_webhook.py -``` - -Keep the FastAPI backend, Celery worker, and Celery beat running. Telegram webhooks write inbound updates into `external_chat_inbound_events`. The FastAPI process owns external chat inbox processing and runs the same SurfSense agent used by web UI chats, then replies back to Telegram. Celery remains maintenance-only for external chat reconciliation, health checks, and retention sweeps. There is no separate gateway service, `SERVICE_ROLE=gateway` process, or Celery agent-processing path. - -For self-hosted BYO Telegram bots without a public HTTPS URL, set `GATEWAY_TELEGRAM_INTAKE_MODE=longpoll`. The FastAPI process starts one lifespan long-poll supervisor per non-system Telegram account and writes updates into the same durable inbox. The FastAPI inbox worker then processes those rows in-process through the canonical `new_chat_*` surface. This fallback is intended for single-replica self-hosted installs. For SaaS-style multi-replica deployments, prefer public webhooks and keep `GATEWAY_TELEGRAM_INTAKE_MODE=webhook` so API replicas skip BYO polling entirely. Telegram does not allow `get_updates()` while a webhook is active, so delete any existing webhook for a BYO bot before relying on long polling. - -When upgrading from an older gateway-runner deployment, apply the rewritten migration 144 external chat schema, deploy the new backend, worker, and beat images, then stop the old `gateway` service. Wait about 30 seconds for any old Telegram `getUpdates` long-poll request to release its advisory lock before starting the new API process. Register each webhook again with the account-id URL above and the per-account `webhook_secret`. If you roll back before using the migration in production, restore the old image and downgrade the schema first. +See [Messaging Channels](/docs/messaging-channels) for the channel-specific +setup guides. ### 1. Environment Configuration @@ -490,7 +466,7 @@ If everything is set up correctly, you should see output indicating the server i ## Zero-Cache Setup -**zero-cache** is the Rocicorp Zero server that sits between PostgreSQL and the browser. It streams real-time updates (notifications, document indexing status, chat comments, collaboration indicators) to all connected clients via WebSocket. The frontend connects to it on startup — without zero-cache running, you will not see live updates and many parts of the UI will sit on stale data. +**zero-cache** is the Rocicorp Zero server that sits between PostgreSQL and the browser. It streams real-time updates (notifications, document indexing status, chat comments, collaboration indicators) to all connected clients via WebSocket. The frontend connects to it on startup. Without zero-cache running, you will not see live updates and many parts of the UI will sit on stale data. For an overview of how Zero works and the list of synced tables, see the [Real-Time Sync with Zero](/docs/how-to/zero-sync) guide. @@ -572,7 +548,7 @@ cd ../docker docker compose -f docker-compose.deps-only.yml up -d ``` -The deps-only stack exposes zero-cache on port `4848` (default) — keep `NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848` in your `surfsense_web/.env`. +The deps-only stack exposes zero-cache on port `4848` by default. Keep `NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848` in your `surfsense_web/.env`. ## Frontend Setup @@ -708,7 +684,7 @@ To verify your installation: 1. Open your browser and navigate to `http://localhost:3000` 2. Sign in with your Google account (or local credentials if `AUTH_TYPE=LOCAL`) 3. Create a search space and try uploading a document -4. Watch the upload status update live without refreshing — this confirms zero-cache is wired up correctly +4. Watch the upload status update live without refreshing. This confirms zero-cache is wired up correctly 5. Test the chat functionality with your uploaded content ## Troubleshooting diff --git a/surfsense_web/content/docs/messaging-channels/discord.mdx b/surfsense_web/content/docs/messaging-channels/discord.mdx new file mode 100644 index 000000000..c0874dfe3 --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/discord.mdx @@ -0,0 +1,76 @@ +--- +title: Discord +description: Enable the SurfSense bot for in-Discord agent chat +--- + +# Discord Messaging Channel + +The Discord messaging channel lets users mention the SurfSense bot in Discord +and chat with the SurfSense backend agent from a Discord channel. + +This is separate from the Discord connector. The messaging channel handles bot +mentions and replies; the connector gives the agent Discord channel/message read +tools. + +## Discord Application Settings + +Create or reuse a Discord application in the +[Discord Developer Portal](https://discord.com/developers/applications). + +In **OAuth2 > Redirects**, add both callback URLs if the same application powers +the connector and messaging channel: + +```bash +https://your-backend.example.com/api/v1/auth/discord/connector/callback +https://your-backend.example.com/api/v1/gateway/discord/callback +``` + +For local OAuth testing, replace the host with your local or public tunnel URL, +and make sure `DISCORD_REDIRECT_URI` and `GATEWAY_DISCORD_REDIRECT_URI` match +the Discord dashboard exactly. + +## Bot Permissions And Intents + +In **Bot > Privileged Gateway Intents**, enable: + +- **Message Content Intent** so SurfSense can read text after a bot mention. + +When installing the bot, grant: + +- View Channels +- Send Messages +- Send Messages in Threads +- Read Message History + +## Environment Variables + +For Docker installs, add these to `docker/.env`. For manual installs, add them to +`surfsense_backend/.env`. + +```bash +DISCORD_CLIENT_ID=your_discord_client_id +DISCORD_CLIENT_SECRET=your_discord_client_secret +DISCORD_BOT_TOKEN=your_discord_bot_token +GATEWAY_DISCORD_ENABLED=TRUE +GATEWAY_DISCORD_REDIRECT_URI=https://your-backend.example.com/api/v1/gateway/discord/callback +``` + +The messaging channel uses the same Discord app credentials as the Discord +connector. `DISCORD_REDIRECT_URI` remains the connector callback; +`GATEWAY_DISCORD_REDIRECT_URI` is the separate messaging channel install +callback. + +## Runtime Behavior + +1. Discord sends a `MESSAGE_CREATE` event over its WebSocket API. +2. SurfSense stores the event in the durable gateway inbox. +3. SurfSense resolves the Discord user binding to a SurfSense user and search space. +4. SurfSense runs the backend agent with that user's permissions. +5. The agent reply is posted back to the same Discord channel. + +## Deployment Note + +Only one running backend process should connect to Discord with a +given bot token. For multi-replica deployments, enable +`GATEWAY_DISCORD_ENABLED=TRUE` in a single backend process and leave it disabled +in other API replicas. diff --git a/surfsense_web/content/docs/messaging-channels/docker.mdx b/surfsense_web/content/docs/messaging-channels/docker.mdx new file mode 100644 index 000000000..3a4d4177f --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/docker.mdx @@ -0,0 +1,60 @@ +--- +title: Docker Setup +description: Configure messaging channels for Docker and one-line installs +--- + +# Docker Setup + +For Docker and one-line installs, configure messaging channels in the generated +`docker/.env` file. You do not need to edit `surfsense_backend/.env.example`. + +The Compose stack passes `docker/.env` into the backend, worker, and beat +containers. Database, Redis, SearXNG, and internal Docker networking are already +wired by Compose. + +## Public URLs + +For localhost-only testing, the defaults are enough for the SurfSense UI, but +public webhooks from Telegram, WhatsApp, and Slack require a public HTTPS backend +URL. Use your deployed backend URL or a tunnel such as Cloudflare Tunnel or +ngrok. + +When using a custom domain or tunnel, set: + +```bash +BACKEND_URL=https://api.example.com +GATEWAY_BASE_URL=https://api.example.com +NEXT_FRONTEND_URL=https://app.example.com +NEXT_PUBLIC_FASTAPI_BACKEND_URL=https://api.example.com +``` + +## Environment Variables + +Uncomment only the channel you are enabling in `docker/.env`. + +| Channel | Main variables | +| --- | --- | +| Telegram | `TELEGRAM_SHARED_BOT_TOKEN`, `TELEGRAM_SHARED_BOT_USERNAME`, `TELEGRAM_WEBHOOK_SECRET`, `GATEWAY_BASE_URL`, `GATEWAY_TELEGRAM_INTAKE_MODE` | +| WhatsApp Cloud API | `GATEWAY_WHATSAPP_INTAKE_MODE`, `WHATSAPP_SHARED_BUSINESS_TOKEN`, `WHATSAPP_SHARED_PHONE_NUMBER_ID`, `WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER`, `WHATSAPP_SHARED_WABA_ID`, `WHATSAPP_WEBHOOK_VERIFY_TOKEN`, `WHATSAPP_WEBHOOK_APP_SECRET` | +| WhatsApp Baileys | `GATEWAY_WHATSAPP_INTAKE_MODE`, `WHATSAPP_BRIDGE_URL`, `WHATSAPP_MODE` | +| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `GATEWAY_SLACK_ENABLED`, `GATEWAY_SLACK_SIGNING_SECRET`, `GATEWAY_SLACK_REDIRECT_URI` | +| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `GATEWAY_DISCORD_ENABLED`, `GATEWAY_DISCORD_REDIRECT_URI` | + +After editing `docker/.env`, restart the stack: + +```bash +docker compose up -d +``` + +For WhatsApp Baileys, start the Compose profile: + +```bash +docker compose --profile whatsapp up -d +``` + +Then follow the per-channel setup pages: + +- [Telegram](/docs/messaging-channels/telegram) +- [WhatsApp](/docs/messaging-channels/whatsapp) +- [Slack](/docs/messaging-channels/slack) +- [Discord](/docs/messaging-channels/discord) diff --git a/surfsense_web/content/docs/messaging-channels/index.mdx b/surfsense_web/content/docs/messaging-channels/index.mdx new file mode 100644 index 000000000..d15dc0e6e --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/index.mdx @@ -0,0 +1,42 @@ +--- +title: Messaging Channels +description: Chat with SurfSense from Telegram, WhatsApp, Slack, and Discord +--- + +import { Card, Cards } from 'fumadocs-ui/components/card'; + +Choose the external chat app you want to connect to SurfSense. Each guide shows +the required app setup, environment variables, and pairing flow. + + + + + + + + + diff --git a/surfsense_web/content/docs/messaging-channels/meta.json b/surfsense_web/content/docs/messaging-channels/meta.json new file mode 100644 index 000000000..00647bdb0 --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/meta.json @@ -0,0 +1,13 @@ +{ + "title": "Messaging Channels", + "icon": "MessageCircle", + "pages": [ + "telegram", + "whatsapp", + "slack", + "discord", + "docker", + "troubleshooting" + ], + "defaultOpen": false +} diff --git a/surfsense_web/content/docs/messaging-channels/slack.mdx b/surfsense_web/content/docs/messaging-channels/slack.mdx new file mode 100644 index 000000000..4e001d13a --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/slack.mdx @@ -0,0 +1,84 @@ +--- +title: Slack +description: Enable the SurfSense bot for in-Slack agent chat +--- + +# Slack Messaging Channel + +The Slack messaging channel lets users mention the SurfSense bot in Slack and +chat with the SurfSense backend agent from a Slack thread. + +This is separate from the Slack connector. The messaging channel handles bot +mentions and replies; the connector gives the agent Slack search/read tools. + +## Required Slack App Scopes + +Add these **Bot Token Scopes** in Slack OAuth & Permissions: + +| Scope | Purpose | +| --- | --- | +| `app_mentions:read` | Receive bot mention events | +| `chat:write` | Reply in Slack threads | +| `channels:read` | Read public channel metadata | +| `groups:read` | Read private channel metadata where the bot is present | +| `im:write` | Send onboarding or direct replies | +| `users:read` | Resolve Slack users | +| `team:read` | Resolve workspace metadata | + +Optional scopes: + +- `im:history` if you support direct message chat with the bot. +- `commands` if you add slash commands. + +Avoid `channels:history` and `groups:history` for the messaging channel unless +you specifically need gateway-side context reads. Slack workspace search should +stay with the Slack connector. + +## Event Subscriptions + +Enable Slack Events API and subscribe to: + +- `app_mention` + +Set the request URL to: + +```bash +https://your-backend.example.com/api/v1/gateway/webhooks/slack +``` + +Slack must be able to reach this URL. Do not use `localhost` for event +subscriptions. + +## OAuth Redirect URLs + +If the same Slack app powers both the connector and messaging channel, add both +redirect URLs in **OAuth & Permissions**: + +```bash +https://your-backend.example.com/api/v1/auth/slack/connector/callback +https://your-backend.example.com/api/v1/gateway/slack/callback +``` + +## Environment Variables + +For Docker installs, add these to `docker/.env`. For manual installs, add them to +`surfsense_backend/.env`. + +```bash +SLACK_CLIENT_ID=your_slack_client_id +SLACK_CLIENT_SECRET=your_slack_client_secret +GATEWAY_SLACK_ENABLED=TRUE +GATEWAY_SLACK_SIGNING_SECRET=your_slack_signing_secret +GATEWAY_SLACK_REDIRECT_URI=https://your-backend.example.com/api/v1/gateway/slack/callback +``` + +After changing Slack scopes, redirect URLs, or event subscriptions, reinstall +the Slack app to your workspace so Slack grants the updated permissions. + +## Runtime Behavior + +1. Slack sends an `app_mention` event to SurfSense. +2. SurfSense verifies the Slack signature and stores the event in the gateway inbox. +3. SurfSense resolves the Slack user binding to a SurfSense user and search space. +4. SurfSense runs the backend agent with that user's permissions. +5. The agent reply is posted back in the same Slack thread. diff --git a/surfsense_web/content/docs/messaging-channels/telegram.mdx b/surfsense_web/content/docs/messaging-channels/telegram.mdx new file mode 100644 index 000000000..3487da864 --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/telegram.mdx @@ -0,0 +1,62 @@ +--- +title: Telegram +description: Enable SurfSense chat from Telegram +--- + +# Telegram Messaging Channel + +Telegram lets users chat with the SurfSense agent from a Telegram bot. Users pair +their Telegram chat from **User Settings > Messaging Channels**. + +## Environment Variables + +For Docker installs, add these to `docker/.env`. For manual installs, add them to +`surfsense_backend/.env`. + +```bash +TELEGRAM_SHARED_BOT_TOKEN=123456:bot-token-from-botfather +TELEGRAM_SHARED_BOT_USERNAME=your_bot_username +TELEGRAM_WEBHOOK_SECRET=generate-a-long-random-secret +GATEWAY_BASE_URL=https://api.example.com +GATEWAY_TELEGRAM_INTAKE_MODE=webhook +``` + +`TELEGRAM_WEBHOOK_SECRET` must be 1-256 characters and contain only `A-Z`, `a-z`, +`0-9`, `_`, or `-`. + +## Intake Modes + +| Mode | Use when | +| --- | --- | +| `webhook` | Production or any deployment with a public HTTPS backend URL | +| `longpoll` | Single-replica self-host installs that cannot expose a public HTTPS webhook | +| `disabled` | You do not want Telegram intake enabled | + +For SaaS-style or multi-replica deployments, use `webhook`. Long polling should +only run in a single backend process. + +## Webhook URL + +Telegram webhooks use this shape: + +```text +${GATEWAY_BASE_URL}/api/v1/gateway/webhooks/telegram/{account_id} +``` + +After deploying the backend, register the webhook: + +```bash +cd surfsense_backend +uv run python scripts/register_webhook.py +``` + +If switching a bot from long polling to webhooks, delete any existing Telegram +webhook or pending `getUpdates` session before relying on the new mode. + +## Pairing Flow + +1. The user opens **User Settings > Messaging Channels**. +2. The user starts Telegram pairing. +3. SurfSense provides a pairing code or bot link. +4. The user sends the pairing command to the Telegram bot. +5. SurfSense binds that Telegram chat to the selected search space. diff --git a/surfsense_web/content/docs/messaging-channels/troubleshooting.mdx b/surfsense_web/content/docs/messaging-channels/troubleshooting.mdx new file mode 100644 index 000000000..bdd385e28 --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/troubleshooting.mdx @@ -0,0 +1,66 @@ +--- +title: Troubleshooting +description: Common messaging channel pairing, webhook, and bot reply issues +--- + +# Messaging Channels Troubleshooting + +## The Bot Does Not Reply + +Check that: + +- The channel is enabled in the backend environment. +- The backend restarted after the environment change. +- The external platform can reach your public HTTPS backend URL. +- The user paired the channel from **User Settings > Messaging Channels**. +- Redis is running, because gateway inbox processing uses backend coordination + and rate-limit state. + +## Telegram + +Check that: + +- `TELEGRAM_SHARED_BOT_TOKEN` and `TELEGRAM_SHARED_BOT_USERNAME` are correct. +- `GATEWAY_TELEGRAM_INTAKE_MODE` is one of `webhook`, `longpoll`, or `disabled`. +- `TELEGRAM_WEBHOOK_SECRET` contains only `A-Z`, `a-z`, `0-9`, `_`, or `-`. +- Webhook mode uses a public HTTPS `GATEWAY_BASE_URL`. +- Long polling runs in only one backend process. + +## WhatsApp + +For Meta Cloud API, check that: + +- `GATEWAY_WHATSAPP_INTAKE_MODE=cloud`. +- The Meta webhook URL is `${GATEWAY_BASE_URL}/api/v1/gateway/webhooks/whatsapp`. +- The Meta verify token matches `WHATSAPP_WEBHOOK_VERIFY_TOKEN`. +- `WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER` contains the public WhatsApp number + users should message. + +For Baileys, check that: + +- `GATEWAY_WHATSAPP_INTAKE_MODE=baileys`. +- The `whatsapp` Compose profile is running. +- The bridge is paired and healthy. +- You are messaging the account's Message Yourself chat. + +## Slack + +Check that: + +- `GATEWAY_SLACK_ENABLED=TRUE`. +- The Slack signing secret matches `GATEWAY_SLACK_SIGNING_SECRET`. +- Slack Events API is enabled and subscribed to `app_mention`. +- The Slack event request URL is public HTTPS and points to + `/api/v1/gateway/webhooks/slack`. +- The Slack app was reinstalled after scope or redirect URL changes. + +## Discord + +Check that: + +- `GATEWAY_DISCORD_ENABLED=TRUE`. +- The bot token is valid. +- Message Content Intent is enabled. +- The bot can view and send messages in the channel. +- Exactly one backend process is running the Discord listener. +- The Discord user is paired to a SurfSense user and search space. diff --git a/surfsense_web/content/docs/messaging-channels/whatsapp.mdx b/surfsense_web/content/docs/messaging-channels/whatsapp.mdx new file mode 100644 index 000000000..56015fd20 --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/whatsapp.mdx @@ -0,0 +1,75 @@ +--- +title: WhatsApp +description: Enable SurfSense chat from WhatsApp +--- + +# WhatsApp Messaging Channel + +WhatsApp supports two intake modes: + +- `cloud` uses the official Meta WhatsApp Cloud API with a SurfSense-owned system + WhatsApp Business Account. +- `baileys` uses the unofficial Baileys WebSocket bridge for self-hosted, + one-tenant Message Yourself installs. + +Use `cloud` for production and shared deployments. + +## Meta Cloud API + +Create a Meta app, provision a WhatsApp Business Account and phone number, and +create a long-lived system user token with WhatsApp permissions. + +Point the Meta app webhook to: + +```text +${GATEWAY_BASE_URL}/api/v1/gateway/webhooks/whatsapp +``` + +Set the webhook verify token in Meta to the same value as +`WHATSAPP_WEBHOOK_VERIFY_TOKEN`. + +For Docker installs, add these to `docker/.env`. For manual installs, add them to +`surfsense_backend/.env`. + +```bash +GATEWAY_WHATSAPP_INTAKE_MODE=cloud +WHATSAPP_SHARED_BUSINESS_TOKEN=your-system-user-token +WHATSAPP_SHARED_PHONE_NUMBER_ID=your-meta-phone-number-id +WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER=15551234567 +WHATSAPP_SHARED_WABA_ID=your-waba-id +WHATSAPP_GRAPH_API_VERSION=v25.0 +WHATSAPP_WEBHOOK_VERIFY_TOKEN=generate-a-long-random-secret +WHATSAPP_WEBHOOK_APP_SECRET=your-meta-app-secret +``` + +Users open **User Settings > Messaging Channels**, click **Pair WhatsApp**, and +open the returned `wa.me` link. WhatsApp pre-fills `/start CODE`; the user must +press send to bind the chat. + +## Baileys Self-Hosted Mode + +Baileys is unofficial. Use it only for single-tenant self-hosted installs where +the operator accepts the risk of a personal WhatsApp session bridge. + +```bash +GATEWAY_WHATSAPP_INTAKE_MODE=baileys +WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:9929 +WHATSAPP_MODE=self-chat +docker compose --profile whatsapp up -d +``` + +After pairing, use WhatsApp's Message Yourself chat. The bridge only forwards +messages from your own self-chat and ignores groups, other chats, and other +people. + +The `whatsapp-bridge` container stores Baileys auth state in the +`surfsense-whatsapp-sessions` Docker volume. That volume contains account +takeover material. Treat it like a secret. + +To intentionally reset pairing: + +```bash +docker compose --profile whatsapp down +docker volume rm surfsense-whatsapp-sessions +docker compose --profile whatsapp up -d +``` diff --git a/surfsense_web/content/docs/meta.json b/surfsense_web/content/docs/meta.json index 13b599118..74be10600 100644 --- a/surfsense_web/content/docs/meta.json +++ b/surfsense_web/content/docs/meta.json @@ -9,6 +9,7 @@ "installation", "manual-installation", "docker-installation", + "messaging-channels", "connectors", "how-to", "---Developers---", diff --git a/surfsense_web/lib/source.ts b/surfsense_web/lib/source.ts index 62fbb362b..f71e8b688 100644 --- a/surfsense_web/lib/source.ts +++ b/surfsense_web/lib/source.ts @@ -7,6 +7,7 @@ import { Download, FlaskConical, Heart, + MessageCircle, Radar, Unplug, Wrench, @@ -27,6 +28,7 @@ const DOCS_ICONS: Record = { Download, FlaskConical, Heart, + MessageCircle, Radar, Unplug, Wrench, From 1f83898a8703ca1891dd4186b78377748b53c0a2 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 2 Jun 2026 01:19:28 +0530 Subject: [PATCH 61/63] feat(gateway): renamed the gateway related alembic migrations --- ...44_add_gateway_tables.py => 148_add_gateway_tables.py} | 8 ++++---- ...eway_platform.py => 149_add_slack_gateway_platform.py} | 8 ++++---- ...ay_platform.py => 150_add_discord_gateway_platform.py} | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) rename surfsense_backend/alembic/versions/{144_add_gateway_tables.py => 148_add_gateway_tables.py} (99%) rename surfsense_backend/alembic/versions/{145_add_slack_gateway_platform.py => 149_add_slack_gateway_platform.py} (97%) rename surfsense_backend/alembic/versions/{146_add_discord_gateway_platform.py => 150_add_discord_gateway_platform.py} (97%) diff --git a/surfsense_backend/alembic/versions/144_add_gateway_tables.py b/surfsense_backend/alembic/versions/148_add_gateway_tables.py similarity index 99% rename from surfsense_backend/alembic/versions/144_add_gateway_tables.py rename to surfsense_backend/alembic/versions/148_add_gateway_tables.py index 91d06f815..72282166a 100644 --- a/surfsense_backend/alembic/versions/144_add_gateway_tables.py +++ b/surfsense_backend/alembic/versions/148_add_gateway_tables.py @@ -1,7 +1,7 @@ """add external chat surface tables -Revision ID: 144 -Revises: 143 +Revision ID: 148 +Revises: 147 Create Date: 2026-05-27 Adds the lean external chat surface schema: @@ -27,8 +27,8 @@ from sqlalchemy.dialects import postgresql from alembic import op -revision: str = "144" -down_revision: str | None = "143" +revision: str = "148" +down_revision: str | None = "147" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None diff --git a/surfsense_backend/alembic/versions/145_add_slack_gateway_platform.py b/surfsense_backend/alembic/versions/149_add_slack_gateway_platform.py similarity index 97% rename from surfsense_backend/alembic/versions/145_add_slack_gateway_platform.py rename to surfsense_backend/alembic/versions/149_add_slack_gateway_platform.py index f4ab18e72..e6f06f2bb 100644 --- a/surfsense_backend/alembic/versions/145_add_slack_gateway_platform.py +++ b/surfsense_backend/alembic/versions/149_add_slack_gateway_platform.py @@ -1,7 +1,7 @@ """add slack gateway platform -Revision ID: 145 -Revises: 144 +Revision ID: 149 +Revises: 148 Create Date: 2026-05-31 """ @@ -13,8 +13,8 @@ import sqlalchemy as sa from alembic import op -revision: str = "145" -down_revision: str | None = "144" +revision: str = "149" +down_revision: str | None = "148" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None diff --git a/surfsense_backend/alembic/versions/146_add_discord_gateway_platform.py b/surfsense_backend/alembic/versions/150_add_discord_gateway_platform.py similarity index 97% rename from surfsense_backend/alembic/versions/146_add_discord_gateway_platform.py rename to surfsense_backend/alembic/versions/150_add_discord_gateway_platform.py index 32b02d059..c6ba0d3b6 100644 --- a/surfsense_backend/alembic/versions/146_add_discord_gateway_platform.py +++ b/surfsense_backend/alembic/versions/150_add_discord_gateway_platform.py @@ -1,7 +1,7 @@ """add discord gateway platform -Revision ID: 146 -Revises: 145 +Revision ID: 150 +Revises: 149 Create Date: 2026-06-01 """ @@ -13,8 +13,8 @@ import sqlalchemy as sa from alembic import op -revision: str = "146" -down_revision: str | None = "145" +revision: str = "150" +down_revision: str | None = "149" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None From 8f8abe6f58709fd50841d237f5f541f28a2f2c35 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 2 Jun 2026 01:24:50 +0530 Subject: [PATCH 62/63] chore: renamed alembic migrations --- ...48_add_gateway_tables.py => 149_add_gateway_tables.py} | 8 ++++---- ...eway_platform.py => 150_add_slack_gateway_platform.py} | 8 ++++---- ...ay_platform.py => 151_add_discord_gateway_platform.py} | 0 3 files changed, 8 insertions(+), 8 deletions(-) rename surfsense_backend/alembic/versions/{148_add_gateway_tables.py => 149_add_gateway_tables.py} (99%) rename surfsense_backend/alembic/versions/{149_add_slack_gateway_platform.py => 150_add_slack_gateway_platform.py} (97%) rename surfsense_backend/alembic/versions/{150_add_discord_gateway_platform.py => 151_add_discord_gateway_platform.py} (100%) diff --git a/surfsense_backend/alembic/versions/148_add_gateway_tables.py b/surfsense_backend/alembic/versions/149_add_gateway_tables.py similarity index 99% rename from surfsense_backend/alembic/versions/148_add_gateway_tables.py rename to surfsense_backend/alembic/versions/149_add_gateway_tables.py index 72282166a..888da0691 100644 --- a/surfsense_backend/alembic/versions/148_add_gateway_tables.py +++ b/surfsense_backend/alembic/versions/149_add_gateway_tables.py @@ -1,7 +1,7 @@ """add external chat surface tables -Revision ID: 148 -Revises: 147 +Revision ID: 149 +Revises: 148 Create Date: 2026-05-27 Adds the lean external chat surface schema: @@ -27,8 +27,8 @@ from sqlalchemy.dialects import postgresql from alembic import op -revision: str = "148" -down_revision: str | None = "147" +revision: str = "149" +down_revision: str | None = "148" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None diff --git a/surfsense_backend/alembic/versions/149_add_slack_gateway_platform.py b/surfsense_backend/alembic/versions/150_add_slack_gateway_platform.py similarity index 97% rename from surfsense_backend/alembic/versions/149_add_slack_gateway_platform.py rename to surfsense_backend/alembic/versions/150_add_slack_gateway_platform.py index e6f06f2bb..474867d5b 100644 --- a/surfsense_backend/alembic/versions/149_add_slack_gateway_platform.py +++ b/surfsense_backend/alembic/versions/150_add_slack_gateway_platform.py @@ -1,7 +1,7 @@ """add slack gateway platform -Revision ID: 149 -Revises: 148 +Revision ID: 151 +Revises: 150 Create Date: 2026-05-31 """ @@ -13,8 +13,8 @@ import sqlalchemy as sa from alembic import op -revision: str = "149" -down_revision: str | None = "148" +revision: str = "151" +down_revision: str | None = "150" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None diff --git a/surfsense_backend/alembic/versions/150_add_discord_gateway_platform.py b/surfsense_backend/alembic/versions/151_add_discord_gateway_platform.py similarity index 100% rename from surfsense_backend/alembic/versions/150_add_discord_gateway_platform.py rename to surfsense_backend/alembic/versions/151_add_discord_gateway_platform.py From 375056858fc7d6eb07bd8c435d1975500c5e089c Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 2 Jun 2026 01:27:34 +0530 Subject: [PATCH 63/63] chore: update migration number --- .../alembic/versions/150_add_slack_gateway_platform.py | 8 ++++---- .../alembic/versions/151_add_discord_gateway_platform.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/surfsense_backend/alembic/versions/150_add_slack_gateway_platform.py b/surfsense_backend/alembic/versions/150_add_slack_gateway_platform.py index 474867d5b..388d8ef42 100644 --- a/surfsense_backend/alembic/versions/150_add_slack_gateway_platform.py +++ b/surfsense_backend/alembic/versions/150_add_slack_gateway_platform.py @@ -1,7 +1,7 @@ """add slack gateway platform -Revision ID: 151 -Revises: 150 +Revision ID: 150 +Revises: 149 Create Date: 2026-05-31 """ @@ -13,8 +13,8 @@ import sqlalchemy as sa from alembic import op -revision: str = "151" -down_revision: str | None = "150" +revision: str = "150" +down_revision: str | None = "149" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None diff --git a/surfsense_backend/alembic/versions/151_add_discord_gateway_platform.py b/surfsense_backend/alembic/versions/151_add_discord_gateway_platform.py index c6ba0d3b6..f91e71210 100644 --- a/surfsense_backend/alembic/versions/151_add_discord_gateway_platform.py +++ b/surfsense_backend/alembic/versions/151_add_discord_gateway_platform.py @@ -1,7 +1,7 @@ """add discord gateway platform -Revision ID: 150 -Revises: 149 +Revision ID: 151 +Revises: 150 Create Date: 2026-06-01 """ @@ -13,8 +13,8 @@ import sqlalchemy as sa from alembic import op -revision: str = "150" -down_revision: str | None = "149" +revision: str = "151" +down_revision: str | None = "150" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None