apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-06-04 20:05:16 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor(gateway): rename persistence models to external chat

Browse source
This commit is contained in:
Anish Sarkar 2026-05-28 04:37:27 +05:30
parent f2d82234d4
commit a57b741d5e
8 changed files with 274 additions and 323 deletions
Download patch file Download diff file Expand all files Collapse all files

211
surfsense_backend/app/db.py
View file

@ -574,62 +574,45 @@ class ChatVisibility(StrEnum):
# PUBLIC = "PUBLIC" # Reserved for future implementation
class GatewayPlatform(StrEnum):
class ExternalChatPlatform(StrEnum):
TELEGRAM = "telegram"
WHATSAPP = "whatsapp"
SIGNAL = "signal"
class GatewayAccountMode(StrEnum):
class ExternalChatAccountMode(StrEnum):
CLOUD_SHARED = "cloud_shared"
SELF_HOST_BYO = "self_host_byo"
class GatewayHealthStatus(StrEnum):
class ExternalChatHealthStatus(StrEnum):
UNKNOWN = "unknown"
OK = "ok"
FAILING = "failing"
class GatewayBindingState(StrEnum):
class ExternalChatBindingState(StrEnum):
PENDING = "pending"
BOUND = "bound"
REVOKED = "revoked"
SUSPENDED = "suspended"
class GatewayPeerKind(StrEnum):
class ExternalChatPeerKind(StrEnum):
DIRECT = "direct"
GROUP = "group"
CHANNEL = "channel"
UNKNOWN = "unknown"
class GatewaySessionScope(StrEnum):
PER_BINDING = "per_binding"
PER_USER_SEARCH_SPACE = "per_user_search_space"
EPHEMERAL = "ephemeral"
class GatewayDmPolicy(StrEnum):
ENABLED = "enabled"
DISABLED = "disabled"
class GatewayGroupPolicy(StrEnum):
DISABLED = "disabled"
ALLOWLIST = "allowlist"
MENTION_REQUIRED = "mention_required"
class GatewayEventKind(StrEnum):
class ExternalChatEventKind(StrEnum):
MESSAGE = "message"
EDITED_MESSAGE = "edited_message"
CALLBACK_QUERY = "callback_query"
OTHER = "other"
class GatewayEventStatus(StrEnum):
class ExternalChatEventStatus(StrEnum):
RECEIVED = "received"
PROCESSING = "processing"
PROCESSED = "processed"
@ -713,12 +696,12 @@ class NewChatThread(BaseModel, TimestampMixin):
# agent_llm_id changes). Unindexed: all reads are by primary key.
pinned_llm_config_id = Column(Integer, nullable=True)
# Gateway-originated threads are persisted for the agent, but the UI Zero
# publication only exposes ``source='web'`` rows.
# Surface metadata for web and external chat threads. Zero publishes all
# chat-message sources; the UI can decide which surfaces to render.
source = Column(Text, nullable=False, default="web", server_default="web")
binding_id = Column(
external_chat_binding_id = Column(
BigInteger,
ForeignKey("gateway_conversation_bindings.id", ondelete="SET NULL"),
ForeignKey("external_chat_bindings.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
@ -743,9 +726,9 @@ class NewChatThread(BaseModel, TimestampMixin):
back_populates="thread",
cascade="all, delete-orphan",
)
gateway_binding = relationship(
"GatewayConversationBinding",
foreign_keys=[binding_id],
external_chat_binding = relationship(
"ExternalChatBinding",
foreign_keys=[external_chat_binding_id],
back_populates="threads",
)
@ -822,23 +805,23 @@ class NewChatMessage(BaseModel, TimestampMixin):
)
class GatewayPlatformAccount(Base, TimestampMixin):
__tablename__ = "gateway_platform_accounts"
class ExternalChatAccount(Base, TimestampMixin):
__tablename__ = "external_chat_accounts"
__allow_unmapped__ = True
id = Column(BigInteger, primary_key=True, index=True)
platform = Column(
SQLAlchemyEnum(
GatewayPlatform,
name="gateway_platform",
ExternalChatPlatform,
name="external_chat_platform",
values_callable=_enum_values,
),
nullable=False,
)
mode = Column(
SQLAlchemyEnum(
GatewayAccountMode,
name="gateway_account_mode",
ExternalChatAccountMode,
name="external_chat_account_mode",
values_callable=_enum_values,
),
nullable=False,
@ -851,17 +834,18 @@ class GatewayPlatformAccount(Base, TimestampMixin):
)
is_system_account = Column(Boolean, nullable=False, default=False, server_default="false")
encrypted_credentials = Column(Text, nullable=True)
account_metadata = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb"))
bot_username = Column(String(255), nullable=True)
webhook_secret = Column(String(64), nullable=True)
cursor_state = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb"))
health_status = Column(
SQLAlchemyEnum(
GatewayHealthStatus,
name="gateway_health_status",
ExternalChatHealthStatus,
name="external_chat_health_status",
values_callable=_enum_values,
),
nullable=False,
default=GatewayHealthStatus.UNKNOWN,
server_default=GatewayHealthStatus.UNKNOWN.value,
default=ExternalChatHealthStatus.UNKNOWN,
server_default=ExternalChatHealthStatus.UNKNOWN.value,
)
last_health_check_at = Column(TIMESTAMP(timezone=True), nullable=True)
suspended_at = Column(TIMESTAMP(timezone=True), nullable=True)
@ -877,12 +861,12 @@ class GatewayPlatformAccount(Base, TimestampMixin):
owner = relationship("User", foreign_keys=[owner_user_id])
owner_search_space = relationship("SearchSpace", foreign_keys=[owner_search_space_id])
bindings = relationship(
"GatewayConversationBinding",
"ExternalChatBinding",
back_populates="account",
cascade="all, delete-orphan",
)
inbound_events = relationship(
"GatewayInboundEvent",
"ExternalChatInboundEvent",
back_populates="account",
cascade="all, delete-orphan",
)
@ -891,32 +875,38 @@ class GatewayPlatformAccount(Base, TimestampMixin):
CheckConstraint(
"(is_system_account = true AND owner_user_id IS NULL) OR "
"(is_system_account = false AND owner_user_id IS NOT NULL)",
name="ck_gateway_accounts_owner_shape",
name="ck_external_chat_accounts_owner_shape",
),
Index(
"uq_gateway_accounts_owner_platform",
"uq_external_chat_accounts_owner_platform",
"owner_user_id",
"platform",
unique=True,
postgresql_where=text("is_system_account = false"),
),
Index(
"uq_gateway_accounts_system_platform",
"uq_external_chat_accounts_system_platform",
"platform",
unique=True,
postgresql_where=text("is_system_account = true"),
),
Index(
"uq_external_chat_accounts_webhook_secret",
"webhook_secret",
unique=True,
postgresql_where=text("webhook_secret IS NOT NULL"),
),
)
class GatewayConversationBinding(Base, TimestampMixin):
__tablename__ = "gateway_conversation_bindings"
class ExternalChatBinding(Base, TimestampMixin):
__tablename__ = "external_chat_bindings"
__allow_unmapped__ = True
id = Column(BigInteger, primary_key=True, index=True)
account_id = Column(
BigInteger,
ForeignKey("gateway_platform_accounts.id", ondelete="CASCADE"),
ForeignKey("external_chat_accounts.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
@ -928,68 +918,37 @@ class GatewayConversationBinding(Base, TimestampMixin):
)
state = Column(
SQLAlchemyEnum(
GatewayBindingState,
name="gateway_binding_state",
ExternalChatBindingState,
name="external_chat_binding_state",
values_callable=_enum_values,
),
nullable=False,
default=GatewayBindingState.PENDING,
server_default=GatewayBindingState.PENDING.value,
default=ExternalChatBindingState.PENDING,
server_default=ExternalChatBindingState.PENDING.value,
)
pairing_code = Column(Text, nullable=True)
pairing_code_expires_at = Column(TIMESTAMP(timezone=True), nullable=True)
external_peer_id = Column(Text, nullable=True)
external_peer_kind = Column(
SQLAlchemyEnum(
GatewayPeerKind,
name="gateway_peer_kind",
ExternalChatPeerKind,
name="external_chat_peer_kind",
values_callable=_enum_values,
),
nullable=False,
default=GatewayPeerKind.UNKNOWN,
server_default=GatewayPeerKind.UNKNOWN.value,
default=ExternalChatPeerKind.UNKNOWN,
server_default=ExternalChatPeerKind.UNKNOWN.value,
)
external_thread_id = Column(Text, nullable=True)
external_display_name = Column(Text, nullable=True)
external_username = Column(Text, nullable=True)
external_pii_hashes = Column(JSONB, nullable=True)
external_metadata = Column(JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb"))
active_thread_id = Column(
new_chat_thread_id = Column(
Integer,
ForeignKey("new_chat_threads.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
session_scope = Column(
SQLAlchemyEnum(
GatewaySessionScope,
name="gateway_session_scope",
values_callable=_enum_values,
),
nullable=False,
default=GatewaySessionScope.PER_BINDING,
server_default=GatewaySessionScope.PER_BINDING.value,
)
dm_policy = Column(
SQLAlchemyEnum(
GatewayDmPolicy,
name="gateway_dm_policy",
values_callable=_enum_values,
),
nullable=False,
default=GatewayDmPolicy.ENABLED,
server_default=GatewayDmPolicy.ENABLED.value,
)
group_policy = Column(
SQLAlchemyEnum(
GatewayGroupPolicy,
name="gateway_group_policy",
values_callable=_enum_values,
),
nullable=False,
default=GatewayGroupPolicy.DISABLED,
server_default=GatewayGroupPolicy.DISABLED.value,
)
revoked_at = Column(TIMESTAMP(timezone=True), nullable=True)
suspended_at = Column(TIMESTAMP(timezone=True), nullable=True)
suspended_reason = Column(Text, nullable=True)
@ -1001,24 +960,24 @@ class GatewayConversationBinding(Base, TimestampMixin):
server_default=text("(now() AT TIME ZONE 'utc')"),
)
account = relationship("GatewayPlatformAccount", back_populates="bindings")
account = relationship("ExternalChatAccount", back_populates="bindings")
user = relationship("User", foreign_keys=[user_id])
search_space = relationship("SearchSpace", foreign_keys=[search_space_id])
active_thread = relationship("NewChatThread", foreign_keys=[active_thread_id])
new_chat_thread = relationship("NewChatThread", foreign_keys=[new_chat_thread_id])
threads = relationship(
"NewChatThread",
back_populates="gateway_binding",
foreign_keys="NewChatThread.binding_id",
back_populates="external_chat_binding",
foreign_keys="NewChatThread.external_chat_binding_id",
)
inbound_events = relationship(
"GatewayInboundEvent",
"ExternalChatInboundEvent",
back_populates="binding",
foreign_keys="GatewayInboundEvent.binding_id",
foreign_keys="ExternalChatInboundEvent.external_chat_binding_id",
)
__table_args__ = (
Index(
"uq_gateway_bindings_account_peer_active",
"uq_external_chat_bindings_account_peer_active",
"account_id",
"external_peer_id",
unique=True,
@ -1027,37 +986,37 @@ class GatewayConversationBinding(Base, TimestampMixin):
),
),
Index(
"uq_gateway_bindings_pairing_code_pending",
"uq_external_chat_bindings_pairing_code_pending",
"pairing_code",
unique=True,
postgresql_where=text("state = 'pending'"),
),
Index("ix_gateway_bindings_user_state", "user_id", "state"),
Index("ix_gateway_bindings_search_space_state", "search_space_id", "state"),
Index("ix_external_chat_bindings_user_state", "user_id", "state"),
Index("ix_external_chat_bindings_search_space_state", "search_space_id", "state"),
)
class GatewayInboundEvent(Base, TimestampMixin):
__tablename__ = "gateway_inbound_events"
class ExternalChatInboundEvent(Base, TimestampMixin):
__tablename__ = "external_chat_inbound_events"
__allow_unmapped__ = True
id = Column(BigInteger, primary_key=True, index=True)
account_id = Column(
BigInteger,
ForeignKey("gateway_platform_accounts.id", ondelete="CASCADE"),
ForeignKey("external_chat_accounts.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
binding_id = Column(
external_chat_binding_id = Column(
BigInteger,
ForeignKey("gateway_conversation_bindings.id", ondelete="SET NULL"),
ForeignKey("external_chat_bindings.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
platform = Column(
SQLAlchemyEnum(
GatewayPlatform,
name="gateway_platform",
ExternalChatPlatform,
name="external_chat_platform",
values_callable=_enum_values,
),
nullable=False,
@ -1067,28 +1026,23 @@ class GatewayInboundEvent(Base, TimestampMixin):
external_message_id = Column(Text, nullable=True)
event_kind = Column(
SQLAlchemyEnum(
GatewayEventKind,
name="gateway_event_kind",
ExternalChatEventKind,
name="external_chat_event_kind",
values_callable=_enum_values,
),
nullable=False,
)
raw_payload = Column(JSONB, nullable=True)
processing_metadata = Column(
JSONB,
nullable=False,
default=dict,
server_default=text("'{}'::jsonb"),
)
request_id = Column(String(64), nullable=True)
status = Column(
SQLAlchemyEnum(
GatewayEventStatus,
name="gateway_event_status",
ExternalChatEventStatus,
name="external_chat_event_status",
values_callable=_enum_values,
),
nullable=False,
default=GatewayEventStatus.RECEIVED,
server_default=GatewayEventStatus.RECEIVED.value,
default=ExternalChatEventStatus.RECEIVED,
server_default=ExternalChatEventStatus.RECEIVED.value,
)
attempt_count = Column(Integer, nullable=False, default=0, server_default="0")
last_error = Column(Text, nullable=True)
@ -1100,17 +1054,26 @@ class GatewayInboundEvent(Base, TimestampMixin):
)
processed_at = Column(TIMESTAMP(timezone=True), nullable=True)
account = relationship("GatewayPlatformAccount", back_populates="inbound_events")
binding = relationship("GatewayConversationBinding", back_populates="inbound_events")
account = relationship("ExternalChatAccount", back_populates="inbound_events")
binding = relationship("ExternalChatBinding", back_populates="inbound_events")
__table_args__ = (
UniqueConstraint(
"account_id",
"event_dedupe_key",
name="uq_gateway_inbound_account_dedupe_key",
name="uq_external_chat_inbound_account_dedupe_key",
),
Index("ix_external_chat_inbound_status_received_at", "status", "received_at"),
Index(
"ix_external_chat_inbound_binding_received_at",
"external_chat_binding_id",
"received_at",
),
Index(
"ix_external_chat_inbound_request_id",
"request_id",
postgresql_where=text("request_id IS NOT NULL"),
),
Index("ix_gateway_inbound_status_received_at", "status", "received_at"),
Index("ix_gateway_inbound_binding_received_at", "binding_id", "received_at"),
)

36
surfsense_backend/app/gateway/accounts.py
View file

@ -1,4 +1,4 @@
"""Gateway account helpers."""
"""External chat account helpers."""
from __future__ import annotations
@ -7,16 +7,16 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.config import config
from app.db import (
GatewayAccountMode,
GatewayHealthStatus,
GatewayPlatform,
GatewayPlatformAccount,
ExternalChatAccountMode,
ExternalChatHealthStatus,
ExternalChatPlatform,
ExternalChatAccount,
)
from app.utils.oauth_security import TokenEncryption
def account_token(account: GatewayPlatformAccount) -> str | None:
if account.is_system_account and account.platform == GatewayPlatform.TELEGRAM:
def account_token(account: ExternalChatAccount) -> str | None:
if account.is_system_account and account.platform == ExternalChatPlatform.TELEGRAM:
return config.TELEGRAM_SHARED_BOT_TOKEN
if not account.encrypted_credentials:
return None
@ -27,26 +27,24 @@ def account_token(account: GatewayPlatformAccount) -> str | None:
async def get_or_create_system_telegram_account(
session: AsyncSession,
) -> GatewayPlatformAccount:
) -> ExternalChatAccount:
result = await session.execute(
select(GatewayPlatformAccount).where(
GatewayPlatformAccount.platform == GatewayPlatform.TELEGRAM,
GatewayPlatformAccount.is_system_account.is_(True),
select(ExternalChatAccount).where(
ExternalChatAccount.platform == ExternalChatPlatform.TELEGRAM,
ExternalChatAccount.is_system_account.is_(True),
)
)
account = result.scalars().first()
if account is not None:
return account
account = GatewayPlatformAccount(
platform=GatewayPlatform.TELEGRAM,
mode=GatewayAccountMode.CLOUD_SHARED,
account = ExternalChatAccount(
platform=ExternalChatPlatform.TELEGRAM,
mode=ExternalChatAccountMode.CLOUD_SHARED,
is_system_account=True,
account_metadata={
"bot_username": config.TELEGRAM_SHARED_BOT_USERNAME,
"webhook_secret": config.TELEGRAM_WEBHOOK_SECRET,
},
bot_username=config.TELEGRAM_SHARED_BOT_USERNAME,
webhook_secret=config.TELEGRAM_WEBHOOK_SECRET,
cursor_state={},
health_status=GatewayHealthStatus.UNKNOWN,
health_status=ExternalChatHealthStatus.UNKNOWN,
)
session.add(account)
await session.flush()

10
surfsense_backend/app/gateway/auth_invariant.py
View file

@ -1,11 +1,11 @@
"""Authorization invariants for gateway-routed turns."""
"""Authorization invariants for external-chat-routed turns."""
from __future__ import annotations
from fastapi import HTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import GatewayConversationBinding, Permission, User
from app.db import ExternalChatBinding, Permission, User
from app.gateway.bindings import suspend_binding
from app.observability.metrics import record_gateway_auth_invariant_failure
from app.utils.rbac import check_permission, check_search_space_access
@ -19,7 +19,7 @@ class GatewaySuspendedError(RuntimeError):
async def _fail(
session: AsyncSession,
binding: GatewayConversationBinding,
binding: ExternalChatBinding,
reason: str,
) -> None:
suspend_binding(binding, reason)
@ -30,7 +30,7 @@ async def _fail(
async def assert_authorization_invariant(
session: AsyncSession,
binding: GatewayConversationBinding,
binding: ExternalChatBinding,
) -> User:
if binding.state != "bound":
await _fail(session, binding, "binding_not_bound")
@ -46,7 +46,7 @@ async def assert_authorization_invariant(
user,
binding.search_space_id,
Permission.CHATS_CREATE.value,
"Gateway owner no longer has permission to chat in this search space",
"External chat owner no longer has permission to chat in this search space",
)
except HTTPException as exc:
await _fail(session, binding, f"rbac_{exc.status_code}")

30
surfsense_backend/app/gateway/bindings.py
View file

@ -1,4 +1,4 @@
"""Gateway binding helpers."""
"""External chat binding helpers."""
from __future__ import annotations
@ -9,19 +9,19 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.db import (
ChatVisibility,
GatewayBindingState,
GatewayConversationBinding,
ExternalChatBindingState,
ExternalChatBinding,
NewChatThread,
)
async def get_or_create_thread_for_binding(
session: AsyncSession,
binding: GatewayConversationBinding,
binding: ExternalChatBinding,
) -> NewChatThread:
if binding.active_thread_id is not None:
if binding.new_chat_thread_id is not None:
result = await session.execute(
select(NewChatThread).where(NewChatThread.id == binding.active_thread_id)
select(NewChatThread).where(NewChatThread.id == binding.new_chat_thread_id)
)
thread = result.scalars().first()
if thread is not None and not thread.archived:
@ -33,30 +33,30 @@ async def get_or_create_thread_for_binding(
created_by_id=binding.user_id,
visibility=ChatVisibility.PRIVATE,
source="telegram",
binding_id=binding.id,
external_chat_binding_id=binding.id,
)
session.add(thread)
await session.flush()
binding.active_thread_id = thread.id
binding.new_chat_thread_id = thread.id
return thread
def suspend_binding(binding: GatewayConversationBinding, reason: str) -> None:
def suspend_binding(binding: ExternalChatBinding, reason: str) -> None:
now = datetime.now(UTC)
binding.state = GatewayBindingState.SUSPENDED
binding.state = ExternalChatBindingState.SUSPENDED
binding.suspended_at = now
binding.suspended_reason = reason
def revoke_binding(binding: GatewayConversationBinding) -> None:
def revoke_binding(binding: ExternalChatBinding) -> None:
now = datetime.now(UTC)
binding.state = GatewayBindingState.REVOKED
binding.state = ExternalChatBindingState.REVOKED
binding.revoked_at = now
binding.active_thread_id = None
binding.new_chat_thread_id = None
def resume_binding(binding: GatewayConversationBinding) -> None:
binding.state = GatewayBindingState.BOUND
def resume_binding(binding: ExternalChatBinding) -> None:
binding.state = ExternalChatBindingState.BOUND
binding.suspended_at = None
binding.suspended_reason = None

10
surfsense_backend/app/gateway/inbox.py
View file

@ -5,7 +5,7 @@ from __future__ import annotations
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import GatewayInboundEvent, GatewayPlatform
from app.db import ExternalChatInboundEvent, ExternalChatPlatform
def telegram_event_dedupe_key(update_id: int | str) -> str:
@ -16,15 +16,16 @@ async def persist_inbound_event(
session: AsyncSession,
*,
account_id: int,
platform: GatewayPlatform,
platform: ExternalChatPlatform,
event_dedupe_key: str,
event_kind: str,
raw_payload: dict,
external_event_id: str | None = None,
external_message_id: str | None = None,
request_id: str | None = None,
) -> int | None:
stmt = (
insert(GatewayInboundEvent)
insert(ExternalChatInboundEvent)
.values(
account_id=account_id,
platform=platform,
@ -33,11 +34,12 @@ async def persist_inbound_event(
external_message_id=external_message_id,
event_kind=event_kind,
raw_payload=raw_payload,
request_id=request_id,
)
.on_conflict_do_nothing(
index_elements=["account_id", "event_dedupe_key"],
)
.returning(GatewayInboundEvent.id)
.returning(ExternalChatInboundEvent.id)
)
result = await session.execute(stmt)
return result.scalar_one_or_none()

16
surfsense_backend/app/gateway/pairing.py
View file

@ -1,4 +1,4 @@
"""Pairing code lifecycle for gateway bindings."""
"""Pairing code lifecycle for external chat bindings."""
from __future__ import annotations
@ -8,7 +8,7 @@ from datetime import UTC, datetime, timedelta
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import GatewayBindingState, GatewayConversationBinding
from app.db import ExternalChatBindingState, ExternalChatBinding
PAIRING_CODE_TTL = timedelta(minutes=10)
@ -30,19 +30,19 @@ async def redeem_pairing_code(
external_display_name: str | None,
external_username: str | None,
external_metadata: dict | None = None,
) -> GatewayConversationBinding | None:
) -> ExternalChatBinding | None:
result = await session.execute(
select(GatewayConversationBinding).where(
GatewayConversationBinding.pairing_code == code,
GatewayConversationBinding.state == GatewayBindingState.PENDING,
GatewayConversationBinding.pairing_code_expires_at > datetime.now(UTC),
select(ExternalChatBinding).where(
ExternalChatBinding.pairing_code == code,
ExternalChatBinding.state == ExternalChatBindingState.PENDING,
ExternalChatBinding.pairing_code_expires_at > datetime.now(UTC),
)
)
binding = result.scalars().first()
if binding is None:
return None
binding.state = GatewayBindingState.BOUND
binding.state = ExternalChatBindingState.BOUND
binding.pairing_code = None
binding.pairing_code_expires_at = None
binding.external_peer_id = external_peer_id