implement chat/podcast staleness

surfsense_backend/alembic/versions/32_add_podcast_staleness_detection.py

@@ -0,0 +1,42 @@
+"""Add podcast staleness detection columns
+
+Revision ID: 32
+Revises: 31
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers
+revision: str = "32"
+down_revision: str | None = "31"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Add state_version to chats table and chat_state_version to podcasts table."""
+
+    # Add state_version column to chats table with default value of 1
+    op.add_column(
+        "chats",
+        sa.Column("state_version", sa.BigInteger(), nullable=False, server_default="1"),
+    )
+
+    # Add chat_state_version column to podcasts table (nullable, set when podcast is generated)
+    op.add_column(
+        "podcasts", sa.Column("chat_state_version", sa.BigInteger(), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    """Remove state_version and chat_state_version columns."""
+
+    # Remove chat_state_version from podcasts table
+    op.drop_column("podcasts", "chat_state_version")
+
+    # Remove state_version from chats table
+    op.drop_column("chats", "state_version")

surfsense_backend/app/db.py

@@ -9,6 +9,7 @@ from sqlalchemy import (
     ARRAY,
     JSON,
     TIMESTAMP,
+    BigInteger,
     Boolean,
     Column,
     Enum as SQLAlchemyEnum,
@@ -157,6 +158,7 @@ class Chat(BaseModel, TimestampMixin):
     title = Column(String, nullable=False, index=True)
     initial_connectors = Column(ARRAY(String), nullable=True)
     messages = Column(JSON, nullable=False)
+    state_version = Column(BigInteger, nullable=False, default=1)
 
     search_space_id = Column(
         Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False
@@ -203,6 +205,7 @@ class Podcast(BaseModel, TimestampMixin):
     title = Column(String, nullable=False, index=True)
     podcast_transcript = Column(JSON, nullable=False, default={})
     file_location = Column(String(500), nullable=False, default="")
+    chat_state_version = Column(BigInteger, nullable=True)
 
     search_space_id = Column(
         Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False

surfsense_backend/app/routes/chats_routes.py

@@ -196,6 +196,7 @@ async def read_chats(
                 Chat.initial_connectors,
                 Chat.search_space_id,
                 Chat.created_at,
+                Chat.state_version,
             )
             .join(SearchSpace)
             .filter(SearchSpace.user_id == user.id)
@@ -259,6 +260,8 @@ async def update_chat(
         update_data = chat_update.model_dump(exclude_unset=True)
         for key, value in update_data.items():
             setattr(db_chat, key, value)
+        # Increment state_version when chat is modified
+        db_chat.state_version = (db_chat.state_version or 0) + 1
         await session.commit()
         await session.refresh(db_chat)
         return db_chat

surfsense_backend/app/schemas/chats.py

@@ -13,12 +13,14 @@ class ChatBase(BaseModel):
     initial_connectors: list[str] | None = None
     messages: list[Any]
     search_space_id: int
+    state_version: int = 1
 
 
 class ChatBaseWithoutMessages(BaseModel):
     type: ChatType
     title: str
     search_space_id: int
+    state_version: int = 1
 
 
 class ClientAttachment(BaseModel):

surfsense_backend/app/schemas/podcasts.py

@@ -10,6 +10,7 @@ class PodcastBase(BaseModel):
     podcast_transcript: list[Any]
     file_location: str = ""
     search_space_id: int
+    chat_state_version: int | None = None
 
 
 class PodcastCreate(PodcastBase):

surfsense_backend/app/tasks/podcast_tasks.py

@@ -144,6 +144,7 @@ async def generate_chat_podcast(
             podcast_transcript=serializable_transcript,
             file_location=result["final_podcast_file_path"],
             search_space_id=search_space_id,
+            chat_state_version=chat.state_version,
         )
 
         # Add to session and commit