Merge remote-tracking branch 'upstream/dev' into feat/composio

surfsense_backend/alembic/versions/74_no_op.py

@@ -0,0 +1,29 @@
+"""No-op migration for Composio support
+
+Revision ID: 74
+Revises: 73
+Create Date: 2026-01-21
+
+NOTE: This migration is a no-op since Composio is not supported yet.
+"""
+
+from collections.abc import Sequence
+
+# revision identifiers, used by Alembic.
+revision: str = "74"
+down_revision: str | None = "73"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """No-op upgrade for Composio support."""
+    pass
+
+
+def downgrade() -> None:
+    """No-op downgrade for Composio support.
+
+    Note: PostgreSQL does not support removing enum values directly.
+    """
+    pass

surfsense_backend/alembic/versions/75_add_chat_session_state_table.py

@@ -0,0 +1,75 @@
+"""Add chat_session_state table for live collaboration
+
+Revision ID: 75
+Revises: 74
+
+Creates chat_session_state table to track AI responding state per thread.
+Enables real-time sync via Electric SQL for shared chat collaboration.
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "75"
+down_revision: str | None = "74"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Create chat_session_state table with Electric SQL replication."""
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS chat_session_state (
+            id SERIAL PRIMARY KEY,
+            thread_id INTEGER NOT NULL REFERENCES new_chat_threads(id) ON DELETE CASCADE,
+            ai_responding_to_user_id UUID REFERENCES "user"(id) ON DELETE SET NULL,
+            updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+            UNIQUE (thread_id)
+        )
+        """
+    )
+
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS idx_chat_session_state_thread_id ON chat_session_state(thread_id)"
+    )
+
+    op.execute("ALTER TABLE chat_session_state REPLICA IDENTITY FULL;")
+
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF NOT EXISTS (
+                SELECT 1 FROM pg_publication_tables 
+                WHERE pubname = 'electric_publication_default' 
+                AND tablename = 'chat_session_state'
+            ) THEN
+                ALTER PUBLICATION electric_publication_default ADD TABLE chat_session_state;
+            END IF;
+        END
+        $$;
+        """
+    )
+
+
+def downgrade() -> None:
+    """Drop chat_session_state table and remove from Electric SQL replication."""
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF EXISTS (
+                SELECT 1 FROM pg_publication_tables 
+                WHERE pubname = 'electric_publication_default' 
+                AND tablename = 'chat_session_state'
+            ) THEN
+                ALTER PUBLICATION electric_publication_default DROP TABLE chat_session_state;
+            END IF;
+        END
+        $$;
+        """
+    )
+
+    op.execute("DROP TABLE IF EXISTS chat_session_state;")

surfsense_backend/alembic/versions/76_add_live_collaboration_tables_electric_replication.py

@@ -0,0 +1,99 @@
+"""Add live collaboration tables to Electric SQL publication
+
+Revision ID: 76
+Revises: 75
+
+Enables real-time sync for live collaboration features:
+- new_chat_messages: Live message sync between users
+- chat_comments: Live comment updates
+
+Note: User/member info is fetched via API (membersAtom) for client-side joins,
+not via Electric SQL, to keep where clauses optimized and reduce complexity.
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "76"
+down_revision: str | None = "75"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Add live collaboration tables to Electric SQL replication."""
+    # Set REPLICA IDENTITY FULL for Electric SQL sync
+    op.execute("ALTER TABLE new_chat_messages REPLICA IDENTITY FULL;")
+    op.execute("ALTER TABLE chat_comments REPLICA IDENTITY FULL;")
+
+    # Add new_chat_messages to Electric publication
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF NOT EXISTS (
+                SELECT 1 FROM pg_publication_tables 
+                WHERE pubname = 'electric_publication_default' 
+                AND tablename = 'new_chat_messages'
+            ) THEN
+                ALTER PUBLICATION electric_publication_default ADD TABLE new_chat_messages;
+            END IF;
+        END
+        $$;
+        """
+    )
+
+    # Add chat_comments to Electric publication
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF NOT EXISTS (
+                SELECT 1 FROM pg_publication_tables 
+                WHERE pubname = 'electric_publication_default' 
+                AND tablename = 'chat_comments'
+            ) THEN
+                ALTER PUBLICATION electric_publication_default ADD TABLE chat_comments;
+            END IF;
+        END
+        $$;
+        """
+    )
+
+
+def downgrade() -> None:
+    """Remove live collaboration tables from Electric SQL replication."""
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF EXISTS (
+                SELECT 1 FROM pg_publication_tables 
+                WHERE pubname = 'electric_publication_default' 
+                AND tablename = 'new_chat_messages'
+            ) THEN
+                ALTER PUBLICATION electric_publication_default DROP TABLE new_chat_messages;
+            END IF;
+        END
+        $$;
+        """
+    )
+
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF EXISTS (
+                SELECT 1 FROM pg_publication_tables 
+                WHERE pubname = 'electric_publication_default' 
+                AND tablename = 'chat_comments'
+            ) THEN
+                ALTER PUBLICATION electric_publication_default DROP TABLE chat_comments;
+            END IF;
+        END
+        $$;
+        """
+    )
+
+    # Note: Not reverting REPLICA IDENTITY as it doesn't harm normal operations

surfsense_backend/alembic/versions/77_add_thread_id_to_chat_comments.py

@@ -0,0 +1,70 @@
+"""Add thread_id to chat_comments for denormalized Electric subscriptions
+
+This denormalization allows a single Electric SQL subscription per thread
+instead of one per message, significantly reducing connection overhead.
+
+Revision ID: 77
+Revises: 76
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "77"
+down_revision: str | None = "76"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Add thread_id column to chat_comments and backfill from messages."""
+    # Add the column (nullable initially for backfill)
+    op.execute(
+        """
+        ALTER TABLE chat_comments
+        ADD COLUMN IF NOT EXISTS thread_id INTEGER;
+        """
+    )
+
+    # Backfill thread_id from the related message
+    op.execute(
+        """
+        UPDATE chat_comments c
+        SET thread_id = m.thread_id
+        FROM new_chat_messages m
+        WHERE c.message_id = m.id
+        AND c.thread_id IS NULL;
+        """
+    )
+
+    # Make it NOT NULL after backfill
+    op.execute(
+        """
+        ALTER TABLE chat_comments
+        ALTER COLUMN thread_id SET NOT NULL;
+        """
+    )
+
+    # Add FK constraint
+    op.execute(
+        """
+        ALTER TABLE chat_comments
+        ADD CONSTRAINT fk_chat_comments_thread_id
+        FOREIGN KEY (thread_id) REFERENCES new_chat_threads(id) ON DELETE CASCADE;
+        """
+    )
+
+    # Add index for efficient Electric subscriptions by thread
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS idx_chat_comments_thread_id ON chat_comments(thread_id)"
+    )
+
+
+def downgrade() -> None:
+    """Remove thread_id column from chat_comments."""
+    op.execute("DROP INDEX IF EXISTS idx_chat_comments_thread_id")
+    op.execute(
+        "ALTER TABLE chat_comments DROP CONSTRAINT IF EXISTS fk_chat_comments_thread_id"
+    )
+    op.execute("ALTER TABLE chat_comments DROP COLUMN IF EXISTS thread_id")

surfsense_backend/alembic/versions/78_add_obsidian_connector.py

@@ -0,0 +1,33 @@
+"""Add Obsidian connector enums
+
+Revision ID: 78
+Revises: 77
+Create Date: 2026-01-21
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "78"
+down_revision: str | None = "77"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # Add OBSIDIAN_CONNECTOR to documenttype enum
+    op.execute("ALTER TYPE documenttype ADD VALUE IF NOT EXISTS 'OBSIDIAN_CONNECTOR'")
+
+    # Add OBSIDIAN_CONNECTOR to searchsourceconnectortype enum
+    op.execute(
+        "ALTER TYPE searchsourceconnectortype ADD VALUE IF NOT EXISTS 'OBSIDIAN_CONNECTOR'"
+    )
+
+
+def downgrade() -> None:
+    # Note: PostgreSQL doesn't support removing enum values directly.
+    # The values will remain in the enum type but won't be used.
+    pass

surfsense_backend/alembic/versions/79_add_composio_connector_enums.py

@@ -1,8 +1,7 @@
 """Add Composio connector types to SearchSourceConnectorType and DocumentType enums
 
-Revision ID: 74
-Revises: 73
-Create Date: 2026-01-21
+Revision ID: 79
+Revises: 78
 
 This migration adds the Composio connector enum values to both:
 - searchsourceconnectortype (for connector type tracking)
@@ -23,8 +22,8 @@ from collections.abc import Sequence
 from alembic import op
 
 # revision identifiers, used by Alembic.
-revision: str = "74"
-down_revision: str | None = "73"
+revision: str = "79"
+down_revision: str | None = "78"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None