Merge branch 'dev' of https://github.com/MODSetter/SurfSense into dev

surfsense_backend/alembic/versions/85_add_public_chat_snapshots_table.py

@@ -0,0 +1,179 @@
+"""Add public_chat_snapshots table and remove deprecated columns from new_chat_threads
+
+Revision ID: 85
+Revises: 84
+Create Date: 2026-01-29
+
+Changes:
+1. Create public_chat_snapshots table for immutable public chat sharing
+2. Drop deprecated columns from new_chat_threads:
+   - public_share_token (moved to snapshots)
+   - public_share_enabled (replaced by snapshot existence)
+   - clone_pending (single-phase clone)
+3. Drop related indexes
+4. Add cloned_from_snapshot_id to new_chat_threads (tracks source snapshot for clones)
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "85"
+down_revision: str | None = "84"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Create public_chat_snapshots table and remove deprecated columns."""
+
+    # 1. Create public_chat_snapshots table
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS public_chat_snapshots (
+            id SERIAL PRIMARY KEY,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+            
+            -- Link to original thread (CASCADE DELETE)
+            thread_id INTEGER NOT NULL
+                REFERENCES new_chat_threads(id) ON DELETE CASCADE,
+            
+            -- Public access token (unique URL identifier)
+            share_token VARCHAR(64) NOT NULL UNIQUE,
+            
+            -- SHA-256 hash of message content for deduplication
+            content_hash VARCHAR(64) NOT NULL,
+            
+            -- Immutable snapshot data (JSONB)
+            snapshot_data JSONB NOT NULL,
+            
+            -- Array of message IDs for cascade delete on edit
+            message_ids INTEGER[] NOT NULL,
+            
+            -- Who created this snapshot
+            created_by_user_id UUID REFERENCES "user"(id) ON DELETE SET NULL,
+            
+            -- Prevent duplicate snapshots of same content for same thread
+            CONSTRAINT uq_snapshot_thread_content_hash UNIQUE (thread_id, content_hash)
+        );
+        """
+    )
+
+    # 2. Create indexes for public_chat_snapshots
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS ix_public_chat_snapshots_thread_id
+        ON public_chat_snapshots(thread_id);
+        """
+    )
+
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS ix_public_chat_snapshots_share_token
+        ON public_chat_snapshots(share_token);
+        """
+    )
+
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS ix_public_chat_snapshots_content_hash
+        ON public_chat_snapshots(content_hash);
+        """
+    )
+
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS ix_public_chat_snapshots_created_by_user_id
+        ON public_chat_snapshots(created_by_user_id);
+        """
+    )
+
+    # 3. Create GIN index for message_ids array (for fast overlap queries)
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS ix_public_chat_snapshots_message_ids
+        ON public_chat_snapshots USING GIN(message_ids);
+        """
+    )
+
+    # 4. Drop deprecated indexes from new_chat_threads
+    op.execute("DROP INDEX IF EXISTS ix_new_chat_threads_public_share_enabled")
+    op.execute("DROP INDEX IF EXISTS ix_new_chat_threads_public_share_token")
+
+    # 5. Drop deprecated columns from new_chat_threads
+    op.execute("ALTER TABLE new_chat_threads DROP COLUMN IF EXISTS clone_pending")
+    op.execute("ALTER TABLE new_chat_threads DROP COLUMN IF EXISTS public_share_enabled")
+    op.execute("ALTER TABLE new_chat_threads DROP COLUMN IF EXISTS public_share_token")
+
+    # 6. Add cloned_from_snapshot_id to new_chat_threads
+    op.execute(
+        """
+        ALTER TABLE new_chat_threads
+        ADD COLUMN IF NOT EXISTS cloned_from_snapshot_id INTEGER
+            REFERENCES public_chat_snapshots(id) ON DELETE SET NULL;
+        """
+    )
+
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS ix_new_chat_threads_cloned_from_snapshot_id
+        ON new_chat_threads(cloned_from_snapshot_id)
+        WHERE cloned_from_snapshot_id IS NOT NULL;
+        """
+    )
+
+
+def downgrade() -> None:
+    """Restore deprecated columns and drop public_chat_snapshots table."""
+
+    # 1. Drop cloned_from_snapshot_id column and index
+    op.execute("DROP INDEX IF EXISTS ix_new_chat_threads_cloned_from_snapshot_id")
+    op.execute("ALTER TABLE new_chat_threads DROP COLUMN IF EXISTS cloned_from_snapshot_id")
+
+    # 2. Restore deprecated columns on new_chat_threads
+    op.execute(
+        """
+        ALTER TABLE new_chat_threads
+        ADD COLUMN IF NOT EXISTS public_share_token VARCHAR(64);
+        """
+    )
+
+    op.execute(
+        """
+        ALTER TABLE new_chat_threads
+        ADD COLUMN IF NOT EXISTS public_share_enabled BOOLEAN NOT NULL DEFAULT FALSE;
+        """
+    )
+
+    op.execute(
+        """
+        ALTER TABLE new_chat_threads
+        ADD COLUMN IF NOT EXISTS clone_pending BOOLEAN NOT NULL DEFAULT FALSE;
+        """
+    )
+
+    # 2. Restore indexes
+    op.execute(
+        """
+        CREATE UNIQUE INDEX IF NOT EXISTS ix_new_chat_threads_public_share_token
+        ON new_chat_threads(public_share_token)
+        WHERE public_share_token IS NOT NULL;
+        """
+    )
+
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS ix_new_chat_threads_public_share_enabled
+        ON new_chat_threads(public_share_enabled)
+        WHERE public_share_enabled = TRUE;
+        """
+    )
+
+    # 3. Drop public_chat_snapshots table and its indexes
+    op.execute("DROP INDEX IF EXISTS ix_public_chat_snapshots_message_ids")
+    op.execute("DROP INDEX IF EXISTS ix_public_chat_snapshots_created_by_user_id")
+    op.execute("DROP INDEX IF EXISTS ix_public_chat_snapshots_content_hash")
+    op.execute("DROP INDEX IF EXISTS ix_public_chat_snapshots_share_token")
+    op.execute("DROP INDEX IF EXISTS ix_public_chat_snapshots_thread_id")
+    op.execute("DROP TABLE IF EXISTS public_chat_snapshots")

surfsense_backend/app/agents/new_chat/llm_config.py

@@ -32,7 +32,7 @@ PROVIDER_MAP = {
     "GROQ": "groq",
     "COHERE": "cohere",
     "GOOGLE": "gemini",
-    "OLLAMA": "ollama",
+    "OLLAMA": "ollama_chat",
     "MISTRAL": "mistral",
     "AZURE_OPENAI": "azure",
     "OPENROUTER": "openrouter",

surfsense_backend/app/celery_app.py

@@ -79,6 +79,7 @@ celery_app = Celery(
         "app.tasks.celery_tasks.schedule_checker_task",
         "app.tasks.celery_tasks.blocknote_migration_tasks",
         "app.tasks.celery_tasks.document_reindex_tasks",
+        "app.tasks.celery_tasks.stale_notification_cleanup_task",
     ],
 )
 
@@ -121,4 +122,14 @@ celery_app.conf.beat_schedule = {
             "expires": 30,  # Task expires after 30 seconds if not picked up
         },
     },
+    # Cleanup stale connector indexing notifications every 5 minutes
+    # This detects tasks that crashed or timed out without proper cleanup
+    # and marks their notifications as failed so users don't see perpetual "syncing"
+    "cleanup-stale-indexing-notifications": {
+        "task": "cleanup_stale_indexing_notifications",
+        "schedule": crontab(minute="*/5"),  # Every 5 minutes
+        "options": {
+            "expires": 60,  # Task expires after 60 seconds if not picked up
+        },
+    },
 }

surfsense_backend/app/connectors/composio_gmail_connector.py

@@ -5,6 +5,8 @@ Provides Gmail specific methods for data retrieval and indexing via Composio.
 """
 
 import logging
+import time
+from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime
 from typing import Any
 
@@ -26,6 +28,10 @@ from app.utils.document_converters import (
     generate_unique_identifier_hash,
 )
 
+# Heartbeat configuration
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 logger = logging.getLogger(__name__)
 
 
@@ -427,6 +433,7 @@ async def index_composio_gmail(
     log_entry,
     update_last_indexed: bool = True,
     max_items: int = 1000,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str]:
     """Index Gmail messages via Composio with pagination and incremental processing."""
     try:
@@ -471,8 +478,16 @@ async def index_composio_gmail(
         total_documents_skipped = 0
         total_messages_fetched = 0
         result_size_estimate = None  # Will be set from first API response
+        last_heartbeat_time = time.time()
 
         while total_messages_fetched < max_items:
+            # Send heartbeat periodically to indicate task is still alive
+            if on_heartbeat_callback:
+                current_time = time.time()
+                if current_time - last_heartbeat_time >= HEARTBEAT_INTERVAL_SECONDS:
+                    await on_heartbeat_callback(total_documents_indexed)
+                    last_heartbeat_time = current_time
+
             # Calculate how many messages to fetch in this batch
             remaining = max_items - total_messages_fetched
             current_batch_size = min(batch_size, remaining)

surfsense_backend/app/connectors/composio_google_calendar_connector.py

@@ -5,6 +5,8 @@ Provides Google Calendar specific methods for data retrieval and indexing via Co
 """
 
 import logging
+import time
+from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime
 from typing import Any
 
@@ -29,6 +31,10 @@ from app.utils.document_converters import (
     generate_unique_identifier_hash,
 )
 
+# Heartbeat configuration
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 logger = logging.getLogger(__name__)
 
 
@@ -191,6 +197,7 @@ async def index_composio_google_calendar(
     log_entry,
     update_last_indexed: bool = True,
     max_items: int = 2500,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str]:
     """Index Google Calendar events via Composio."""
     try:
@@ -262,8 +269,15 @@ async def index_composio_google_calendar(
         duplicate_content_count = (
             0  # Track events skipped due to duplicate content_hash
         )
+        last_heartbeat_time = time.time()
 
         for event in events:
+            # Send heartbeat periodically to indicate task is still alive
+            if on_heartbeat_callback:
+                current_time = time.time()
+                if current_time - last_heartbeat_time >= HEARTBEAT_INTERVAL_SECONDS:
+                    await on_heartbeat_callback(documents_indexed)
+                    last_heartbeat_time = current_time
             try:
                 # Handle both standard Google API and potential Composio variations
                 event_id = event.get("id", "") or event.get("eventId", "")

surfsense_backend/app/connectors/composio_google_drive_connector.py

@@ -9,6 +9,8 @@ import json
 import logging
 import os
 import tempfile
+import time
+from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -29,6 +31,10 @@ from app.utils.document_converters import (
     generate_unique_identifier_hash,
 )
 
+# Heartbeat configuration
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 logger = logging.getLogger(__name__)
 
 
@@ -552,7 +558,9 @@ def generate_indexing_settings_hash(
         "include_subfolders": indexing_options.get("include_subfolders", True),
         "max_files_per_folder": indexing_options.get("max_files_per_folder", 100),
     }
-    return hashlib.md5(json.dumps(settings, sort_keys=True).encode()).hexdigest()
+    return hashlib.md5(
+        json.dumps(settings, sort_keys=True).encode(), usedforsecurity=False
+    ).hexdigest()
 
 
 async def index_composio_google_drive(
@@ -565,6 +573,7 @@ async def index_composio_google_drive(
     log_entry,
     update_last_indexed: bool = True,
     max_items: int = 1000,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, int, str | None]:
     """Index Google Drive files via Composio with delta sync support.
 
@@ -652,6 +661,7 @@ async def index_composio_google_drive(
                 max_items=max_items,
                 task_logger=task_logger,
                 log_entry=log_entry,
+                on_heartbeat_callback=on_heartbeat_callback,
             )
         else:
             logger.info(
@@ -684,6 +694,7 @@ async def index_composio_google_drive(
                 max_items=max_items,
                 task_logger=task_logger,
                 log_entry=log_entry,
+                on_heartbeat_callback=on_heartbeat_callback,
             )
 
         # Get new page token for next sync (always update after successful sync)
@@ -765,6 +776,7 @@ async def _index_composio_drive_delta_sync(
     max_items: int,
     task_logger: TaskLoggingService,
     log_entry,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, int, list[str]]:
     """Index Google Drive files using delta sync (only changed files).
 
@@ -774,6 +786,7 @@ async def _index_composio_drive_delta_sync(
     documents_indexed = 0
     documents_skipped = 0
     processing_errors = []
+    last_heartbeat_time = time.time()
 
     # Fetch all changes with pagination
     all_changes = []
@@ -804,6 +817,13 @@ async def _index_composio_drive_delta_sync(
     logger.info(f"Processing {len(all_changes)} changes from delta sync")
 
     for change in all_changes[:max_items]:
+        # Send heartbeat periodically to indicate task is still alive
+        if on_heartbeat_callback:
+            current_time = time.time()
+            if current_time - last_heartbeat_time >= HEARTBEAT_INTERVAL_SECONDS:
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = current_time
+
         try:
             # Handle removed files
             is_removed = change.get("removed", False)
@@ -886,11 +906,13 @@ async def _index_composio_drive_full_scan(
     max_items: int,
     task_logger: TaskLoggingService,
     log_entry,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, int, list[str]]:
     """Index Google Drive files using full scan (first sync or when no delta token)."""
     documents_indexed = 0
     documents_skipped = 0
     processing_errors = []
+    last_heartbeat_time = time.time()
 
     all_files = []
 
@@ -1001,6 +1023,13 @@ async def _index_composio_drive_full_scan(
     )
 
     for file_info in all_files:
+        # Send heartbeat periodically to indicate task is still alive
+        if on_heartbeat_callback:
+            current_time = time.time()
+            if current_time - last_heartbeat_time >= HEARTBEAT_INTERVAL_SECONDS:
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = current_time
+
         try:
             # Handle both standard Google API and potential Composio variations
             file_id = file_info.get("id", "") or file_info.get("fileId", "")

surfsense_backend/app/connectors/discord_connector.py

@@ -61,6 +61,9 @@ class DiscordConnector(commands.Bot):
             self.token = None
         self._bot_task = None  # Holds the async bot task
         self._is_running = False  # Flag to track if the bot is running
+        self._start_called_event = (
+            asyncio.Event()
+        )  # Event to signal when start() is called
 
         # Event to confirm bot is ready
         @self.event
@@ -226,6 +229,9 @@ class DiscordConnector(commands.Bot):
                 )
                 return
 
+            # Signal that we're about to call start() - this allows _wait_until_ready() to proceed
+            self._start_called_event.set()
+
             await self.start(self.token)
             logger.info("Discord bot started successfully.")
         except discord.LoginFailure:
@@ -260,6 +266,9 @@ class DiscordConnector(commands.Bot):
         else:
             logger.info("Bot is not running or already disconnected.")
 
+        # Reset the start event so the connector can be reused
+        self._start_called_event.clear()
+
     def set_token(self, token: str) -> None:
         """
         Set the discord bot token (for backward compatibility).
@@ -277,10 +286,16 @@ class DiscordConnector(commands.Bot):
         """Helper to wait until the bot is connected and ready."""
         logger.info("Waiting for the bot to be ready...")
 
-        # Give the event loop a chance to switch to the bot's startup task.
+        # Wait for start_bot() to actually call self.start()
-        # This allows self.start() to begin initializing the client.
+        # This ensures we don't call wait_until_ready() before the client is initialized
-        # Terrible solution, but necessary to avoid blocking the event loop.
+        try:
-        await asyncio.sleep(1)  # Yield control to the event loop
+            await asyncio.wait_for(self._start_called_event.wait(), timeout=30.0)
+            logger.info("Bot start() has been called, now waiting for ready state...")
+        except TimeoutError:
+            logger.error("start_bot() did not call start() within 30 seconds")
+            raise RuntimeError(
+                "Discord client failed to initialize - start() was never called"
+            ) from None
 
         try:
             await asyncio.wait_for(self.wait_until_ready(), timeout=60.0)

surfsense_backend/app/connectors/google_calendar_connector.py

@@ -252,12 +252,16 @@ class GoogleCalendarConnector:
             if dt_start.tzinfo is None:
                 dt_start = dt_start.replace(hour=0, minute=0, second=0, tzinfo=pytz.UTC)
             else:
-                dt_start = dt_start.astimezone(pytz.UTC).replace(hour=0, minute=0, second=0)
+                dt_start = dt_start.astimezone(pytz.UTC).replace(
+                    hour=0, minute=0, second=0
+                )
 
             if dt_end.tzinfo is None:
                 dt_end = dt_end.replace(hour=23, minute=59, second=59, tzinfo=pytz.UTC)
             else:
-                dt_end = dt_end.astimezone(pytz.UTC).replace(hour=23, minute=59, second=59)
+                dt_end = dt_end.astimezone(pytz.UTC).replace(
+                    hour=23, minute=59, second=59
+                )
 
             if dt_start >= dt_end:
                 return [], (

surfsense_backend/app/connectors/google_drive/credentials.py

@@ -132,6 +132,15 @@ async def get_valid_credentials(
             await session.commit()
 
         except Exception as e:
+            error_str = str(e)
+            # Check if this is an invalid_grant error (token expired/revoked)
+            if (
+                "invalid_grant" in error_str.lower()
+                or "token has been expired or revoked" in error_str.lower()
+            ):
+                raise Exception(
+                    "Google Drive authentication failed. Please re-authenticate."
+                ) from e
             raise Exception(f"Failed to refresh Google OAuth credentials: {e!s}") from e
 
     return credentials

surfsense_backend/app/db.py

@@ -411,21 +411,6 @@ class NewChatThread(BaseModel, TimestampMixin):
         index=True,
     )
 
-    # Public sharing - cryptographic token for public URL access
-    public_share_token = Column(
-        String(64),
-        nullable=True,
-        unique=True,
-        index=True,
-    )
-    # Whether public sharing is currently enabled for this thread
-    public_share_enabled = Column(
-        Boolean,
-        nullable=False,
-        default=False,
-        server_default="false",
-    )
-
     # Clone tracking - for audit and history bootstrap
     cloned_from_thread_id = Column(
         Integer,
@@ -433,6 +418,12 @@ class NewChatThread(BaseModel, TimestampMixin):
         nullable=True,
         index=True,
     )
+    cloned_from_snapshot_id = Column(
+        Integer,
+        ForeignKey("public_chat_snapshots.id", ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
     cloned_at = Column(
         TIMESTAMP(timezone=True),
         nullable=True,
@@ -444,13 +435,6 @@ class NewChatThread(BaseModel, TimestampMixin):
         default=False,
         server_default="false",
     )
-    # Flag indicating content clone is pending (two-phase clone)
-    clone_pending = Column(
-        Boolean,
-        nullable=False,
-        default=False,
-        server_default="false",
-    )
 
     # Relationships
     search_space = relationship("SearchSpace", back_populates="new_chat_threads")
@@ -461,6 +445,12 @@ class NewChatThread(BaseModel, TimestampMixin):
         order_by="NewChatMessage.created_at",
         cascade="all, delete-orphan",
     )
+    snapshots = relationship(
+        "PublicChatSnapshot",
+        back_populates="thread",
+        cascade="all, delete-orphan",
+        foreign_keys="[PublicChatSnapshot.thread_id]",
+    )
 
 
 class NewChatMessage(BaseModel, TimestampMixin):
@@ -501,6 +491,65 @@ class NewChatMessage(BaseModel, TimestampMixin):
     )
 
 
+class PublicChatSnapshot(BaseModel, TimestampMixin):
+    """
+    Immutable snapshot of a chat thread for public sharing.
+
+    Each snapshot is a frozen copy of the chat at a specific point in time.
+    The snapshot_data JSONB contains all messages and metadata needed to
+    render the public chat without querying the original thread.
+    """
+
+    __tablename__ = "public_chat_snapshots"
+
+    # Link to original thread - CASCADE DELETE when thread is deleted
+    thread_id = Column(
+        Integer,
+        ForeignKey("new_chat_threads.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+
+    # Public access token (unique URL identifier)
+    share_token = Column(
+        String(64),
+        nullable=False,
+        unique=True,
+        index=True,
+    )
+
+    content_hash = Column(
+        String(64),
+        nullable=False,
+        index=True,
+    )
+
+    snapshot_data = Column(JSONB, nullable=False)
+
+    message_ids = Column(ARRAY(Integer), nullable=False)
+
+    created_by_user_id = Column(
+        UUID(as_uuid=True),
+        ForeignKey("user.id", ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
+
+    # Relationships
+    thread = relationship(
+        "NewChatThread",
+        back_populates="snapshots",
+        foreign_keys="[PublicChatSnapshot.thread_id]",
+    )
+    created_by = relationship("User")
+
+    # Constraints
+    __table_args__ = (
+        # Prevent duplicate snapshots of the same content for the same thread
+        UniqueConstraint("thread_id", "content_hash", name="uq_snapshot_thread_content_hash"),
+    )
+
+
 class ChatComment(BaseModel, TimestampMixin):
     """
     Comment model for comments on AI chat responses.

surfsense_backend/app/routes/airtable_add_connector_route.py

@@ -442,11 +442,24 @@ async def refresh_airtable_token(
 
         if token_response.status_code != 200:
             error_detail = token_response.text
+            error_code = ""
             try:
                 error_json = token_response.json()
                 error_detail = error_json.get("error_description", error_detail)
+                error_code = error_json.get("error", "")
             except Exception:
                 pass
+            # Check if this is a token expiration/revocation error
+            error_lower = (error_detail + error_code).lower()
+            if (
+                "invalid_grant" in error_lower
+                or "expired" in error_lower
+                or "revoked" in error_lower
+            ):
+                raise HTTPException(
+                    status_code=401,
+                    detail="Airtable authentication failed. Please re-authenticate.",
+                )
             raise HTTPException(
                 status_code=400, detail=f"Token refresh failed: {error_detail}"
             )

surfsense_backend/app/routes/clickup_add_connector_route.py

@@ -417,6 +417,17 @@ async def refresh_clickup_token(
                 error_detail = error_json.get("error", error_detail)
             except Exception:
                 pass
+            # Check if this is a token expiration/revocation error
+            error_lower = error_detail.lower()
+            if (
+                "invalid_grant" in error_lower
+                or "expired" in error_lower
+                or "revoked" in error_lower
+            ):
+                raise HTTPException(
+                    status_code=401,
+                    detail="ClickUp authentication failed. Please re-authenticate.",
+                )
             raise HTTPException(
                 status_code=400, detail=f"Token refresh failed: {error_detail}"
             )

surfsense_backend/app/routes/confluence_add_connector_route.py

@@ -428,13 +428,26 @@ async def refresh_confluence_token(
 
         if token_response.status_code != 200:
             error_detail = token_response.text
+            error_code = ""
             try:
                 error_json = token_response.json()
                 error_detail = error_json.get(
                     "error_description", error_json.get("error", error_detail)
                 )
+                error_code = error_json.get("error", "")
             except Exception:
                 pass
+            # Check if this is a token expiration/revocation error
+            error_lower = (error_detail + error_code).lower()
+            if (
+                "invalid_grant" in error_lower
+                or "expired" in error_lower
+                or "revoked" in error_lower
+            ):
+                raise HTTPException(
+                    status_code=401,
+                    detail="Confluence authentication failed. Please re-authenticate.",
+                )
             raise HTTPException(
                 status_code=400, detail=f"Token refresh failed: {error_detail}"
             )

surfsense_backend/app/routes/discord_add_connector_route.py

@@ -46,6 +46,11 @@ SCOPES = [
     "guilds.members.read",  # Read member information
 ]
 
+# Discord permission bits
+VIEW_CHANNEL = 1 << 10  # 1024
+READ_MESSAGE_HISTORY = 1 << 16  # 65536
+ADMINISTRATOR = 1 << 3  # 8
+
 # Initialize security utilities
 _state_manager = None
 _token_encryption = None
@@ -531,3 +536,296 @@ async def refresh_discord_token(
         raise HTTPException(
             status_code=500, detail=f"Failed to refresh Discord tokens: {e!s}"
         ) from e
+
+
+def _compute_channel_permissions(
+    base_permissions: int,
+    bot_role_ids: set[str],
+    bot_user_id: str | None,
+    channel_overwrites: list[dict],
+    guild_id: str,
+) -> int:
+    """
+    Compute effective permissions for a channel based on role permissions and overwrites.
+
+    Discord permission computation follows this order (per official docs):
+    1. Start with base permissions from roles
+    2. Apply @everyone role overwrites (deny, then allow)
+    3. Apply role-specific overwrites (deny, then allow)
+    4. Apply member-specific overwrites (deny, then allow)
+
+    Args:
+        base_permissions: Combined permissions from all bot roles
+        bot_role_ids: Set of role IDs the bot has
+        bot_user_id: The bot's user ID for member-specific overwrites
+        channel_overwrites: List of permission overwrites for the channel
+        guild_id: Guild ID (same as @everyone role ID)
+
+    Returns:
+        Computed permission integer
+    """
+    permissions = base_permissions
+
+    # Permission overwrites are applied in order: @everyone, roles, member
+    everyone_allow = 0
+    everyone_deny = 0
+    role_allow = 0
+    role_deny = 0
+    member_allow = 0
+    member_deny = 0
+
+    for overwrite in channel_overwrites:
+        overwrite_id = overwrite.get("id")
+        overwrite_type = overwrite.get("type")  # 0 = role, 1 = member
+        allow = int(overwrite.get("allow", 0))
+        deny = int(overwrite.get("deny", 0))
+
+        if overwrite_type == 0:  # Role overwrite
+            if overwrite_id == guild_id:  # @everyone role
+                everyone_allow = allow
+                everyone_deny = deny
+            elif overwrite_id in bot_role_ids:
+                role_allow |= allow
+                role_deny |= deny
+        elif overwrite_type == 1 and bot_user_id and overwrite_id == bot_user_id:
+            # Member-specific overwrite for the bot
+            member_allow = allow
+            member_deny = deny
+
+    # Apply in order per Discord docs:
+    # 1. @everyone deny, then allow
+    permissions &= ~everyone_deny
+    permissions |= everyone_allow
+    # 2. Role deny, then allow
+    permissions &= ~role_deny
+    permissions |= role_allow
+    # 3. Member deny, then allow (applied LAST, highest priority)
+    permissions &= ~member_deny
+    permissions |= member_allow
+
+    return permissions
+
+
+@router.get("/discord/connector/{connector_id}/channels", response_model=None)
+async def get_discord_channels(
+    connector_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    """
+    Get list of Discord text channels for a connector with permission info.
+
+    Uses Discord's HTTP REST API directly instead of WebSocket bot connection.
+    Computes effective permissions to determine if bot can read message history.
+
+    Args:
+        connector_id: The Discord connector ID
+        session: Database session
+        user: Current authenticated user
+
+    Returns:
+        List of channels with id, name, type, position, category_id, and can_index fields
+    """
+    from sqlalchemy import select
+
+    try:
+        # Get connector and verify ownership
+        result = await session.execute(
+            select(SearchSourceConnector).where(
+                SearchSourceConnector.id == connector_id,
+                SearchSourceConnector.user_id == user.id,
+                SearchSourceConnector.connector_type
+                == SearchSourceConnectorType.DISCORD_CONNECTOR,
+            )
+        )
+        connector = result.scalar_one_or_none()
+
+        if not connector:
+            raise HTTPException(
+                status_code=404,
+                detail="Discord connector not found or access denied",
+            )
+
+        # Get credentials and decrypt bot token
+        credentials = DiscordAuthCredentialsBase.from_dict(connector.config)
+        token_encryption = get_token_encryption()
+        is_encrypted = connector.config.get("_token_encrypted", False)
+
+        bot_token = credentials.bot_token
+        if is_encrypted and bot_token:
+            try:
+                bot_token = token_encryption.decrypt_token(bot_token)
+            except Exception as e:
+                logger.error(f"Failed to decrypt bot token: {e!s}")
+                raise HTTPException(
+                    status_code=500, detail="Failed to decrypt stored bot token"
+                ) from e
+
+        if not bot_token:
+            raise HTTPException(
+                status_code=400,
+                detail="No bot token available. Please re-authenticate.",
+            )
+
+        # Get guild_id from connector config
+        guild_id = connector.config.get("guild_id")
+        if not guild_id:
+            raise HTTPException(
+                status_code=400,
+                detail="No guild_id associated with this connector. Please reconnect the Discord server.",
+            )
+
+        headers = {"Authorization": f"Bot {bot_token}"}
+
+        async with httpx.AsyncClient() as client:
+            # Fetch bot's user info to get bot user ID
+            bot_user_response = await client.get(
+                "https://discord.com/api/v10/users/@me",
+                headers=headers,
+                timeout=30.0,
+            )
+
+            if bot_user_response.status_code != 200:
+                logger.warning(
+                    f"Failed to fetch bot user info: {bot_user_response.text}"
+                )
+                bot_user_id = None
+            else:
+                bot_user_id = bot_user_response.json().get("id")
+
+            # Fetch guild info to get roles
+            guild_response = await client.get(
+                f"https://discord.com/api/v10/guilds/{guild_id}",
+                headers=headers,
+                timeout=30.0,
+            )
+
+            if guild_response.status_code != 200:
+                raise HTTPException(
+                    status_code=guild_response.status_code,
+                    detail="Failed to fetch guild information",
+                )
+
+            guild_data = guild_response.json()
+            guild_roles = {role["id"]: role for role in guild_data.get("roles", [])}
+
+            # Fetch bot's member info to get its roles
+            bot_member_response = await client.get(
+                f"https://discord.com/api/v10/guilds/{guild_id}/members/{bot_user_id}",
+                headers=headers,
+                timeout=30.0,
+            )
+
+            if bot_member_response.status_code != 200:
+                logger.warning(
+                    f"Failed to fetch bot member info: {bot_member_response.text}"
+                )
+                bot_role_ids = {guild_id}  # At minimum, bot has @everyone role
+                base_permissions = int(
+                    guild_roles.get(guild_id, {}).get("permissions", 0)
+                )
+            else:
+                bot_member_data = bot_member_response.json()
+                bot_role_ids = set(bot_member_data.get("roles", []))
+                bot_role_ids.add(guild_id)  # @everyone role is always included
+
+                # Compute base permissions from all bot roles
+                base_permissions = 0
+                for role_id in bot_role_ids:
+                    if role_id in guild_roles:
+                        role_perms = int(guild_roles[role_id].get("permissions", 0))
+                        base_permissions |= role_perms
+
+            # Check if bot has administrator permission (bypasses all checks)
+            is_admin = (base_permissions & ADMINISTRATOR) == ADMINISTRATOR
+
+            # Fetch channels
+            channels_response = await client.get(
+                f"https://discord.com/api/v10/guilds/{guild_id}/channels",
+                headers=headers,
+                timeout=30.0,
+            )
+
+        if channels_response.status_code == 403:
+            raise HTTPException(
+                status_code=403,
+                detail="Bot does not have permission to view channels in this server. Please ensure the bot has the 'View Channels' permission.",
+            )
+        elif channels_response.status_code == 404:
+            raise HTTPException(
+                status_code=404,
+                detail="Discord server not found. The bot may have been removed from the server.",
+            )
+        elif channels_response.status_code != 200:
+            error_detail = channels_response.text
+            try:
+                error_json = channels_response.json()
+                error_detail = error_json.get("message", error_detail)
+            except Exception:
+                pass
+            raise HTTPException(
+                status_code=channels_response.status_code,
+                detail=f"Failed to fetch Discord channels: {error_detail}",
+            )
+
+        channels_data = channels_response.json()
+
+        # Discord channel types:
+        # 0 = GUILD_TEXT, 2 = GUILD_VOICE, 4 = GUILD_CATEGORY, 5 = GUILD_ANNOUNCEMENT
+        # We want text channels (type 0) and announcement channels (type 5)
+        text_channel_types = {0, 5}
+
+        text_channels = []
+        for ch in channels_data:
+            if ch.get("type") in text_channel_types:
+                # Compute effective permissions for this channel
+                if is_admin:
+                    # Administrators bypass all permission checks
+                    can_index = True
+                else:
+                    channel_overwrites = ch.get("permission_overwrites", [])
+                    effective_perms = _compute_channel_permissions(
+                        base_permissions,
+                        bot_role_ids,
+                        bot_user_id,
+                        channel_overwrites,
+                        guild_id,
+                    )
+
+                    # Bot can index if it has both VIEW_CHANNEL and READ_MESSAGE_HISTORY
+                    has_view = (effective_perms & VIEW_CHANNEL) == VIEW_CHANNEL
+                    has_read_history = (
+                        effective_perms & READ_MESSAGE_HISTORY
+                    ) == READ_MESSAGE_HISTORY
+                    can_index = has_view and has_read_history
+
+                text_channels.append(
+                    {
+                        "id": ch["id"],
+                        "name": ch["name"],
+                        "type": "text" if ch["type"] == 0 else "announcement",
+                        "position": ch.get("position", 0),
+                        "category_id": ch.get("parent_id"),
+                        "can_index": can_index,
+                    }
+                )
+
+        # Sort by position
+        text_channels.sort(key=lambda x: x["position"])
+
+        logger.info(
+            f"Fetched {len(text_channels)} text channels for Discord connector {connector_id}"
+        )
+
+        return text_channels
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(
+            f"Failed to get Discord channels for connector {connector_id}: {e!s}",
+            exc_info=True,
+        )
+        raise HTTPException(
+            status_code=500, detail=f"Failed to get Discord channels: {e!s}"
+        ) from e

surfsense_backend/app/routes/jira_add_connector_route.py

@@ -446,13 +446,26 @@ async def refresh_jira_token(
 
         if token_response.status_code != 200:
             error_detail = token_response.text
+            error_code = ""
             try:
                 error_json = token_response.json()
                 error_detail = error_json.get(
                     "error_description", error_json.get("error", error_detail)
                 )
+                error_code = error_json.get("error", "")
             except Exception:
                 pass
+            # Check if this is a token expiration/revocation error
+            error_lower = (error_detail + error_code).lower()
+            if (
+                "invalid_grant" in error_lower
+                or "expired" in error_lower
+                or "revoked" in error_lower
+            ):
+                raise HTTPException(
+                    status_code=401,
+                    detail="Jira authentication failed. Please re-authenticate.",
+                )
             raise HTTPException(
                 status_code=400, detail=f"Token refresh failed: {error_detail}"
             )

surfsense_backend/app/routes/linear_add_connector_route.py

@@ -403,11 +403,24 @@ async def refresh_linear_token(
 
         if token_response.status_code != 200:
             error_detail = token_response.text
+            error_code = ""
             try:
                 error_json = token_response.json()
                 error_detail = error_json.get("error_description", error_detail)
+                error_code = error_json.get("error", "")
             except Exception:
                 pass
+            # Check if this is a token expiration/revocation error
+            error_lower = (error_detail + error_code).lower()
+            if (
+                "invalid_grant" in error_lower
+                or "expired" in error_lower
+                or "revoked" in error_lower
+            ):
+                raise HTTPException(
+                    status_code=401,
+                    detail="Linear authentication failed. Please re-authenticate.",
+                )
             raise HTTPException(
                 status_code=400, detail=f"Token refresh failed: {error_detail}"
             )

surfsense_backend/app/routes/new_chat_routes.py

@@ -37,7 +37,6 @@ from app.db import (
     get_async_session,
 )
 from app.schemas.new_chat import (
-    CompleteCloneResponse,
     NewChatMessageAppend,
     NewChatMessageRead,
     NewChatRequest,
@@ -46,14 +45,13 @@ from app.schemas.new_chat import (
     NewChatThreadUpdate,
     NewChatThreadVisibilityUpdate,
     NewChatThreadWithMessages,
-    PublicShareToggleRequest,
-    PublicShareToggleResponse,
     RegenerateRequest,
+    SnapshotCreateResponse,
+    SnapshotListResponse,
     ThreadHistoryLoadResponse,
     ThreadListItem,
     ThreadListResponse,
 )
-from app.services.public_chat_service import toggle_public_share
 from app.tasks.chat.stream_new_chat import stream_new_chat
 from app.users import current_active_user
 from app.utils.rbac import check_permission
@@ -219,7 +217,6 @@ async def list_threads(
                 visibility=thread.visibility,
                 created_by_id=thread.created_by_id,
                 is_own_thread=is_own_thread,
-                public_share_enabled=thread.public_share_enabled,
                 created_at=thread.created_at,
                 updated_at=thread.updated_at,
             )
@@ -321,7 +318,6 @@ async def search_threads(
                     thread.created_by_id == user.id
                     or (thread.created_by_id is None and is_search_space_owner)
                 ),
-                public_share_enabled=thread.public_share_enabled,
                 created_at=thread.created_at,
                 updated_at=thread.updated_at,
             )
@@ -670,66 +666,6 @@ async def delete_thread(
         ) from None
 
 
-@router.post(
-    "/threads/{thread_id}/complete-clone", response_model=CompleteCloneResponse
-)
-async def complete_clone(
-    thread_id: int,
-    session: AsyncSession = Depends(get_async_session),
-    user: User = Depends(current_active_user),
-):
-    """
-    Complete the cloning process for a thread.
-
-    Copies messages and podcasts from the source thread.
-    Sets clone_pending=False and needs_history_bootstrap=True when done.
-
-    Requires authentication and ownership of the thread.
-    """
-    from app.services.public_chat_service import complete_clone_content
-
-    try:
-        result = await session.execute(
-            select(NewChatThread).filter(NewChatThread.id == thread_id)
-        )
-        thread = result.scalars().first()
-
-        if not thread:
-            raise HTTPException(status_code=404, detail="Thread not found")
-
-        if thread.created_by_id != user.id:
-            raise HTTPException(status_code=403, detail="Not authorized")
-
-        if not thread.clone_pending:
-            raise HTTPException(status_code=400, detail="Clone already completed")
-
-        if not thread.cloned_from_thread_id:
-            raise HTTPException(
-                status_code=400, detail="No source thread to clone from"
-            )
-
-        message_count = await complete_clone_content(
-            session=session,
-            target_thread=thread,
-            source_thread_id=thread.cloned_from_thread_id,
-            target_search_space_id=thread.search_space_id,
-        )
-
-        return CompleteCloneResponse(
-            status="success",
-            message_count=message_count,
-        )
-
-    except HTTPException:
-        raise
-    except Exception as e:
-        await session.rollback()
-        raise HTTPException(
-            status_code=500,
-            detail=f"An unexpected error occurred while completing clone: {e!s}",
-        ) from None
-
-
 @router.patch("/threads/{thread_id}/visibility", response_model=NewChatThreadRead)
 async def update_thread_visibility(
     thread_id: int,
@@ -795,32 +731,83 @@ async def update_thread_visibility(
         ) from None
 
 
-@router.patch(
+# =============================================================================
-    "/threads/{thread_id}/public-share", response_model=PublicShareToggleResponse
+# Snapshot Endpoints
-)
+# =============================================================================
-async def update_thread_public_share(
+
+
+@router.post("/threads/{thread_id}/snapshots", response_model=SnapshotCreateResponse)
+async def create_thread_snapshot(
     thread_id: int,
     request: Request,
-    toggle_request: PublicShareToggleRequest,
     session: AsyncSession = Depends(get_async_session),
     user: User = Depends(current_active_user),
 ):
     """
-    Enable or disable public sharing for a thread.
+    Create a public snapshot of the thread.
 
-    Only the creator of the thread can manage public sharing.
+    Returns existing snapshot URL if content unchanged (deduplication).
-    When enabled, returns a public URL that anyone can use to view the chat.
+    Only the thread owner can create snapshots.
     """
+    from app.services.public_chat_service import create_snapshot
+
     base_url = str(request.base_url).rstrip("/")
-    return await toggle_public_share(
+    return await create_snapshot(
         session=session,
         thread_id=thread_id,
-        enabled=toggle_request.enabled,
         user=user,
         base_url=base_url,
     )
 
 
+@router.get("/threads/{thread_id}/snapshots", response_model=SnapshotListResponse)
+async def list_thread_snapshots(
+    thread_id: int,
+    request: Request,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    """
+    List all public snapshots for this thread.
+
+    Only the thread owner can view snapshots.
+    """
+    from app.services.public_chat_service import list_snapshots_for_thread
+
+    base_url = str(request.base_url).rstrip("/")
+    return SnapshotListResponse(
+        snapshots=await list_snapshots_for_thread(
+            session=session,
+            thread_id=thread_id,
+            user=user,
+            base_url=base_url,
+        )
+    )
+
+
+@router.delete("/threads/{thread_id}/snapshots/{snapshot_id}")
+async def delete_thread_snapshot(
+    thread_id: int,
+    snapshot_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    """
+    Delete a specific snapshot.
+
+    Only the thread owner can delete snapshots.
+    """
+    from app.services.public_chat_service import delete_snapshot
+
+    await delete_snapshot(
+        session=session,
+        thread_id=thread_id,
+        snapshot_id=snapshot_id,
+        user=user,
+    )
+    return {"message": "Snapshot deleted successfully"}
+
+
 # =============================================================================
 # Message Endpoints
 # =============================================================================
@@ -1286,6 +1273,8 @@ async def regenerate_response(
             .limit(2)
         )
         messages_to_delete = list(last_messages_result.scalars().all())
+        
+        message_ids_to_delete = [msg.id for msg in messages_to_delete]
 
         # Get search space for LLM config
         search_space_result = await session.execute(
@@ -1329,6 +1318,15 @@ async def regenerate_response(
                         for msg in messages_to_delete:
                             await session.delete(msg)
                         await session.commit()
+
+                        # Delete any public snapshots that contain the modified messages
+                        from app.services.public_chat_service import (
+                            delete_affected_snapshots,
+                        )
+
+                        await delete_affected_snapshots(
+                            session, thread_id, message_ids_to_delete
+                        )
                     except Exception as cleanup_error:
                         # Log but don't fail - the new messages are already streamed
                         print(

surfsense_backend/app/routes/notion_add_connector_route.py

@@ -407,11 +407,24 @@ async def refresh_notion_token(
 
         if token_response.status_code != 200:
             error_detail = token_response.text
+            error_code = ""
             try:
                 error_json = token_response.json()
                 error_detail = error_json.get("error_description", error_detail)
+                error_code = error_json.get("error", "")
             except Exception:
                 pass
+            # Check if this is a token expiration/revocation error
+            error_lower = (error_detail + error_code).lower()
+            if (
+                "invalid_grant" in error_lower
+                or "expired" in error_lower
+                or "revoked" in error_lower
+            ):
+                raise HTTPException(
+                    status_code=401,
+                    detail="Notion authentication failed. Please re-authenticate.",
+                )
             raise HTTPException(
                 status_code=400, detail=f"Token refresh failed: {error_detail}"
             )

surfsense_backend/app/routes/podcasts_routes.py

@@ -23,7 +23,7 @@ from app.db import (
     get_async_session,
 )
 from app.schemas import PodcastRead
-from app.users import current_active_user, current_optional_user
+from app.users import current_active_user
 from app.utils.rbac import check_permission
 
 router = APIRouter()
@@ -82,17 +82,14 @@ async def read_podcasts(
 async def read_podcast(
     podcast_id: int,
     session: AsyncSession = Depends(get_async_session),
-    user: User | None = Depends(current_optional_user),
+    user: User = Depends(current_active_user),
 ):
     """
     Get a specific podcast by ID.
 
-    Access is allowed if:
+    Requires authentication with PODCASTS_READ permission.
-    - User is authenticated with PODCASTS_READ permission, OR
+    For public podcast access, use /public/{share_token}/podcasts/{podcast_id}/stream
-    - Podcast belongs to a publicly shared thread
     """
-    from app.services.public_chat_service import is_podcast_publicly_accessible
-
     try:
         result = await session.execute(select(Podcast).filter(Podcast.id == podcast_id))
         podcast = result.scalars().first()
@@ -103,18 +100,13 @@ async def read_podcast(
                 detail="Podcast not found",
             )
 
-        is_public = await is_podcast_publicly_accessible(session, podcast_id)
+        await check_permission(
-
+            session,
-        if not is_public:
+            user,
-            if not user:
+            podcast.search_space_id,
-                raise HTTPException(status_code=401, detail="Authentication required")
+            Permission.PODCASTS_READ.value,
-            await check_permission(
+            "You don't have permission to read podcasts in this search space",
-                session,
+        )
-                user,
-                podcast.search_space_id,
-                Permission.PODCASTS_READ.value,
-                "You don't have permission to read podcasts in this search space",
-            )
 
         return PodcastRead.from_orm_with_entries(podcast)
     except HTTPException as he:
@@ -168,19 +160,16 @@ async def delete_podcast(
 async def stream_podcast(
     podcast_id: int,
     session: AsyncSession = Depends(get_async_session),
-    user: User | None = Depends(current_optional_user),
+    user: User = Depends(current_active_user),
 ):
     """
     Stream a podcast audio file.
 
-    Access is allowed if:
+    Requires authentication with PODCASTS_READ permission.
-    - User is authenticated with PODCASTS_READ permission, OR
+    For public podcast access, use /public/{share_token}/podcasts/{podcast_id}/stream
-    - Podcast belongs to a publicly shared thread
 
     Note: Both /stream and /audio endpoints are supported for compatibility.
     """
-    from app.services.public_chat_service import is_podcast_publicly_accessible
-
     try:
         result = await session.execute(select(Podcast).filter(Podcast.id == podcast_id))
         podcast = result.scalars().first()
@@ -188,19 +177,13 @@ async def stream_podcast(
         if not podcast:
             raise HTTPException(status_code=404, detail="Podcast not found")
 
-        is_public = await is_podcast_publicly_accessible(session, podcast_id)
+        await check_permission(
-
+            session,
-        if not is_public:
+            user,
-            if not user:
+            podcast.search_space_id,
-                raise HTTPException(status_code=401, detail="Authentication required")
+            Permission.PODCASTS_READ.value,
-
+            "You don't have permission to access podcasts in this search space",
-            await check_permission(
+        )
-                session,
-                user,
-                podcast.search_space_id,
-                Permission.PODCASTS_READ.value,
-                "You don't have permission to access podcasts in this search space",
-            )
 
         file_path = podcast.file_location
 

surfsense_backend/app/routes/public_chat_routes.py

@@ -1,21 +1,25 @@
 """
-Routes for public chat access (unauthenticated and mixed-auth endpoints).
+Routes for public chat access via immutable snapshots.
+
+All public endpoints use share_token for access - no authentication required
+for read operations. Clone requires authentication.
 """
 
-from datetime import UTC, datetime
+import os
 
 from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import StreamingResponse
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db import ChatVisibility, NewChatThread, User, get_async_session
+from app.db import User, get_async_session
 from app.schemas.new_chat import (
-    CloneInitResponse,
+    CloneResponse,
     PublicChatResponse,
 )
 from app.services.public_chat_service import (
+    clone_from_snapshot,
     get_public_chat,
-    get_thread_by_share_token,
+    get_snapshot_podcast,
-    get_user_default_search_space,
 )
 from app.users import current_active_user
 
@@ -28,57 +32,85 @@ async def read_public_chat(
     session: AsyncSession = Depends(get_async_session),
 ):
     """
-    Get a public chat by share token.
+    Get a public chat snapshot by share token.
 
     No authentication required.
-    Returns sanitized content (citations stripped).
+    Returns immutable snapshot data (sanitized, citations stripped).
     """
     return await get_public_chat(session, share_token)
 
 
-@router.post("/{share_token}/clone", response_model=CloneInitResponse)
+@router.post("/{share_token}/clone", response_model=CloneResponse)
-async def clone_public_chat_endpoint(
+async def clone_public_chat(
     share_token: str,
     session: AsyncSession = Depends(get_async_session),
     user: User = Depends(current_active_user),
 ):
     """
-    Initialize cloning a public chat to the user's account.
+    Clone a public chat snapshot to the user's account.
-
-    Creates an empty thread with clone_pending=True.
-    Frontend should redirect to the new thread and call /complete-clone.
 
+    Creates thread and copies messages.
     Requires authentication.
     """
-    source_thread = await get_thread_by_share_token(session, share_token)
+    return await clone_from_snapshot(session, share_token, user)
 
-    if not source_thread:
-        raise HTTPException(
-            status_code=404, detail="Chat not found or no longer public"
-        )
 
-    target_search_space_id = await get_user_default_search_space(session, user.id)
+@router.get("/{share_token}/podcasts/{podcast_id}")
+async def get_public_podcast(
+    share_token: str,
+    podcast_id: int,
+    session: AsyncSession = Depends(get_async_session),
+):
+    """
+    Get podcast details from a public chat snapshot.
 
-    if target_search_space_id is None:
+    No authentication required - the share_token provides access.
-        raise HTTPException(status_code=400, detail="No search space found for user")
+    Returns podcast info including transcript.
+    """
+    podcast_info = await get_snapshot_podcast(session, share_token, podcast_id)
 
-    new_thread = NewChatThread(
+    if not podcast_info:
-        title=source_thread.title,
+        raise HTTPException(status_code=404, detail="Podcast not found")
-        archived=False,
+
-        visibility=ChatVisibility.PRIVATE,
+    return {
-        search_space_id=target_search_space_id,
+        "id": podcast_info.get("original_id"),
-        created_by_id=user.id,
+        "title": podcast_info.get("title"),
-        public_share_enabled=False,
+        "status": "ready",
-        cloned_from_thread_id=source_thread.id,
+        "podcast_transcript": podcast_info.get("transcript"),
-        cloned_at=datetime.now(UTC),
+    }
-        clone_pending=True,
+
-    )
+
-    session.add(new_thread)
+@router.get("/{share_token}/podcasts/{podcast_id}/stream")
-    await session.commit()
+async def stream_public_podcast(
-    await session.refresh(new_thread)
+    share_token: str,
-
+    podcast_id: int,
-    return CloneInitResponse(
+    session: AsyncSession = Depends(get_async_session),
-        thread_id=new_thread.id,
+):
-        search_space_id=target_search_space_id,
+    """
-        share_token=share_token,
+    Stream a podcast from a public chat snapshot.
+
+    No authentication required - the share_token provides access.
+    Looks up podcast by original_id in the snapshot's podcasts array.
+    """
+    podcast_info = await get_snapshot_podcast(session, share_token, podcast_id)
+
+    if not podcast_info:
+        raise HTTPException(status_code=404, detail="Podcast not found")
+
+    file_path = podcast_info.get("file_path")
+
+    if not file_path or not os.path.isfile(file_path):
+        raise HTTPException(status_code=404, detail="Podcast audio file not found")
+
+    def iterfile():
+        with open(file_path, mode="rb") as file_like:
+            yield from file_like
+
+    return StreamingResponse(
+        iterfile(),
+        media_type="audio/mpeg",
+        headers={
+            "Accept-Ranges": "bytes",
+            "Content-Disposition": f"inline; filename={os.path.basename(file_path)}",
+        },
     )

surfsense_backend/app/routes/search_source_connectors_routes.py

@@ -19,10 +19,12 @@ Non-OAuth connectors (BookStack, GitHub, etc.) are limited to one per search spa
 """
 
 import logging
+import os
 from datetime import UTC, datetime, timedelta
 from typing import Any
 
 import pytz
+import redis
 from dateutil.parser import isoparse
 from fastapi import APIRouter, Body, Depends, HTTPException, Query
 from pydantic import BaseModel, Field, ValidationError
@@ -78,6 +80,27 @@ from app.utils.rbac import check_permission
 # Set up logging
 logger = logging.getLogger(__name__)
 
+# Redis client for heartbeat tracking
+_heartbeat_redis_client: redis.Redis | None = None
+
+# Redis key TTL - notification is stale if no heartbeat in this time
+HEARTBEAT_TTL_SECONDS = 120  # 2 minutes
+
+
+def get_heartbeat_redis_client() -> redis.Redis:
+    """Get or create Redis client for heartbeat tracking."""
+    global _heartbeat_redis_client
+    if _heartbeat_redis_client is None:
+        redis_url = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
+        _heartbeat_redis_client = redis.from_url(redis_url, decode_responses=True)
+    return _heartbeat_redis_client
+
+
+def _get_heartbeat_key(notification_id: int) -> str:
+    """Generate Redis key for notification heartbeat."""
+    return f"indexing:heartbeat:{notification_id}"
+
+
 router = APIRouter()
 
 
@@ -1137,6 +1160,7 @@ async def run_slack_indexing(
         end_date=end_date,
         indexing_function=index_slack_messages,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -1150,6 +1174,7 @@ async def _run_indexing_with_notifications(
     indexing_function,
     update_timestamp_func=None,
     supports_retry_callback: bool = False,
+    supports_heartbeat_callback: bool = False,
 ):
     """
     Generic helper to run indexing with real-time notifications.
@@ -1164,11 +1189,14 @@ async def _run_indexing_with_notifications(
         indexing_function: Async function that performs the indexing
         update_timestamp_func: Optional function to update connector timestamp
         supports_retry_callback: Whether the indexing function supports on_retry_callback
+        supports_heartbeat_callback: Whether the indexing function supports on_heartbeat_callback
     """
     from uuid import UUID
 
+    from celery.exceptions import SoftTimeLimitExceeded
+
     notification = None
-    # Track indexed count for retry notifications
+    # Track indexed count for retry notifications and heartbeat
     current_indexed_count = 0
 
     try:
@@ -1195,6 +1223,16 @@ async def _run_indexing_with_notifications(
                 )
             )
 
+            # Set initial Redis heartbeat for stale detection
+            if notification:
+                try:
+                    heartbeat_key = _get_heartbeat_key(notification.id)
+                    get_heartbeat_redis_client().setex(
+                        heartbeat_key, HEARTBEAT_TTL_SECONDS, "0"
+                    )
+                except Exception as e:
+                    logger.warning(f"Failed to set initial Redis heartbeat: {e}")
+
         # Update notification to fetching stage
         if notification:
             await NotificationService.connector_indexing.notify_indexing_progress(
@@ -1227,6 +1265,40 @@ async def _run_indexing_with_notifications(
                     # Don't let notification errors break the indexing
                     logger.warning(f"Failed to update retry notification: {e}")
 
+        # Create heartbeat callback for connectors that support it
+        # This updates the notification periodically during long-running indexing loops
+        # to prevent the task from appearing stuck if the worker crashes
+        async def on_heartbeat_callback(indexed_count: int) -> None:
+            """Callback to update notification during indexing (heartbeat)."""
+            nonlocal notification, current_indexed_count
+            current_indexed_count = indexed_count
+            if notification:
+                try:
+                    # Set Redis heartbeat key with TTL (fast, for stale detection)
+                    heartbeat_key = _get_heartbeat_key(notification.id)
+                    get_heartbeat_redis_client().setex(
+                        heartbeat_key, HEARTBEAT_TTL_SECONDS, str(indexed_count)
+                    )
+                except Exception as e:
+                    # Don't let Redis errors break the indexing
+                    logger.warning(f"Failed to set Redis heartbeat: {e}")
+
+                try:
+                    # Still update DB notification for progress display
+                    await session.refresh(notification)
+                    await (
+                        NotificationService.connector_indexing.notify_indexing_progress(
+                            session=session,
+                            notification=notification,
+                            indexed_count=indexed_count,
+                            stage="processing",
+                        )
+                    )
+                    await session.commit()
+                except Exception as e:
+                    # Don't let notification errors break the indexing
+                    logger.warning(f"Failed to update heartbeat notification: {e}")
+
         # Build kwargs for indexing function
         indexing_kwargs = {
             "session": session,
@@ -1242,6 +1314,10 @@ async def _run_indexing_with_notifications(
         if supports_retry_callback:
             indexing_kwargs["on_retry_callback"] = on_retry_callback
 
+        # Add heartbeat callback for connectors that support it
+        if supports_heartbeat_callback:
+            indexing_kwargs["on_heartbeat_callback"] = on_heartbeat_callback
+
         # Run the indexing function
         # Some indexers return (indexed, error), others return (indexed, skipped, error)
         result = await indexing_function(**indexing_kwargs)
@@ -1398,6 +1474,32 @@ async def _run_indexing_with_notifications(
                     await (
                         session.commit()
                     )  # Commit to ensure Electric SQL syncs the notification update
+    except SoftTimeLimitExceeded:
+        # Celery soft time limit was reached - task is about to be killed
+        # Gracefully save progress and mark as interrupted
+        logger.warning(
+            f"Soft time limit reached for connector {connector_id}. "
+            f"Saving partial progress: {current_indexed_count} items indexed."
+        )
+
+        if notification:
+            try:
+                await session.refresh(notification)
+                await NotificationService.connector_indexing.notify_indexing_completed(
+                    session=session,
+                    notification=notification,
+                    indexed_count=current_indexed_count,
+                    error_message="Time limit reached. Partial sync completed. Please run again for remaining items.",
+                    is_warning=True,  # Mark as warning since partial data was indexed
+                )
+                await session.commit()
+            except Exception as notif_error:
+                logger.error(
+                    f"Failed to update notification on soft timeout: {notif_error!s}"
+                )
+
+        # Re-raise so Celery knows the task was terminated
+        raise
     except Exception as e:
         logger.error(f"Error in indexing task: {e!s}", exc_info=True)
 
@@ -1409,12 +1511,20 @@ async def _run_indexing_with_notifications(
                 await NotificationService.connector_indexing.notify_indexing_completed(
                     session=session,
                     notification=notification,
-                    indexed_count=0,
+                    indexed_count=current_indexed_count,  # Use tracked count, not 0
                     error_message=str(e),
                     skipped_count=None,  # Unknown on exception
                 )
             except Exception as notif_error:
                 logger.error(f"Failed to update notification: {notif_error!s}")
+    finally:
+        # Clean up Redis heartbeat key when task completes (success or failure)
+        if notification:
+            try:
+                heartbeat_key = _get_heartbeat_key(notification.id)
+                get_heartbeat_redis_client().delete(heartbeat_key)
+            except Exception:
+                pass  # Ignore cleanup errors - key will expire anyway
 
 
 async def run_notion_indexing_with_new_session(
@@ -1439,6 +1549,7 @@ async def run_notion_indexing_with_new_session(
             indexing_function=index_notion_pages,
             update_timestamp_func=_update_connector_timestamp_by_id,
             supports_retry_callback=True,  # Notion connector supports retry notifications
+            supports_heartbeat_callback=True,  # Notion connector supports heartbeat notifications
         )
 
 
@@ -1471,6 +1582,7 @@ async def run_notion_indexing(
         indexing_function=index_notion_pages,
         update_timestamp_func=_update_connector_timestamp_by_id,
         supports_retry_callback=True,  # Notion connector supports retry notifications
+        supports_heartbeat_callback=True,  # Notion connector supports heartbeat notifications
     )
 
 
@@ -1521,6 +1633,7 @@ async def run_github_indexing(
         end_date=end_date,
         indexing_function=index_github_repos,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -1571,6 +1684,7 @@ async def run_linear_indexing(
         end_date=end_date,
         indexing_function=index_linear_issues,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -1620,6 +1734,7 @@ async def run_discord_indexing(
         end_date=end_date,
         indexing_function=index_discord_messages,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -1670,6 +1785,7 @@ async def run_teams_indexing(
         end_date=end_date,
         indexing_function=index_teams_messages,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -1720,6 +1836,7 @@ async def run_jira_indexing(
         end_date=end_date,
         indexing_function=index_jira_issues,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -1772,6 +1889,7 @@ async def run_confluence_indexing(
         end_date=end_date,
         indexing_function=index_confluence_pages,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -1822,6 +1940,7 @@ async def run_clickup_indexing(
         end_date=end_date,
         indexing_function=index_clickup_tasks,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -1872,6 +1991,7 @@ async def run_airtable_indexing(
         end_date=end_date,
         indexing_function=index_airtable_records,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -1924,6 +2044,7 @@ async def run_google_calendar_indexing(
         end_date=end_date,
         indexing_function=index_google_calendar_events,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -1998,6 +2119,7 @@ async def run_google_gmail_indexing(
         end_date=end_date,
         indexing_function=gmail_indexing_wrapper,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -2206,6 +2328,7 @@ async def run_luma_indexing(
         end_date=end_date,
         indexing_function=index_luma_events,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -2257,6 +2380,7 @@ async def run_elasticsearch_indexing(
         end_date=end_date,
         indexing_function=index_elasticsearch_documents,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -2306,6 +2430,7 @@ async def run_web_page_indexing(
         end_date=end_date,
         indexing_function=index_crawled_urls,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -2360,6 +2485,7 @@ async def run_bookstack_indexing(
         end_date=end_date,
         indexing_function=index_bookstack_pages,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -2412,6 +2538,7 @@ async def run_obsidian_indexing(
         end_date=end_date,
         indexing_function=index_obsidian_vault,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 
@@ -2465,6 +2592,7 @@ async def run_composio_indexing(
         end_date=end_date,
         indexing_function=index_composio_connector,
         update_timestamp_func=_update_connector_timestamp_by_id,
+        supports_heartbeat_callback=True,
     )
 
 

surfsense_backend/app/routes/slack_add_connector_route.py

@@ -6,6 +6,7 @@ Handles OAuth 2.0 authentication flow for Slack connector.
 
 import logging
 from datetime import UTC, datetime, timedelta
+from typing import Any
 from uuid import UUID
 
 import httpx
@@ -14,6 +15,7 @@ from fastapi.responses import RedirectResponse
 from pydantic import ValidationError
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.future import select
 
 from app.config import config
 from app.db import (
@@ -418,6 +420,19 @@ async def refresh_slack_token(
                 error_detail = error_json.get("error", error_detail)
             except Exception:
                 pass
+            # Check if this is a token expiration/revocation error
+            error_lower = error_detail.lower()
+            if (
+                "invalid_grant" in error_lower
+                or "invalid_auth" in error_lower
+                or "token_revoked" in error_lower
+                or "expired" in error_lower
+                or "revoked" in error_lower
+            ):
+                raise HTTPException(
+                    status_code=401,
+                    detail="Slack authentication failed. Please re-authenticate.",
+                )
             raise HTTPException(
                 status_code=400, detail=f"Token refresh failed: {error_detail}"
             )
@@ -427,6 +442,20 @@ async def refresh_slack_token(
         # Slack OAuth v2 returns success status in the JSON
         if not token_json.get("ok", False):
             error_msg = token_json.get("error", "Unknown error")
+            # Check if this is a token expiration/revocation error
+            error_lower = error_msg.lower()
+            if (
+                "invalid_grant" in error_lower
+                or "invalid_auth" in error_lower
+                or "invalid_refresh_token" in error_lower
+                or "token_revoked" in error_lower
+                or "expired" in error_lower
+                or "revoked" in error_lower
+            ):
+                raise HTTPException(
+                    status_code=401,
+                    detail="Slack authentication failed. Please re-authenticate.",
+                )
             raise HTTPException(
                 status_code=400, detail=f"Slack OAuth refresh error: {error_msg}"
             )
@@ -490,3 +519,88 @@ async def refresh_slack_token(
         raise HTTPException(
             status_code=500, detail=f"Failed to refresh Slack token: {e!s}"
         ) from e
+
+
+@router.get("/slack/connector/{connector_id}/channels")
+async def get_slack_channels(
+    connector_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+) -> list[dict[str, Any]]:
+    """
+    Get list of Slack channels with bot membership status.
+
+    This endpoint fetches all channels the bot can see and indicates
+    whether the bot is a member of each channel (required for accessing messages).
+
+    Args:
+        connector_id: The Slack connector ID
+        session: Database session
+        user: Current authenticated user
+
+    Returns:
+        List of channels with id, name, is_private, and is_member fields
+    """
+    try:
+        # Get the connector and verify ownership
+        result = await session.execute(
+            select(SearchSourceConnector).where(
+                SearchSourceConnector.id == connector_id,
+                SearchSourceConnector.user_id == user.id,
+                SearchSourceConnector.connector_type
+                == SearchSourceConnectorType.SLACK_CONNECTOR,
+            )
+        )
+        connector = result.scalar_one_or_none()
+
+        if not connector:
+            raise HTTPException(
+                status_code=404,
+                detail="Slack connector not found or access denied",
+            )
+
+        # Get credentials and decrypt bot token
+        credentials = SlackAuthCredentialsBase.from_dict(connector.config)
+        token_encryption = get_token_encryption()
+        is_encrypted = connector.config.get("_token_encrypted", False)
+
+        bot_token = credentials.bot_token
+        if is_encrypted and bot_token:
+            try:
+                bot_token = token_encryption.decrypt_token(bot_token)
+            except Exception as e:
+                logger.error(f"Failed to decrypt bot token: {e!s}")
+                raise HTTPException(
+                    status_code=500, detail="Failed to decrypt stored bot token"
+                ) from e
+
+        if not bot_token:
+            raise HTTPException(
+                status_code=400,
+                detail="No bot token available. Please re-authenticate.",
+            )
+
+        # Import SlackHistory here to avoid circular imports
+        from app.connectors.slack_history import SlackHistory
+
+        # Create Slack client with direct token (simple pattern for quick operations)
+        slack_client = SlackHistory(token=bot_token)
+
+        channels = await slack_client.get_all_channels(include_private=True)
+
+        logger.info(
+            f"Fetched {len(channels)} channels for Slack connector {connector_id}"
+        )
+
+        return channels
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(
+            f"Failed to get Slack channels for connector {connector_id}: {e!s}",
+            exc_info=True,
+        )
+        raise HTTPException(
+            status_code=500, detail=f"Failed to get Slack channels: {e!s}"
+        ) from e

surfsense_backend/app/routes/teams_add_connector_route.py

@@ -420,11 +420,24 @@ async def refresh_teams_token(
 
     if token_response.status_code != 200:
         error_detail = token_response.text
+        error_code = ""
         try:
             error_json = token_response.json()
             error_detail = error_json.get("error_description", error_detail)
+            error_code = error_json.get("error", "")
         except Exception:
             pass
+        # Check if this is a token expiration/revocation error
+        error_lower = (error_detail + error_code).lower()
+        if (
+            "invalid_grant" in error_lower
+            or "expired" in error_lower
+            or "revoked" in error_lower
+        ):
+            raise HTTPException(
+                status_code=401,
+                detail="Microsoft Teams authentication failed. Please re-authenticate.",
+            )
         raise HTTPException(
             status_code=400, detail=f"Token refresh failed: {error_detail}"
         )

surfsense_backend/app/schemas/new_chat.py

@@ -95,9 +95,6 @@ class NewChatThreadRead(NewChatThreadBase, IDModel):
     search_space_id: int
     visibility: ChatVisibility
     created_by_id: UUID | None = None
-    public_share_enabled: bool = False
-    public_share_token: str | None = None
-    clone_pending: bool = False
     created_at: datetime
     updated_at: datetime
 
@@ -137,7 +134,6 @@ class ThreadListItem(BaseModel):
     visibility: ChatVisibility
     created_by_id: UUID | None = None
     is_own_thread: bool = False
-    public_share_enabled: bool = False
     created_at: datetime = Field(alias="createdAt")
     updated_at: datetime = Field(alias="updatedAt")
 
@@ -211,22 +207,33 @@ class RegenerateRequest(BaseModel):
 
 
 # =============================================================================
-# Public Sharing Schemas
+# Public Chat Snapshot Schemas
 # =============================================================================
 
 
-class PublicShareToggleRequest(BaseModel):
+class SnapshotCreateResponse(BaseModel):
-    """Request to enable/disable public sharing for a thread."""
+    """Response after creating a public snapshot."""
 
-    enabled: bool
+    snapshot_id: int
+    share_token: str
+    public_url: str
+    is_new: bool  # False if existing snapshot returned (same content)
 
 
-class PublicShareToggleResponse(BaseModel):
+class SnapshotInfo(BaseModel):
-    """Response after toggling public sharing."""
+    """Info about a single snapshot."""
 
-    enabled: bool
+    id: int
-    public_url: str | None = None
+    share_token: str
-    share_token: str | None = None
+    public_url: str
+    created_at: datetime
+    message_count: int
+
+
+class SnapshotListResponse(BaseModel):
+    """List of snapshots for a thread."""
+
+    snapshots: list[SnapshotInfo]
 
 
 # =============================================================================
@@ -256,12 +263,8 @@ class PublicChatResponse(BaseModel):
     messages: list[PublicChatMessage]
 
 
-class CloneInitResponse(BaseModel):
+class CloneResponse(BaseModel):
+    """Response after cloning a public snapshot."""
+
     thread_id: int
     search_space_id: int
-    share_token: str
-
-
-class CompleteCloneResponse(BaseModel):
-    status: str
-    message_count: int

surfsense_backend/app/services/llm_router_service.py

@@ -32,7 +32,7 @@ PROVIDER_MAP = {
     "GROQ": "groq",
     "COHERE": "cohere",
     "GOOGLE": "gemini",
-    "OLLAMA": "ollama",
+    "OLLAMA": "ollama_chat",
     "MISTRAL": "mistral",
     "AZURE_OPENAI": "azure",
     "OPENROUTER": "openrouter",

surfsense_backend/app/services/llm_service.py

@@ -94,7 +94,7 @@ async def validate_llm_config(
                 "GROQ": "groq",
                 "COHERE": "cohere",
                 "GOOGLE": "gemini",
-                "OLLAMA": "ollama",
+                "OLLAMA": "ollama_chat",
                 "MISTRAL": "mistral",
                 "AZURE_OPENAI": "azure",
                 "OPENROUTER": "openrouter",
@@ -241,7 +241,7 @@ async def get_search_space_llm_instance(
                     "GROQ": "groq",
                     "COHERE": "cohere",
                     "GOOGLE": "gemini",
-                    "OLLAMA": "ollama",
+                    "OLLAMA": "ollama_chat",
                     "MISTRAL": "mistral",
                     "AZURE_OPENAI": "azure",
                     "OPENROUTER": "openrouter",
@@ -311,7 +311,7 @@ async def get_search_space_llm_instance(
                 "GROQ": "groq",
                 "COHERE": "cohere",
                 "GOOGLE": "gemini",
-                "OLLAMA": "ollama",
+                "OLLAMA": "ollama_chat",
                 "MISTRAL": "mistral",
                 "AZURE_OPENAI": "azure",
                 "OPENROUTER": "openrouter",

surfsense_backend/app/services/public_chat_service.py

@@ -1,17 +1,35 @@
 """
-Service layer for public chat sharing and cloning.
+Service layer for public chat sharing via immutable snapshots.
+
+Key concepts:
+- Snapshots are frozen copies of a chat at a specific point in time
+- Content hash enables deduplication (same content = same URL)
+- Podcasts are embedded in snapshot_data for self-contained public views
+- Single-phase clone reads directly from snapshot_data
 """
 
+import hashlib
+import json
 import re
 import secrets
+from datetime import UTC, datetime
 from uuid import UUID
 
 from fastapi import HTTPException
-from sqlalchemy import select
+from sqlalchemy import delete, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import selectinload
 
-from app.db import NewChatThread, User
+from app.db import (
+    ChatVisibility,
+    NewChatMessage,
+    NewChatThread,
+    Podcast,
+    PodcastStatus,
+    PublicChatSnapshot,
+    SearchSpaceMembership,
+    User,
+)
 
 UI_TOOLS = {
     "display_image",
@@ -100,20 +118,242 @@ async def get_author_display(
     return user_cache[author_id]
 
 
-async def toggle_public_share(
+# =============================================================================
+# Content Hashing
+# =============================================================================
+
+
+def compute_content_hash(messages: list[dict]) -> str:
+    """
+    Compute SHA-256 hash of message content for deduplication.
+
+    The hash is based on message IDs and content, ensuring that:
+    - Same messages = same hash = same URL (deduplication)
+    - Any change = different hash = new URL
+    """
+    # Sort by message ID to ensure consistent ordering
+    sorted_messages = sorted(messages, key=lambda m: m.get("id", 0))
+
+    # Create normalized representation
+    normalized = []
+    for msg in sorted_messages:
+        normalized.append(
+            {
+                "id": msg.get("id"),
+                "role": msg.get("role"),
+                "content": msg.get("content"),
+            }
+        )
+
+    content_str = json.dumps(normalized, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(content_str.encode()).hexdigest()
+
+
+# =============================================================================
+# Snapshot Creation
+# =============================================================================
+
+
+async def create_snapshot(
     session: AsyncSession,
     thread_id: int,
-    enabled: bool,
     user: User,
     base_url: str,
 ) -> dict:
     """
-    Enable or disable public sharing for a thread.
+    Create a public snapshot of a chat thread.
 
-    Only the thread owner can toggle public sharing.
+    Returns existing snapshot if content unchanged (same hash).
-    When enabling, generates a new token if one doesn't exist.
+    Returns new snapshot with unique URL if content changed.
-    When disabling, keeps the token for potential re-enable.
     """
+    result = await session.execute(
+        select(NewChatThread)
+        .options(selectinload(NewChatThread.messages))
+        .filter(NewChatThread.id == thread_id)
+    )
+    thread = result.scalars().first()
+
+    if not thread:
+        raise HTTPException(status_code=404, detail="Thread not found")
+
+    if thread.created_by_id != user.id:
+        raise HTTPException(
+            status_code=403,
+            detail="Only the creator of this chat can create public snapshots",
+        )
+
+    # Build snapshot data
+    user_cache: dict[UUID, dict] = {}
+    messages_data = []
+    message_ids = []
+    podcasts_data = []
+    podcast_ids_seen: set[int] = set()
+
+    for msg in sorted(thread.messages, key=lambda m: m.created_at):
+        author = await get_author_display(session, msg.author_id, user_cache)
+        sanitized_content = sanitize_content_for_public(msg.content)
+
+        # Extract podcast references and update status to "ready" for completed podcasts
+        if isinstance(sanitized_content, list):
+            for part in sanitized_content:
+                if (
+                    isinstance(part, dict)
+                    and part.get("type") == "tool-call"
+                    and part.get("toolName") == "generate_podcast"
+                ):
+                    result_data = part.get("result", {})
+                    podcast_id = result_data.get("podcast_id")
+                    if podcast_id and podcast_id not in podcast_ids_seen:
+                        podcast_info = await _get_podcast_for_snapshot(
+                            session, podcast_id
+                        )
+                        if podcast_info:
+                            podcasts_data.append(podcast_info)
+                            podcast_ids_seen.add(podcast_id)
+                            # Update status to "ready" so frontend renders PodcastPlayer
+                            part["result"] = {**result_data, "status": "ready"}
+
+
+        messages_data.append(
+            {
+                "id": msg.id,
+                "role": msg.role.value if hasattr(msg.role, "value") else str(msg.role),
+                "content": sanitized_content,
+                "author": author,
+                "author_id": str(msg.author_id) if msg.author_id else None,
+                "created_at": msg.created_at.isoformat() if msg.created_at else None,
+            }
+        )
+        message_ids.append(msg.id)
+
+    if not messages_data:
+        raise HTTPException(status_code=400, detail="Cannot share an empty chat")
+
+    # Compute content hash for deduplication
+    content_hash = compute_content_hash(messages_data)
+
+    # Check if identical snapshot already exists
+    existing_result = await session.execute(
+        select(PublicChatSnapshot).filter(
+            PublicChatSnapshot.thread_id == thread_id,
+            PublicChatSnapshot.content_hash == content_hash,
+        )
+    )
+    existing = existing_result.scalars().first()
+
+    if existing:
+        # Return existing snapshot URL
+        return {
+            "snapshot_id": existing.id,
+            "share_token": existing.share_token,
+            "public_url": f"{base_url}/public/{existing.share_token}",
+            "is_new": False,
+        }
+
+    # Get thread author info
+    thread_author = await get_author_display(session, thread.created_by_id, user_cache)
+
+    # Create snapshot data
+    snapshot_data = {
+        "title": thread.title,
+        "snapshot_at": datetime.now(UTC).isoformat(),
+        "author": thread_author,
+        "messages": messages_data,
+        "podcasts": podcasts_data,
+    }
+
+    # Create new snapshot
+    share_token = secrets.token_urlsafe(48)
+    snapshot = PublicChatSnapshot(
+        thread_id=thread_id,
+        share_token=share_token,
+        content_hash=content_hash,
+        snapshot_data=snapshot_data,
+        message_ids=message_ids,
+        created_by_user_id=user.id,
+    )
+    session.add(snapshot)
+    await session.commit()
+    await session.refresh(snapshot)
+
+    return {
+        "snapshot_id": snapshot.id,
+        "share_token": snapshot.share_token,
+        "public_url": f"{base_url}/public/{snapshot.share_token}",
+        "is_new": True,
+    }
+
+
+async def _get_podcast_for_snapshot(
+    session: AsyncSession,
+    podcast_id: int,
+) -> dict | None:
+    """Get podcast info for embedding in snapshot_data."""
+    result = await session.execute(select(Podcast).filter(Podcast.id == podcast_id))
+    podcast = result.scalars().first()
+
+    if not podcast or podcast.status != PodcastStatus.READY:
+        return None
+
+    return {
+        "original_id": podcast.id,
+        "title": podcast.title,
+        "transcript": podcast.podcast_transcript,
+        "file_path": podcast.file_location,
+    }
+
+
+# =============================================================================
+# Snapshot Retrieval
+# =============================================================================
+
+
+async def get_snapshot_by_token(
+    session: AsyncSession,
+    share_token: str,
+) -> PublicChatSnapshot | None:
+    """Get a snapshot by its share token."""
+    result = await session.execute(
+        select(PublicChatSnapshot).filter(
+            PublicChatSnapshot.share_token == share_token
+        )
+    )
+    return result.scalars().first()
+
+
+async def get_public_chat(
+    session: AsyncSession,
+    share_token: str,
+) -> dict:
+    """
+    Get public chat data from a snapshot.
+
+    Returns sanitized content suitable for public viewing.
+    """
+    snapshot = await get_snapshot_by_token(session, share_token)
+
+    if not snapshot:
+        raise HTTPException(status_code=404, detail="Not found")
+
+    data = snapshot.snapshot_data
+
+    return {
+        "thread": {
+            "title": data.get("title", "Untitled"),
+            "created_at": data.get("snapshot_at"),
+        },
+        "messages": data.get("messages", []),
+    }
+
+
+async def list_snapshots_for_thread(
+    session: AsyncSession,
+    thread_id: int,
+    user: User,
+    base_url: str,
+) -> list[dict]:
+    """List all public snapshots for a thread."""
+    # Verify ownership
     result = await session.execute(
         select(NewChatThread).filter(NewChatThread.id == thread_id)
     )
@@ -125,92 +365,101 @@ async def toggle_public_share(
     if thread.created_by_id != user.id:
         raise HTTPException(
             status_code=403,
-            detail="Only the creator of this chat can manage public sharing",
+            detail="Only the creator can view snapshots",
         )
 
-    if enabled and not thread.public_share_token:
+    # Get snapshots
-        thread.public_share_token = secrets.token_urlsafe(48)
+    result = await session.execute(
+        select(PublicChatSnapshot)
+        .filter(PublicChatSnapshot.thread_id == thread_id)
+        .order_by(PublicChatSnapshot.created_at.desc())
+    )
+    snapshots = result.scalars().all()
 
-    thread.public_share_enabled = enabled
+    return [
-
+        {
-    await session.commit()
+            "id": s.id,
-    await session.refresh(thread)
+            "share_token": s.share_token,
-
+            "public_url": f"{base_url}/public/{s.share_token}",
-    if enabled:
+            "created_at": s.created_at.isoformat() if s.created_at else None,
-        return {
+            "message_count": len(s.message_ids) if s.message_ids else 0,
-            "enabled": True,
-            "public_url": f"{base_url}/public/{thread.public_share_token}",
-            "share_token": thread.public_share_token,
         }
-
+        for s in snapshots
-    return {
+    ]
-        "enabled": False,
-        "public_url": None,
-        "share_token": None,
-    }
 
 
-async def get_public_chat(
+# =============================================================================
+# Snapshot Deletion
+# =============================================================================
+
+
+async def delete_snapshot(
     session: AsyncSession,
-    share_token: str,
+    thread_id: int,
-) -> dict:
+    snapshot_id: int,
-    """
+    user: User,
-    Get a public chat by share token.
+) -> bool:
-
+    """Delete a specific snapshot. Only thread owner can delete."""
-    Returns sanitized content suitable for public viewing.
+    # Get snapshot with thread
-    """
     result = await session.execute(
-        select(NewChatThread)
+        select(PublicChatSnapshot)
-        .options(selectinload(NewChatThread.messages))
+        .options(selectinload(PublicChatSnapshot.thread))
         .filter(
-            NewChatThread.public_share_token == share_token,
+            PublicChatSnapshot.id == snapshot_id,
-            NewChatThread.public_share_enabled.is_(True),
+            PublicChatSnapshot.thread_id == thread_id,
         )
     )
-    thread = result.scalars().first()
+    snapshot = result.scalars().first()
 
-    if not thread:
+    if not snapshot:
-        raise HTTPException(status_code=404, detail="Not found")
+        raise HTTPException(status_code=404, detail="Snapshot not found")
 
-    user_cache: dict[UUID, dict] = {}
+    if snapshot.thread.created_by_id != user.id:
-
+        raise HTTPException(
-    messages = []
+            status_code=403,
-    for msg in sorted(thread.messages, key=lambda m: m.created_at):
+            detail="Only the creator can delete snapshots",
-        author = await get_author_display(session, msg.author_id, user_cache)
-        sanitized_content = sanitize_content_for_public(msg.content)
-
-        messages.append(
-            {
-                "role": msg.role,
-                "content": sanitized_content,
-                "author": author,
-                "created_at": msg.created_at,
-            }
         )
 
-    return {
+    await session.delete(snapshot)
-        "thread": {
+    await session.commit()
-            "title": thread.title,
+    return True
-            "created_at": thread.created_at,
-        },
-        "messages": messages,
-    }
 
 
-async def get_thread_by_share_token(
+async def delete_affected_snapshots(
-    session: AsyncSession,
+    session: AsyncSession,  # noqa: ARG001 - kept for API compatibility
-    share_token: str,
+    thread_id: int,
-) -> NewChatThread | None:
+    message_ids: list[int],
-    """Get a thread by its public share token if sharing is enabled."""
+) -> int:
-    result = await session.execute(
+    """
-        select(NewChatThread)
+    Delete snapshots that contain any of the given message IDs.
-        .options(selectinload(NewChatThread.messages))
+
-        .filter(
+    Called when messages are edited/deleted/regenerated.
-            NewChatThread.public_share_token == share_token,
+    Uses independent session to work reliably in streaming response cleanup.
-            NewChatThread.public_share_enabled.is_(True),
+    """
+    if not message_ids:
+        return 0
+
+    from sqlalchemy.dialects.postgresql import array
+
+    from app.db import async_session_maker
+
+    async with async_session_maker() as independent_session:
+        result = await independent_session.execute(
+            delete(PublicChatSnapshot)
+            .where(PublicChatSnapshot.thread_id == thread_id)
+            .where(PublicChatSnapshot.message_ids.op("&&")(array(message_ids)))
+            .returning(PublicChatSnapshot.id)
         )
-    )
+
-    return result.scalars().first()
+        deleted_ids = result.scalars().all()
+        await independent_session.commit()
+
+        return len(deleted_ids)
+
+
+# =============================================================================
+# Cloning from Snapshot
+# =============================================================================
 
 
 async def get_user_default_search_space(
@@ -222,8 +471,6 @@ async def get_user_default_search_space(
 
     Returns the first search space where user is owner, or None if not found.
     """
-    from app.db import SearchSpaceMembership
-
     result = await session.execute(
         select(SearchSpaceMembership)
         .filter(
@@ -240,140 +487,153 @@ async def get_user_default_search_space(
     return None
 
 
-async def complete_clone_content(
+async def clone_from_snapshot(
     session: AsyncSession,
-    target_thread: NewChatThread,
+    share_token: str,
-    source_thread_id: int,
+    user: User,
-    target_search_space_id: int,
+) -> dict:
-) -> int:
     """
     Copy messages and podcasts from source thread to target thread.
 
-    Sets clone_pending=False and needs_history_bootstrap=True when done.
+    Creates thread and copies messages from snapshot_data.
-    Returns the number of messages copied.
+    When encountering generate_podcast tool-calls, creates cloned podcast records
+    and updates the podcast_id references inline.
+    Returns the new thread info.
     """
-    from app.db import NewChatMessage
+    import copy
 
-    result = await session.execute(
+    snapshot = await get_snapshot_by_token(session, share_token)
-        select(NewChatThread)
+
-        .options(selectinload(NewChatThread.messages))
+    if not snapshot:
-        .filter(NewChatThread.id == source_thread_id)
+        raise HTTPException(
+            status_code=404, detail="Chat not found or no longer public"
+        )
+
+    target_search_space_id = await get_user_default_search_space(session, user.id)
+
+    if target_search_space_id is None:
+        raise HTTPException(status_code=400, detail="No search space found for user")
+
+    data = snapshot.snapshot_data
+    messages_data = data.get("messages", [])
+    podcasts_lookup = {p.get("original_id"): p for p in data.get("podcasts", [])}
+
+    new_thread = NewChatThread(
+        title=data.get("title", "Cloned Chat"),
+        archived=False,
+        visibility=ChatVisibility.PRIVATE,
+        search_space_id=target_search_space_id,
+        created_by_id=user.id,
+        cloned_from_thread_id=snapshot.thread_id,
+        cloned_from_snapshot_id=snapshot.id,
+        cloned_at=datetime.now(UTC),
+        needs_history_bootstrap=True,
     )
-    source_thread = result.scalars().first()
+    session.add(new_thread)
+    await session.flush()
 
-    if not source_thread:
+    podcast_id_mapping: dict[int, int] = {}
-        raise ValueError("Source thread not found")
 
-    podcast_id_map: dict[int, int] = {}
+    # Check which authors from snapshot still exist in DB
-    message_count = 0
+    author_ids_from_snapshot: set[UUID] = set()
+    for msg_data in messages_data:
+        if author_str := msg_data.get("author_id"):
+            try:
+                author_ids_from_snapshot.add(UUID(author_str))
+            except (ValueError, TypeError):
+                pass
 
-    for msg in sorted(source_thread.messages, key=lambda m: m.created_at):
+    existing_authors: set[UUID] = set()
-        new_content = sanitize_content_for_public(msg.content)
+    if author_ids_from_snapshot:
+        result = await session.execute(
+            select(User.id).where(User.id.in_(author_ids_from_snapshot))
+        )
+        existing_authors = {row[0] for row in result.fetchall()}
 
-        if isinstance(new_content, list):
+    for msg_data in messages_data:
-            for part in new_content:
+        role = msg_data.get("role", "user")
+
+        # Use original author if exists, otherwise None
+        author_id = None
+        if author_str := msg_data.get("author_id"):
+            try:
+                parsed_id = UUID(author_str)
+                if parsed_id in existing_authors:
+                    author_id = parsed_id
+            except (ValueError, TypeError):
+                pass
+
+        content = copy.deepcopy(msg_data.get("content", []))
+
+        if isinstance(content, list):
+            for part in content:
                 if (
                     isinstance(part, dict)
                     and part.get("type") == "tool-call"
                     and part.get("toolName") == "generate_podcast"
                 ):
-                    result_data = part.get("result", {})
+                    result = part.get("result", {})
-                    old_podcast_id = result_data.get("podcast_id")
+                    old_podcast_id = result.get("podcast_id")
-                    if old_podcast_id and old_podcast_id not in podcast_id_map:
-                        new_podcast_id = await _clone_podcast(
-                            session,
-                            old_podcast_id,
-                            target_search_space_id,
-                            target_thread.id,
-                        )
-                        if new_podcast_id:
-                            podcast_id_map[old_podcast_id] = new_podcast_id
 
-                    if old_podcast_id and old_podcast_id in podcast_id_map:
+                    if old_podcast_id and old_podcast_id not in podcast_id_mapping:
-                        result_data["podcast_id"] = podcast_id_map[old_podcast_id]
+                        podcast_info = podcasts_lookup.get(old_podcast_id)
-                    elif old_podcast_id:
+                        if podcast_info:
-                        # Podcast couldn't be cloned (not ready), remove reference
+                            new_podcast = Podcast(
-                        result_data.pop("podcast_id", None)
+                                title=podcast_info.get("title", "Cloned Podcast"),
+                                podcast_transcript=podcast_info.get("transcript"),
+                                file_location=podcast_info.get("file_path"),
+                                status=PodcastStatus.READY,
+                                search_space_id=target_search_space_id,
+                                thread_id=new_thread.id,
+                            )
+                            session.add(new_podcast)
+                            await session.flush()
+                            podcast_id_mapping[old_podcast_id] = new_podcast.id
+
+                    if old_podcast_id and old_podcast_id in podcast_id_mapping:
+                        part["result"] = {
+                            **result,
+                            "podcast_id": podcast_id_mapping[old_podcast_id],
+                        }
 
         new_message = NewChatMessage(
-            thread_id=target_thread.id,
+            thread_id=new_thread.id,
-            role=msg.role,
+            role=role,
-            content=new_content,
+            content=content,
-            author_id=msg.author_id,
+            author_id=author_id,
-            created_at=msg.created_at,
         )
         session.add(new_message)
-        message_count += 1
-
-    target_thread.clone_pending = False
-    target_thread.needs_history_bootstrap = True
 
     await session.commit()
+    await session.refresh(new_thread)
 
-    return message_count
+    return {
+        "thread_id": new_thread.id,
+        "search_space_id": target_search_space_id,
+    }
 
 
-async def _clone_podcast(
+async def get_snapshot_podcast(
     session: AsyncSession,
+    share_token: str,
     podcast_id: int,
-    target_search_space_id: int,
+) -> dict | None:
-    target_thread_id: int,
+    """
-) -> int | None:
+    Get podcast info from a snapshot by original podcast ID.
-    """Clone a podcast record and its audio file. Only clones ready podcasts."""
-    import shutil
-    import uuid
-    from pathlib import Path
 
-    from app.db import Podcast, PodcastStatus
+    Used for streaming podcast audio from public view.
+    Looks up the podcast by its original_id in the snapshot's podcasts array.
+    """
+    snapshot = await get_snapshot_by_token(session, share_token)
 
-    result = await session.execute(select(Podcast).filter(Podcast.id == podcast_id))
+    if not snapshot:
-    original = result.scalars().first()
-    if not original or original.status != PodcastStatus.READY:
         return None
 
-    new_file_path = None
+    podcasts = snapshot.snapshot_data.get("podcasts", [])
-    if original.file_location:
-        original_path = Path(original.file_location)
-        if original_path.exists():
-            new_filename = f"{uuid.uuid4()}_podcast.mp3"
-            new_dir = Path("podcasts")
-            new_dir.mkdir(parents=True, exist_ok=True)
-            new_file_path = str(new_dir / new_filename)
-            shutil.copy2(original.file_location, new_file_path)
 
-    new_podcast = Podcast(
+    # Find podcast by original_id
-        title=original.title,
+    for podcast in podcasts:
-        podcast_transcript=original.podcast_transcript,
+        if podcast.get("original_id") == podcast_id:
-        file_location=new_file_path,
+            return podcast
-        status=PodcastStatus.READY,
-        search_space_id=target_search_space_id,
-        thread_id=target_thread_id,
-    )
-    session.add(new_podcast)
-    await session.flush()
 
-    return new_podcast.id
+    return None
-
-
-async def is_podcast_publicly_accessible(
-    session: AsyncSession,
-    podcast_id: int,
-) -> bool:
-    """
-    Check if a podcast belongs to a publicly shared thread.
-
-    Uses the thread_id foreign key for efficient lookup.
-    """
-    from app.db import Podcast
-
-    result = await session.execute(
-        select(Podcast)
-        .options(selectinload(Podcast.thread))
-        .filter(Podcast.id == podcast_id)
-    )
-    podcast = result.scalars().first()
-
-    if not podcast or not podcast.thread:
-        return False
-
-    return podcast.thread.public_share_enabled

surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py

@@ -0,0 +1,164 @@
+"""Celery task to detect and mark stale connector indexing notifications as failed.
+
+This task runs periodically (every 5 minutes by default) to find notifications
+that are stuck in "in_progress" status but don't have an active Redis heartbeat key.
+These are marked as "failed" to prevent the frontend from showing a perpetual "syncing" state.
+
+Detection mechanism:
+- Active indexing tasks set a Redis key with TTL (2 minutes) as a heartbeat
+- If the task crashes, the Redis key expires automatically
+- This cleanup task checks for in-progress notifications without a Redis heartbeat key
+- Such notifications are marked as failed with O(1) batch UPDATE
+"""
+
+import json
+import logging
+import os
+from datetime import UTC, datetime
+
+import redis
+from sqlalchemy import and_, text
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+from sqlalchemy.future import select
+from sqlalchemy.pool import NullPool
+
+from app.celery_app import celery_app
+from app.config import config
+from app.db import Notification
+
+logger = logging.getLogger(__name__)
+
+# Redis client for checking heartbeats
+_redis_client: redis.Redis | None = None
+
+
+def get_redis_client() -> redis.Redis:
+    """Get or create Redis client for heartbeat checking."""
+    global _redis_client
+    if _redis_client is None:
+        redis_url = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
+        _redis_client = redis.from_url(redis_url, decode_responses=True)
+    return _redis_client
+
+
+def _get_heartbeat_key(notification_id: int) -> str:
+    """Generate Redis key for notification heartbeat."""
+    return f"indexing:heartbeat:{notification_id}"
+
+
+def get_celery_session_maker():
+    """Create async session maker for Celery tasks."""
+    engine = create_async_engine(
+        config.DATABASE_URL,
+        poolclass=NullPool,
+        echo=False,
+    )
+    return async_sessionmaker(engine, expire_on_commit=False)
+
+
+@celery_app.task(name="cleanup_stale_indexing_notifications")
+def cleanup_stale_indexing_notifications_task():
+    """
+    Check for stale connector indexing notifications and mark them as failed.
+
+    This task finds notifications that:
+    - Have type = 'connector_indexing'
+    - Have metadata.status = 'in_progress'
+    - Do NOT have a corresponding Redis heartbeat key (meaning task crashed)
+
+    And marks them as failed with O(1) batch UPDATE.
+    """
+    import asyncio
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+
+    try:
+        loop.run_until_complete(_cleanup_stale_notifications())
+    finally:
+        loop.close()
+
+
+async def _cleanup_stale_notifications():
+    """Find and mark stale connector indexing notifications as failed.
+
+    Uses Redis TTL-based detection:
+    1. Find all in-progress notifications
+    2. Check which ones are missing their Redis heartbeat key
+    3. Mark those as failed with O(1) batch UPDATE using JSONB || operator
+    """
+    async with get_celery_session_maker()() as session:
+        try:
+            # Find all in-progress connector indexing notifications
+            result = await session.execute(
+                select(Notification.id).where(
+                    and_(
+                        Notification.type == "connector_indexing",
+                        Notification.notification_metadata["status"].astext
+                        == "in_progress",
+                    )
+                )
+            )
+            in_progress_ids = [row[0] for row in result.fetchall()]
+
+            if not in_progress_ids:
+                logger.debug("No in-progress connector indexing notifications found")
+                return
+
+            # Check which ones are missing heartbeat keys in Redis
+            redis_client = get_redis_client()
+            stale_notification_ids = []
+
+            for notification_id in in_progress_ids:
+                heartbeat_key = _get_heartbeat_key(notification_id)
+                if not redis_client.exists(heartbeat_key):
+                    stale_notification_ids.append(notification_id)
+
+            if not stale_notification_ids:
+                logger.debug(
+                    f"All {len(in_progress_ids)} in-progress notifications have active Redis heartbeats"
+                )
+                return
+
+            logger.warning(
+                f"Found {len(stale_notification_ids)} stale connector indexing notifications "
+                f"(no Redis heartbeat key): {stale_notification_ids}"
+            )
+
+            # O(1) Batch UPDATE using JSONB || operator
+            # This merges the update data into existing notification_metadata
+            # Also updates title and message for proper UI display
+            error_message = (
+                "Something went wrong while syncing your content. Please retry."
+            )
+
+            update_data = {
+                "status": "failed",
+                "completed_at": datetime.now(UTC).isoformat(),
+                "error_message": error_message,
+                "sync_stage": "failed",
+            }
+
+            await session.execute(
+                text("""
+                    UPDATE notifications 
+                    SET metadata = metadata || CAST(:update_json AS jsonb),
+                        title = 'Failed: ' || COALESCE(metadata->>'connector_name', 'Connector'),
+                        message = :display_message
+                    WHERE id = ANY(:ids)
+                """),
+                {
+                    "update_json": json.dumps(update_data),
+                    "display_message": f"{error_message}",
+                    "ids": stale_notification_ids,
+                },
+            )
+
+            await session.commit()
+            logger.info(
+                f"Successfully marked {len(stale_notification_ids)} stale notifications as failed (batch UPDATE)"
+            )
+
+        except Exception as e:
+            logger.error(f"Error cleaning up stale notifications: {e!s}", exc_info=True)
+            await session.rollback()

surfsense_backend/app/tasks/composio_indexer.py

@@ -9,6 +9,7 @@ to avoid circular import issues with the connector_indexers package.
 """
 
 import logging
+from collections.abc import Awaitable, Callable
 from importlib import import_module
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -22,6 +23,9 @@ from app.db import (
 from app.services.composio_service import INDEXABLE_TOOLKITS, TOOLKIT_TO_INDEXER
 from app.services.task_logging_service import TaskLoggingService
 
+# Type alias for heartbeat callback function
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
 # Set up logging
 logger = logging.getLogger(__name__)
 
@@ -86,6 +90,7 @@ async def index_composio_connector(
     end_date: str | None = None,
     update_last_indexed: bool = True,
     max_items: int = 1000,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, int, str | None]:
     """
     Index content from a Composio connector.
@@ -102,6 +107,7 @@ async def index_composio_connector(
         end_date: End date for filtering (YYYY-MM-DD format)
         update_last_indexed: Whether to update the last_indexed_at timestamp
         max_items: Maximum number of items to fetch
+        on_heartbeat_callback: Optional callback to report progress for heartbeat updates
 
     Returns:
         Tuple of (number_of_indexed_items, number_of_skipped_items, error_message or None)
@@ -180,6 +186,7 @@ async def index_composio_connector(
             "log_entry": log_entry,
             "update_last_indexed": update_last_indexed,
             "max_items": max_items,
+            "on_heartbeat_callback": on_heartbeat_callback,
         }
 
         # Add date params for toolkits that support them

surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py

@@ -2,6 +2,9 @@
 Airtable connector indexer.
 """
 
+import time
+from collections.abc import Awaitable, Callable
+
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -27,6 +30,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_airtable_records(
     session: AsyncSession,
@@ -37,6 +46,7 @@ async def index_airtable_records(
     end_date: str | None = None,
     max_records: int = 2500,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Airtable records for a given connector.
@@ -50,6 +60,7 @@ async def index_airtable_records(
         end_date: End date for filtering records (YYYY-MM-DD)
         max_records: Maximum number of records to fetch per table
         update_last_indexed: Whether to update the last_indexed_at timestamp
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple of (number_of_documents_processed, error_message)
@@ -127,8 +138,20 @@ async def index_airtable_records(
 
             logger.info(f"Found {len(bases)} Airtable bases to process")
 
+            # Heartbeat tracking - update notification periodically to prevent appearing stuck
+            last_heartbeat_time = time.time()
+            total_documents_indexed = 0
+
             # Process each base
             for base in bases:
+                # Check if it's time for a heartbeat update
+                if (
+                    on_heartbeat_callback
+                    and (time.time() - last_heartbeat_time)
+                    >= HEARTBEAT_INTERVAL_SECONDS
+                ):
+                    await on_heartbeat_callback(total_documents_indexed)
+                    last_heartbeat_time = time.time()
                 base_id = base.get("id")
                 base_name = base.get("name", "Unknown Base")
 
@@ -204,6 +227,15 @@ async def index_airtable_records(
                     documents_skipped = 0
                     # Process each record
                     for record in records:
+                        # Check if it's time for a heartbeat update
+                        if (
+                            on_heartbeat_callback
+                            and (time.time() - last_heartbeat_time)
+                            >= HEARTBEAT_INTERVAL_SECONDS
+                        ):
+                            await on_heartbeat_callback(total_documents_indexed)
+                            last_heartbeat_time = time.time()
+
                         try:
                             # Generate markdown content
                             markdown_content = (

surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py

@@ -2,6 +2,8 @@
 BookStack connector indexer.
 """
 
+import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -29,6 +31,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_bookstack_pages(
     session: AsyncSession,
@@ -38,6 +46,7 @@ async def index_bookstack_pages(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index BookStack pages.
@@ -50,6 +59,7 @@ async def index_bookstack_pages(
         start_date: Start date for indexing (YYYY-MM-DD format)
         end_date: End date for indexing (YYYY-MM-DD format)
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -179,7 +189,17 @@ async def index_bookstack_pages(
         skipped_pages = []
         documents_skipped = 0
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         for page in pages:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
             try:
                 page_id = page.get("id")
                 page_name = page.get("name", "")

surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py

@@ -3,6 +3,8 @@ ClickUp connector indexer.
 """
 
 import contextlib
+import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -29,6 +31,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_clickup_tasks(
     session: AsyncSession,
@@ -38,6 +46,7 @@ async def index_clickup_tasks(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index tasks from ClickUp workspace.
@@ -50,6 +59,7 @@ async def index_clickup_tasks(
         start_date: Start date for filtering tasks (YYYY-MM-DD format)
         end_date: End date for filtering tasks (YYYY-MM-DD format)
         update_last_indexed: Whether to update the last_indexed_at timestamp
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple of (number of indexed tasks, error message if any)
@@ -132,6 +142,9 @@ async def index_clickup_tasks(
         documents_indexed = 0
         documents_skipped = 0
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         # Iterate workspaces and fetch tasks
         for workspace in workspaces:
             workspace_id = workspace.get("id")
@@ -170,6 +183,15 @@ async def index_clickup_tasks(
             )
 
             for task in tasks:
+                # Check if it's time for a heartbeat update
+                if (
+                    on_heartbeat_callback
+                    and (time.time() - last_heartbeat_time)
+                    >= HEARTBEAT_INTERVAL_SECONDS
+                ):
+                    await on_heartbeat_callback(documents_indexed)
+                    last_heartbeat_time = time.time()
+
                 try:
                     task_id = task.get("id")
                     task_name = task.get("name", "Untitled Task")

surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py

@@ -3,6 +3,8 @@ Confluence connector indexer.
 """
 
 import contextlib
+import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -30,6 +32,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_confluence_pages(
     session: AsyncSession,
@@ -39,6 +47,7 @@ async def index_confluence_pages(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Confluence pages and comments.
@@ -51,6 +60,7 @@ async def index_confluence_pages(
         start_date: Start date for indexing (YYYY-MM-DD format)
         end_date: End date for indexing (YYYY-MM-DD format)
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -175,7 +185,17 @@ async def index_confluence_pages(
         skipped_pages = []
         documents_skipped = 0
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         for page in pages:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
             try:
                 page_id = page.get("id")
                 page_title = page.get("title", "")

surfsense_backend/app/tasks/connector_indexers/discord_indexer.py

@@ -3,6 +3,8 @@ Discord connector indexer.
 """
 
 import asyncio
+import time
+from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime, timedelta
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -28,6 +30,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds - update notification every 30 seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_discord_messages(
     session: AsyncSession,
@@ -37,6 +45,7 @@ async def index_discord_messages(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Discord messages from all accessible channels.
@@ -49,6 +58,8 @@ async def index_discord_messages(
         start_date: Start date for indexing (YYYY-MM-DD format)
         end_date: End date for indexing (YYYY-MM-DD format)
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
+            Called periodically with (indexed_count) to prevent task appearing stuck.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -281,6 +292,9 @@ async def index_discord_messages(
         documents_skipped = 0
         skipped_channels: list[str] = []
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         # Process each guild and channel
         await task_logger.log_task_progress(
             log_entry,
@@ -290,6 +304,14 @@ async def index_discord_messages(
 
         try:
             for guild in guilds:
+                # Check if it's time for a heartbeat update
+                if (
+                    on_heartbeat_callback
+                    and (time.time() - last_heartbeat_time)
+                    >= HEARTBEAT_INTERVAL_SECONDS
+                ):
+                    await on_heartbeat_callback(documents_indexed)
+                    last_heartbeat_time = time.time()
                 guild_id = guild["id"]
                 guild_name = guild["name"]
                 logger.info(f"Processing guild: {guild_name} ({guild_id})")

surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py

@@ -4,6 +4,8 @@ Elasticsearch indexer for SurfSense
 
 import json
 import logging
+import time
+from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime
 from typing import Any
 
@@ -25,6 +27,12 @@ from .base import (
     get_current_timestamp,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 logger = logging.getLogger(__name__)
 
 
@@ -36,6 +44,7 @@ async def index_elasticsearch_documents(
     start_date: str,
     end_date: str,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index documents from Elasticsearch into SurfSense
@@ -48,6 +57,7 @@ async def index_elasticsearch_documents(
         start_date: Start date for indexing (not used for Elasticsearch, kept for compatibility)
         end_date: End date for indexing (not used for Elasticsearch, kept for compatibility)
         update_last_indexed: Whether to update the last indexed timestamp
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple of (number of documents processed, error message if any)
@@ -155,6 +165,9 @@ async def index_elasticsearch_documents(
 
         documents_processed = 0
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         try:
             await task_logger.log_task_progress(
                 log_entry,
@@ -172,6 +185,15 @@ async def index_elasticsearch_documents(
                 size=min(max_documents, 100),  # Scroll in batches
                 fields=config.get("ELASTICSEARCH_FIELDS"),
             ):
+                # Check if it's time for a heartbeat update
+                if (
+                    on_heartbeat_callback
+                    and (time.time() - last_heartbeat_time)
+                    >= HEARTBEAT_INTERVAL_SECONDS
+                ):
+                    await on_heartbeat_callback(documents_processed)
+                    last_heartbeat_time = time.time()
+
                 if documents_processed >= max_documents:
                     break
 

surfsense_backend/app/tasks/connector_indexers/github_indexer.py

@@ -5,6 +5,8 @@ This indexer processes entire repository digests in one pass, dramatically
 reducing LLM API calls compared to the previous file-by-file approach.
 """
 
+import time
+from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -30,6 +32,12 @@ from .base import (
     logger,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds - update notification every 30 seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 # Maximum tokens for a single digest before splitting
 # Most LLMs can handle 128k+ tokens now, but we'll be conservative
 MAX_DIGEST_CHARS = 500_000  # ~125k tokens
@@ -43,6 +51,7 @@ async def index_github_repos(
     start_date: str | None = None,  # Ignored - GitHub indexes full repo snapshots
     end_date: str | None = None,  # Ignored - GitHub indexes full repo snapshots
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index GitHub repositories using gitingest for efficient processing.
@@ -62,6 +71,7 @@ async def index_github_repos(
         start_date: Ignored - kept for API compatibility
         end_date: Ignored - kept for API compatibility
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -168,7 +178,18 @@ async def index_github_repos(
             f"Starting gitingest indexing for {len(repo_full_names_to_index)} repositories."
         )
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+        documents_indexed = 0
+
         for repo_full_name in repo_full_names_to_index:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
             if not repo_full_name or not isinstance(repo_full_name, str):
                 logger.warning(f"Skipping invalid repository entry: {repo_full_name}")
                 continue

surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py

@@ -2,10 +2,10 @@
 Google Calendar connector indexer.
 """
 
+import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime, timedelta
 
-import pytz
-from dateutil.parser import isoparse
 from google.oauth2.credentials import Credentials
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -30,6 +30,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_google_calendar_events(
     session: AsyncSession,
@@ -39,6 +45,7 @@ async def index_google_calendar_events(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Google Calendar events.
@@ -52,6 +59,7 @@ async def index_google_calendar_events(
         end_date: End date for indexing (YYYY-MM-DD format). Can be in the future to index upcoming events.
                   Defaults to today if not provided.
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -281,7 +289,17 @@ async def index_google_calendar_events(
             0  # Track events skipped due to duplicate content_hash
         )
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         for event in events:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
             try:
                 event_id = event.get("id")
                 event_summary = event.get("summary", "No Title")

surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py

@@ -1,6 +1,8 @@
 """Google Drive indexer using Surfsense file processors."""
 
 import logging
+import time
+from collections.abc import Awaitable, Callable
 
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -24,6 +26,12 @@ from app.tasks.connector_indexers.base import (
 )
 from app.utils.document_converters import generate_unique_identifier_hash
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 logger = logging.getLogger(__name__)
 
 
@@ -38,6 +46,7 @@ async def index_google_drive_files(
     update_last_indexed: bool = True,
     max_files: int = 500,
     include_subfolders: bool = False,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Google Drive files for a specific connector.
@@ -53,6 +62,7 @@ async def index_google_drive_files(
         update_last_indexed: Whether to update last_indexed_at timestamp
         max_files: Maximum number of files to index
         include_subfolders: Whether to recursively index files in subfolders
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple of (number_of_indexed_files, error_message)
@@ -147,6 +157,7 @@ async def index_google_drive_files(
                 log_entry=log_entry,
                 max_files=max_files,
                 include_subfolders=include_subfolders,
+                on_heartbeat_callback=on_heartbeat_callback,
             )
         else:
             logger.info(f"Using full scan for connector {connector_id}")
@@ -163,6 +174,7 @@ async def index_google_drive_files(
                 log_entry=log_entry,
                 max_files=max_files,
                 include_subfolders=include_subfolders,
+                on_heartbeat_callback=on_heartbeat_callback,
             )
 
         documents_indexed, documents_skipped = result
@@ -383,6 +395,7 @@ async def _index_full_scan(
     log_entry: any,
     max_files: int,
     include_subfolders: bool = False,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, int]:
     """Perform full scan indexing of a folder."""
     await task_logger.log_task_progress(
@@ -399,10 +412,20 @@ async def _index_full_scan(
     documents_skipped = 0
     files_processed = 0
 
+    # Heartbeat tracking - update notification periodically to prevent appearing stuck
+    last_heartbeat_time = time.time()
+
     # Queue of folders to process: (folder_id, folder_name)
     folders_to_process = [(folder_id, folder_name)]
 
     while folders_to_process and files_processed < max_files:
+        # Check if it's time for a heartbeat update
+        if (
+            on_heartbeat_callback
+            and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+        ):
+            await on_heartbeat_callback(documents_indexed)
+            last_heartbeat_time = time.time()
         current_folder_id, current_folder_name = folders_to_process.pop(0)
         logger.info(f"Processing folder: {current_folder_name} ({current_folder_id})")
         page_token = None
@@ -485,6 +508,7 @@ async def _index_with_delta_sync(
     log_entry: any,
     max_files: int,
     include_subfolders: bool = False,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, int]:
     """Perform delta sync indexing using change tracking.
 
@@ -515,7 +539,17 @@ async def _index_with_delta_sync(
     documents_skipped = 0
     files_processed = 0
 
+    # Heartbeat tracking - update notification periodically to prevent appearing stuck
+    last_heartbeat_time = time.time()
+
     for change in changes:
+        # Check if it's time for a heartbeat update
+        if (
+            on_heartbeat_callback
+            and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+        ):
+            await on_heartbeat_callback(documents_indexed)
+            last_heartbeat_time = time.time()
         if files_processed >= max_files:
             break
 

surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py

@@ -2,6 +2,8 @@
 Google Gmail connector indexer.
 """
 
+import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime
 
 from google.oauth2.credentials import Credentials
@@ -33,6 +35,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_google_gmail_messages(
     session: AsyncSession,
@@ -43,6 +51,7 @@ async def index_google_gmail_messages(
     end_date: str | None = None,
     update_last_indexed: bool = True,
     max_messages: int = 1000,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str]:
     """
     Index Gmail messages for a specific connector.
@@ -56,6 +65,7 @@ async def index_google_gmail_messages(
         end_date: End date for filtering messages (YYYY-MM-DD format)
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
         max_messages: Maximum number of messages to fetch (default: 100)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple of (number_of_indexed_messages, status_message)
@@ -212,7 +222,18 @@ async def index_google_gmail_messages(
         documents_indexed = 0
         skipped_messages = []
         documents_skipped = 0
+
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         for message in messages:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
             try:
                 # Extract message information
                 message_id = message.get("id", "")

surfsense_backend/app/tasks/connector_indexers/jira_indexer.py

@@ -3,6 +3,8 @@ Jira connector indexer.
 """
 
 import contextlib
+import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -30,6 +32,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds - update notification every 30 seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_jira_issues(
     session: AsyncSession,
@@ -39,6 +47,7 @@ async def index_jira_issues(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Jira issues and comments.
@@ -51,6 +60,7 @@ async def index_jira_issues(
         start_date: Start date for indexing (YYYY-MM-DD format)
         end_date: End date for indexing (YYYY-MM-DD format)
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -169,7 +179,17 @@ async def index_jira_issues(
         skipped_issues = []
         documents_skipped = 0
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         for issue in issues:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
             try:
                 issue_id = issue.get("key")
                 issue_identifier = issue.get("key", "")

surfsense_backend/app/tasks/connector_indexers/linear_indexer.py

@@ -2,6 +2,8 @@
 Linear connector indexer.
 """
 
+import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -29,6 +31,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds - update notification every 30 seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_linear_issues(
     session: AsyncSession,
@@ -38,6 +46,7 @@ async def index_linear_issues(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Linear issues and comments.
@@ -50,6 +59,7 @@ async def index_linear_issues(
         start_date: Start date for indexing (YYYY-MM-DD format)
         end_date: End date for indexing (YYYY-MM-DD format)
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -188,6 +198,9 @@ async def index_linear_issues(
         documents_skipped = 0
         skipped_issues = []
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         await task_logger.log_task_progress(
             log_entry,
             f"Starting to process {len(issues)} Linear issues",
@@ -196,6 +209,14 @@ async def index_linear_issues(
 
         # Process each issue
         for issue in issues:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
+
             try:
                 issue_id = issue.get("id", "")
                 issue_identifier = issue.get("identifier", "")

surfsense_backend/app/tasks/connector_indexers/luma_indexer.py

@@ -2,6 +2,8 @@
 Luma connector indexer.
 """
 
+import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime, timedelta
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -28,6 +30,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_luma_events(
     session: AsyncSession,
@@ -37,6 +45,7 @@ async def index_luma_events(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Luma events.
@@ -50,6 +59,7 @@ async def index_luma_events(
         end_date: End date for indexing (YYYY-MM-DD format). Can be in the future to index upcoming events.
                   Defaults to today if not provided.
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -221,7 +231,17 @@ async def index_luma_events(
         documents_skipped = 0
         skipped_events = []
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         for event in events:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
             try:
                 # Luma event structure fields - events have nested 'event' field
                 event_data = event.get("event", {})

surfsense_backend/app/tasks/connector_indexers/notion_indexer.py

@@ -2,6 +2,7 @@
 Notion connector indexer.
 """
 
+import time
 from collections.abc import Awaitable, Callable
 from datetime import datetime
 
@@ -34,6 +35,13 @@ from .base import (
 # Signature: async callback(retry_reason, attempt, max_attempts, wait_seconds) -> None
 RetryCallbackType = Callable[[str, int, int, float], Awaitable[None]]
 
+# Type alias for heartbeat callback
+# Signature: async callback(indexed_count) -> None
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds - update notification every 30 seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_notion_pages(
     session: AsyncSession,
@@ -44,6 +52,7 @@ async def index_notion_pages(
     end_date: str | None = None,
     update_last_indexed: bool = True,
     on_retry_callback: RetryCallbackType | None = None,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Notion pages from all accessible pages.
@@ -59,6 +68,8 @@ async def index_notion_pages(
         on_retry_callback: Optional callback for retry progress notifications.
             Signature: async callback(retry_reason, attempt, max_attempts, wait_seconds)
             retry_reason is one of: 'rate_limit', 'server_error', 'timeout'
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
+            Called periodically with (indexed_count) to prevent task appearing stuck.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -211,6 +222,9 @@ async def index_notion_pages(
         documents_skipped = 0
         skipped_pages = []
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         await task_logger.log_task_progress(
             log_entry,
             f"Starting to process {len(pages)} Notion pages",
@@ -219,6 +233,14 @@ async def index_notion_pages(
 
         # Process each page
         for page in pages:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
+
             try:
                 page_id = page.get("page_id")
                 page_title = page.get("title", f"Untitled page ({page_id})")

surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py

@@ -7,6 +7,8 @@ This connector is only available in self-hosted mode.
 
 import os
 import re
+import time
+from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -35,6 +37,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 def parse_frontmatter(content: str) -> tuple[dict | None, str]:
     """
@@ -152,6 +160,7 @@ async def index_obsidian_vault(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index notes from a local Obsidian vault.
@@ -167,6 +176,7 @@ async def index_obsidian_vault(
         start_date: Start date for filtering (YYYY-MM-DD format) - optional
         end_date: End date for filtering (YYYY-MM-DD format) - optional
         update_last_indexed: Whether to update the last_indexed_at timestamp
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -305,7 +315,17 @@ async def index_obsidian_vault(
         indexed_count = 0
         skipped_count = 0
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         for file_info in files:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(indexed_count)
+                last_heartbeat_time = time.time()
             try:
                 file_path = file_info["path"]
                 relative_path = file_info["relative_path"]

surfsense_backend/app/tasks/connector_indexers/slack_indexer.py

@@ -2,6 +2,8 @@
 Slack connector indexer.
 """
 
+import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime
 
 from slack_sdk.errors import SlackApiError
@@ -29,6 +31,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds - update notification every 30 seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_slack_messages(
     session: AsyncSession,
@@ -38,6 +46,7 @@ async def index_slack_messages(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Slack messages from all accessible channels.
@@ -50,6 +59,8 @@ async def index_slack_messages(
         start_date: Start date for indexing (YYYY-MM-DD format)
         end_date: End date for indexing (YYYY-MM-DD format)
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
+            Called periodically with (indexed_count) to prevent task appearing stuck.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -164,6 +175,9 @@ async def index_slack_messages(
         documents_skipped = 0
         skipped_channels = []
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         await task_logger.log_task_progress(
             log_entry,
             f"Starting to process {len(channels)} Slack channels",
@@ -172,6 +186,13 @@ async def index_slack_messages(
 
         # Process each channel
         for channel_obj in channels:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
             channel_id = channel_obj["id"]
             channel_name = channel_obj["name"]
             is_private = channel_obj["is_private"]

surfsense_backend/app/tasks/connector_indexers/teams_indexer.py

@@ -2,6 +2,8 @@
 Microsoft Teams connector indexer.
 """
 
+import time
+from collections.abc import Awaitable, Callable
 from datetime import UTC
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -28,6 +30,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds - update notification every 30 seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_teams_messages(
     session: AsyncSession,
@@ -37,6 +45,7 @@ async def index_teams_messages(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index Microsoft Teams messages from all accessible teams and channels.
@@ -49,6 +58,8 @@ async def index_teams_messages(
         start_date: Start date for indexing (YYYY-MM-DD format)
         end_date: End date for indexing (YYYY-MM-DD format)
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
+            Called periodically with (indexed_count) to prevent task appearing stuck.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -161,6 +172,9 @@ async def index_teams_messages(
         documents_skipped = 0
         skipped_channels = []
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         await task_logger.log_task_progress(
             log_entry,
             f"Starting to process {len(teams)} Teams",
@@ -185,6 +199,14 @@ async def index_teams_messages(
 
         # Process each team
         for team in teams:
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
+
             team_id = team.get("id")
             team_name = team.get("displayName", "Unknown Team")
 

surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py

@@ -2,6 +2,8 @@
 Webcrawler connector indexer.
 """
 
+import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime
 
 from sqlalchemy.exc import SQLAlchemyError
@@ -29,6 +31,12 @@ from .base import (
     update_connector_last_indexed,
 )
 
+# Type hint for heartbeat callback
+HeartbeatCallbackType = Callable[[int], Awaitable[None]]
+
+# Heartbeat interval in seconds
+HEARTBEAT_INTERVAL_SECONDS = 30
+
 
 async def index_crawled_urls(
     session: AsyncSession,
@@ -38,6 +46,7 @@ async def index_crawled_urls(
     start_date: str | None = None,
     end_date: str | None = None,
     update_last_indexed: bool = True,
+    on_heartbeat_callback: HeartbeatCallbackType | None = None,
 ) -> tuple[int, str | None]:
     """
     Index web page URLs.
@@ -50,6 +59,7 @@ async def index_crawled_urls(
         start_date: Start date for filtering (YYYY-MM-DD format) - optional
         end_date: End date for filtering (YYYY-MM-DD format) - optional
         update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+        on_heartbeat_callback: Optional callback to update notification during long-running indexing.
 
     Returns:
         Tuple containing (number of documents indexed, error message or None)
@@ -140,7 +150,17 @@ async def index_crawled_urls(
         documents_skipped = 0
         failed_urls = []
 
+        # Heartbeat tracking - update notification periodically to prevent appearing stuck
+        last_heartbeat_time = time.time()
+
         for idx, url in enumerate(urls, 1):
+            # Check if it's time for a heartbeat update
+            if (
+                on_heartbeat_callback
+                and (time.time() - last_heartbeat_time) >= HEARTBEAT_INTERVAL_SECONDS
+            ):
+                await on_heartbeat_callback(documents_indexed)
+                last_heartbeat_time = time.time()
             try:
                 logger.info(f"Processing URL {idx}/{len(urls)}: {url}")
 

surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx

@@ -42,7 +42,6 @@ import { RecallMemoryToolUI, SaveMemoryToolUI } from "@/components/tool-ui/user-
 import { Spinner } from "@/components/ui/spinner";
 import { useChatSessionStateSync } from "@/hooks/use-chat-session-state";
 import { useMessagesElectric } from "@/hooks/use-messages-electric";
-import { publicChatApiService } from "@/lib/apis/public-chat-api.service";
 // import { WriteTodosToolUI } from "@/components/tool-ui/write-todos";
 import { getBearerToken } from "@/lib/auth-utils";
 import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter";
@@ -142,8 +141,6 @@ export default function NewChatPage() {
 	const params = useParams();
 	const queryClient = useQueryClient();
 	const [isInitializing, setIsInitializing] = useState(true);
-	const [isCompletingClone, setIsCompletingClone] = useState(false);
-	const [cloneError, setCloneError] = useState(false);
 	const [threadId, setThreadId] = useState<number | null>(null);
 	const [currentThread, setCurrentThread] = useState<ThreadRecord | null>(null);
 	const [messages, setMessages] = useState<ThreadMessageLike[]>([]);
@@ -332,42 +329,6 @@ export default function NewChatPage() {
 		initializeThread();
 	}, [initializeThread]);
 
-	// Handle clone completion when thread has clone_pending flag
-	useEffect(() => {
-		if (!currentThread?.clone_pending || isCompletingClone || cloneError) return;
-
-		const completeClone = async () => {
-			setIsCompletingClone(true);
-
-			try {
-				await publicChatApiService.completeClone({ thread_id: currentThread.id });
-
-				// Re-initialize thread to fetch cloned content using existing logic
-				await initializeThread();
-
-				// Invalidate threads query to update sidebar
-				queryClient.invalidateQueries({
-					predicate: (query) => Array.isArray(query.queryKey) && query.queryKey[0] === "threads",
-				});
-			} catch (error) {
-				console.error("[NewChatPage] Failed to complete clone:", error);
-				toast.error("Failed to copy chat content. Please try again.");
-				setCloneError(true);
-			} finally {
-				setIsCompletingClone(false);
-			}
-		};
-
-		completeClone();
-	}, [
-		currentThread?.clone_pending,
-		currentThread?.id,
-		isCompletingClone,
-		cloneError,
-		initializeThread,
-		queryClient,
-	]);
-
 	// Handle scroll to comment from URL query params (e.g., from inbox item click)
 	const searchParams = useSearchParams();
 	const targetCommentIdParam = searchParams.get("commentId");
@@ -394,8 +355,6 @@ export default function NewChatPage() {
 			visibility: currentThread?.visibility ?? null,
 			hasComments: currentThread?.has_comments ?? false,
 			addingCommentToMessageId: null,
-			publicShareEnabled: currentThread?.public_share_enabled ?? false,
-			publicShareToken: currentThread?.public_share_token ?? null,
 		}));
 	}, [currentThread, setCurrentThreadState]);
 
@@ -1420,16 +1379,6 @@ export default function NewChatPage() {
 		);
 	}
 
-	// Show loading state while completing clone
-	if (isCompletingClone) {
-		return (
-			<div className="flex h-[calc(100vh-64px)] flex-col items-center justify-center gap-4">
-				<Spinner size="lg" />
-				<div className="text-sm text-muted-foreground">Copying chat content...</div>
-			</div>
-		);
-	}
-
 	// Show error state only if we tried to load an existing thread but failed
 	// For new chats (urlChatId === 0), threadId being null is expected (lazy creation)
 	if (!threadId && urlChatId > 0) {

surfsense_web/atoms/chat/chat-thread-mutation.atoms.ts

@@ -1,28 +1,31 @@
 import { atomWithMutation } from "jotai-tanstack-query";
 import { toast } from "sonner";
 import type {
-	TogglePublicShareRequest,
+	CreateSnapshotRequest,
-	TogglePublicShareResponse,
+	CreateSnapshotResponse,
 } from "@/contracts/types/chat-threads.types";
 import { chatThreadsApiService } from "@/lib/apis/chat-threads-api.service";
 
-export const togglePublicShareMutationAtom = atomWithMutation(() => ({
+export const createSnapshotMutationAtom = atomWithMutation(() => ({
-	mutationFn: async (request: TogglePublicShareRequest) => {
+	mutationFn: async (request: CreateSnapshotRequest) => {
-		return chatThreadsApiService.togglePublicShare(request);
+		return chatThreadsApiService.createSnapshot(request);
 	},
-	onSuccess: (response: TogglePublicShareResponse) => {
+	onSuccess: (response: CreateSnapshotResponse) => {
-		if (response.enabled && response.share_token) {
+		// Construct URL using frontend origin (backend returns its own URL which differs)
-			const publicUrl = `${window.location.origin}/public/${response.share_token}`;
+		const publicUrl = `${window.location.origin}/public/${response.share_token}`;
-			navigator.clipboard.writeText(publicUrl);
+		navigator.clipboard.writeText(publicUrl);
-			toast.success("Public link copied to clipboard", {
+		if (response.is_new) {
-				description: "Anyone with this link can view the chat",
+			toast.success("Public link created and copied to clipboard", {
+				description: "Anyone with this link can view a snapshot of this chat",
 			});
 		} else {
-			toast.success("Public sharing disabled");
+			toast.success("Public link copied to clipboard", {
+				description: "This snapshot already exists",
+			});
 		}
 	},
 	onError: (error: Error) => {
-		console.error("Failed to toggle public share:", error);
+		console.error("Failed to create snapshot:", error);
-		toast.error("Failed to update public sharing");
+		toast.error("Failed to create public link");
 	},
 }));

surfsense_web/atoms/chat/current-thread.atom.ts

@@ -19,8 +19,6 @@ interface CurrentThreadState {
 	addingCommentToMessageId: number | null;
 	/** Whether the right-side comments panel is collapsed (desktop only) */
 	commentsCollapsed: boolean;
-	publicShareEnabled: boolean;
-	publicShareToken: string | null;
 }
 
 const initialState: CurrentThreadState = {
@@ -29,8 +27,6 @@ const initialState: CurrentThreadState = {
 	hasComments: false,
 	addingCommentToMessageId: null,
 	commentsCollapsed: false,
-	publicShareEnabled: false,
-	publicShareToken: null,
 };
 
 export const currentThreadAtom = atom<CurrentThreadState>(initialState);

surfsense_web/components/assistant-ui/connector-popup.tsx

@@ -240,6 +240,10 @@ export const ConnectorIndicator: FC = () => {
 							...editingConnector,
 							config: connectorConfig || editingConnector.config,
 							name: editingConnector.name,
+							// Sync last_indexed_at with live data from Electric SQL for real-time updates
+							last_indexed_at:
+								(connectors as SearchSourceConnector[]).find((c) => c.id === editingConnector.id)
+									?.last_indexed_at ?? editingConnector.last_indexed_at,
 						}}
 						startDate={startDate}
 						endDate={endDate}

surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx

@@ -7,6 +7,7 @@ import { Button } from "@/components/ui/button";
 import { Calendar } from "@/components/ui/calendar";
 import { Label } from "@/components/ui/label";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
+import { formatRelativeDate } from "@/lib/format-date";
 import { cn } from "@/lib/utils";
 
 interface DateRangeSelectorProps {
@@ -26,19 +27,10 @@ export const DateRangeSelector: FC<DateRangeSelectorProps> = ({
 	allowFutureDates = false,
 	lastIndexedAt,
 }) => {
-	// Get the placeholder text for start date based on whether connector was previously indexed
+	const startDatePlaceholder = lastIndexedAt
-	const getStartDatePlaceholder = () => {
+		? `From ${formatRelativeDate(lastIndexedAt)}`
-		if (lastIndexedAt) {
+		: "Default (1 year)";
-			const date = new Date(lastIndexedAt);
+
-			const currentYear = new Date().getFullYear();
-			const indexedYear = date.getFullYear();
-			// Show year only if different from current year
-			const formatStr = indexedYear === currentYear ? "MMM d, HH:mm" : "MMM d, yyyy HH:mm";
-			const formattedDate = format(date, formatStr);
-			return `Since (${formattedDate})`;
-		}
-		return "Default (1 year ago)";
-	};
 	const handleLast30Days = () => {
 		const today = new Date();
 		onStartDateChange(subDays(today, 30));
@@ -88,7 +80,7 @@ export const DateRangeSelector: FC<DateRangeSelectorProps> = ({
 								)}
 							>
 								<CalendarIcon className="mr-2 h-4 w-4" />
-								{startDate ? format(startDate, "PPP") : getStartDatePlaceholder()}
+								{startDate ? format(startDate, "PPP") : startDatePlaceholder}
 							</Button>
 						</PopoverTrigger>
 						<PopoverContent className="w-auto p-0 z-[100]" align="start">

surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/discord-config.tsx

@@ -1,29 +1,188 @@
 "use client";
 
-import { Info } from "lucide-react";
+import { AlertCircle, CheckCircle2, Hash, Info, Megaphone, RefreshCw } from "lucide-react";
-import type { FC } from "react";
+import { type FC, useCallback, useEffect, useState } from "react";
+import { Button } from "@/components/ui/button";
+import { Spinner } from "@/components/ui/spinner";
+import { connectorsApiService, type DiscordChannel } from "@/lib/apis/connectors-api.service";
+import { cn } from "@/lib/utils";
 import type { ConnectorConfigProps } from "../index";
 
 export interface DiscordConfigProps extends ConnectorConfigProps {
 	onNameChange?: (name: string) => void;
 }
 
-export const DiscordConfig: FC<DiscordConfigProps> = () => {
+export const DiscordConfig: FC<DiscordConfigProps> = ({ connector }) => {
+	const [channels, setChannels] = useState<DiscordChannel[]>([]);
+	const [isLoading, setIsLoading] = useState(false);
+	const [error, setError] = useState<string | null>(null);
+	const [lastFetched, setLastFetched] = useState<Date | null>(null);
+
+	const fetchChannels = useCallback(async () => {
+		if (!connector?.id) return;
+
+		setIsLoading(true);
+		setError(null);
+
+		try {
+			const data = await connectorsApiService.getDiscordChannels(connector.id);
+			setChannels(data);
+			setLastFetched(new Date());
+		} catch (err) {
+			console.error("Failed to fetch Discord channels:", err);
+			setError(err instanceof Error ? err.message : "Failed to fetch channels");
+		} finally {
+			setIsLoading(false);
+		}
+	}, [connector?.id]);
+
+	// Fetch channels on mount
+	useEffect(() => {
+		fetchChannels();
+	}, [fetchChannels]);
+
+	// Auto-refresh when user returns to tab
+	useEffect(() => {
+		const handleVisibilityChange = () => {
+			if (document.visibilityState === "visible" && connector?.id) {
+				fetchChannels();
+			}
+		};
+
+		document.addEventListener("visibilitychange", handleVisibilityChange);
+		return () => document.removeEventListener("visibilitychange", handleVisibilityChange);
+	}, [connector?.id, fetchChannels]);
+
+	// Separate channels by indexing capability
+	const readyToIndex = channels.filter((ch) => ch.can_index);
+	const needsPermissions = channels.filter((ch) => !ch.can_index);
+
+	// Format last fetched time
+	const formatLastFetched = () => {
+		if (!lastFetched) return null;
+		const now = new Date();
+		const diffMs = now.getTime() - lastFetched.getTime();
+		const diffSecs = Math.floor(diffMs / 1000);
+		const diffMins = Math.floor(diffSecs / 60);
+
+		if (diffSecs < 60) return "just now";
+		if (diffMins === 1) return "1 minute ago";
+		if (diffMins < 60) return `${diffMins} minutes ago`;
+		return lastFetched.toLocaleTimeString();
+	};
+
 	return (
 		<div className="space-y-6">
+			{/* Info box */}
 			<div className="rounded-xl border border-border bg-primary/5 p-4 flex items-start gap-3">
 				<div className="flex h-8 w-8 items-center justify-center rounded-lg bg-primary/10 shrink-0 mt-0.5">
 					<Info className="size-4" />
 				</div>
 				<div className="text-xs sm:text-sm">
-					<p className="font-medium text-xs sm:text-sm">Add Bot to Servers</p>
 					<p className="text-muted-foreground mt-1 text-[10px] sm:text-sm">
-						Before indexing, make sure the Discord bot has been added to the servers (guilds) you
+						The bot needs &quot;Read Message History&quot; permission to index channels. Ask a
-						want to index. The bot can only access messages from servers it's been added to. Use the
+						server admin to grant this permission for channels shown below.
-						OAuth authorization flow to add the bot to your servers.
 					</p>
 				</div>
 			</div>
+
+			{/* Channels Section */}
+			<div className="space-y-3">
+				<div className="flex items-center justify-between">
+					<div className="flex items-center gap-3">
+						<h3 className="text-sm font-semibold">Channel Access</h3>
+					</div>
+					<div className="flex items-center gap-2">
+						{lastFetched && (
+							<span className="text-[10px] text-muted-foreground">{formatLastFetched()}</span>
+						)}
+						<Button
+							variant="secondary"
+							size="sm"
+							onClick={fetchChannels}
+							disabled={isLoading}
+							className="h-7 px-2.5 text-[11px] bg-slate-400/10 dark:bg-white/10 hover:bg-slate-400/20 dark:hover:bg-white/20 border-slate-400/20 dark:border-white/20"
+						>
+							<RefreshCw className={cn("mr-1.5 size-3", isLoading && "animate-spin")} />
+							Refresh
+						</Button>
+					</div>
+				</div>
+
+				{error && (
+					<div className="rounded-lg border border-destructive/50 bg-destructive/10 p-3 text-xs text-destructive">
+						{error}
+					</div>
+				)}
+
+				{isLoading && channels.length === 0 ? (
+					<div className="flex items-center justify-center py-8">
+						<Spinner size="sm" />
+						<span className="ml-2 text-sm text-muted-foreground">Loading channels</span>
+					</div>
+				) : channels.length === 0 && !error ? (
+					<div className="text-center py-8 text-sm text-muted-foreground">
+						No channels found. Make sure the bot has been added to your Discord server with proper
+						permissions.
+					</div>
+				) : (
+					<div className="rounded-xl bg-slate-400/5 dark:bg-white/5 overflow-hidden">
+						{/* Ready to index */}
+						{readyToIndex.length > 0 && (
+							<div className={cn("p-3", needsPermissions.length > 0 && "border-b border-border")}>
+								<div className="flex items-center gap-2 mb-2">
+									<CheckCircle2 className="size-3.5 text-emerald-500" />
+									<span className="text-[11px] font-medium">Ready to index</span>
+									<span className="text-[10px] text-muted-foreground">
+										{readyToIndex.length} {readyToIndex.length === 1 ? "channel" : "channels"}
+									</span>
+								</div>
+								<div className="flex flex-wrap gap-1.5">
+									{readyToIndex.map((channel) => (
+										<ChannelPill key={channel.id} channel={channel} />
+									))}
+								</div>
+							</div>
+						)}
+
+						{/* Needs permissions */}
+						{needsPermissions.length > 0 && (
+							<div className="p-3">
+								<div className="flex items-center gap-2 mb-2">
+									<AlertCircle className="size-3.5 text-amber-500" />
+									<span className="text-[11px] font-medium">Grant permissions to index</span>
+									<span className="text-[10px] text-muted-foreground">
+										{needsPermissions.length}{" "}
+										{needsPermissions.length === 1 ? "channel" : "channels"}
+									</span>
+								</div>
+								<div className="flex flex-wrap gap-1.5">
+									{needsPermissions.map((channel) => (
+										<ChannelPill key={channel.id} channel={channel} />
+									))}
+								</div>
+							</div>
+						)}
+					</div>
+				)}
+			</div>
+		</div>
+	);
+};
+
+interface ChannelPillProps {
+	channel: DiscordChannel;
+}
+
+const ChannelPill: FC<ChannelPillProps> = ({ channel }) => {
+	return (
+		<div className="inline-flex items-center gap-1 px-2 py-1 rounded-md text-[11px] font-medium bg-slate-400/10 dark:bg-white/10 hover:bg-slate-400/20 dark:hover:bg-white/20 transition-colors">
+			{channel.type === "announcement" ? (
+				<Megaphone className="size-2.5 text-muted-foreground" />
+			) : (
+				<Hash className="size-2.5 text-muted-foreground" />
+			)}
+			<span>{channel.name}</span>
 		</div>
 	);
 };

surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/slack-config.tsx

@@ -1,16 +1,79 @@
 "use client";
 
-import { Info } from "lucide-react";
+import { AlertCircle, CheckCircle2, Hash, Info, Lock, RefreshCw } from "lucide-react";
-import type { FC } from "react";
+import { type FC, useCallback, useEffect, useState } from "react";
+import { Button } from "@/components/ui/button";
+import { Spinner } from "@/components/ui/spinner";
+import { connectorsApiService, type SlackChannel } from "@/lib/apis/connectors-api.service";
+import { cn } from "@/lib/utils";
 import type { ConnectorConfigProps } from "../index";
 
 export interface SlackConfigProps extends ConnectorConfigProps {
 	onNameChange?: (name: string) => void;
 }
 
-export const SlackConfig: FC<SlackConfigProps> = () => {
+export const SlackConfig: FC<SlackConfigProps> = ({ connector }) => {
+	const [channels, setChannels] = useState<SlackChannel[]>([]);
+	const [isLoading, setIsLoading] = useState(false);
+	const [error, setError] = useState<string | null>(null);
+	const [lastFetched, setLastFetched] = useState<Date | null>(null);
+
+	const fetchChannels = useCallback(async () => {
+		if (!connector?.id) return;
+
+		setIsLoading(true);
+		setError(null);
+
+		try {
+			const data = await connectorsApiService.getSlackChannels(connector.id);
+			setChannels(data);
+			setLastFetched(new Date());
+		} catch (err) {
+			console.error("Failed to fetch Slack channels:", err);
+			setError(err instanceof Error ? err.message : "Failed to fetch channels");
+		} finally {
+			setIsLoading(false);
+		}
+	}, [connector?.id]);
+
+	// Fetch channels on mount
+	useEffect(() => {
+		fetchChannels();
+	}, [fetchChannels]);
+
+	// Auto-refresh when user returns to tab
+	useEffect(() => {
+		const handleVisibilityChange = () => {
+			if (document.visibilityState === "visible" && connector?.id) {
+				fetchChannels();
+			}
+		};
+
+		document.addEventListener("visibilitychange", handleVisibilityChange);
+		return () => document.removeEventListener("visibilitychange", handleVisibilityChange);
+	}, [connector?.id, fetchChannels]);
+
+	// Separate channels by bot membership
+	const channelsWithBot = channels.filter((ch) => ch.is_member);
+	const channelsWithoutBot = channels.filter((ch) => !ch.is_member);
+
+	// Format last fetched time
+	const formatLastFetched = () => {
+		if (!lastFetched) return null;
+		const now = new Date();
+		const diffMs = now.getTime() - lastFetched.getTime();
+		const diffSecs = Math.floor(diffMs / 1000);
+		const diffMins = Math.floor(diffSecs / 60);
+
+		if (diffSecs < 60) return "just now";
+		if (diffMins === 1) return "1 minute ago";
+		if (diffMins < 60) return `${diffMins} minutes ago`;
+		return lastFetched.toLocaleTimeString();
+	};
+
 	return (
 		<div className="space-y-6">
+			{/* Info box */}
 			<div className="rounded-xl border border-border bg-primary/5 p-4 flex items-start gap-3">
 				<div className="flex h-8 w-8 items-center justify-center rounded-lg bg-primary/10 shrink-0 mt-0.5">
 					<Info className="size-4" />
@@ -25,6 +88,103 @@ export const SlackConfig: FC<SlackConfigProps> = () => {
 					</p>
 				</div>
 			</div>
+
+			{/* Channels Section */}
+			<div className="space-y-3">
+				<div className="flex items-center justify-between">
+					<div className="flex items-center gap-3">
+						<h3 className="text-sm font-semibold">Channel Access</h3>
+					</div>
+					<div className="flex items-center gap-2">
+						{lastFetched && (
+							<span className="text-[10px] text-muted-foreground">{formatLastFetched()}</span>
+						)}
+						<Button
+							variant="secondary"
+							size="sm"
+							onClick={fetchChannels}
+							disabled={isLoading}
+							className="h-7 px-2.5 text-[11px] bg-slate-400/10 dark:bg-white/10 hover:bg-slate-400/20 dark:hover:bg-white/20 border-slate-400/20 dark:border-white/20"
+						>
+							<RefreshCw className={cn("mr-1.5 size-3", isLoading && "animate-spin")} />
+							Refresh
+						</Button>
+					</div>
+				</div>
+
+				{error && (
+					<div className="rounded-lg border border-destructive/50 bg-destructive/10 p-3 text-xs text-destructive">
+						{error}
+					</div>
+				)}
+
+				{isLoading && channels.length === 0 ? (
+					<div className="flex items-center justify-center py-8">
+						<Spinner size="sm" />
+						<span className="ml-2 text-sm text-muted-foreground">Loading channels</span>
+					</div>
+				) : channels.length === 0 && !error ? (
+					<div className="text-center py-8 text-sm text-muted-foreground">
+						No channels found. Make sure the bot has been added to your Slack workspace.
+					</div>
+				) : (
+					<div className="rounded-xl bg-slate-400/5 dark:bg-white/5 overflow-hidden">
+						{/* Channels with bot access */}
+						{channelsWithBot.length > 0 && (
+							<div className={cn("p-3", channelsWithoutBot.length > 0 && "border-b border-border")}>
+								<div className="flex items-center gap-2 mb-2">
+									<CheckCircle2 className="size-3.5 text-emerald-500" />
+									<span className="text-[11px] font-medium">Ready to index</span>
+									<span className="text-[10px] text-muted-foreground">
+										{channelsWithBot.length} {channelsWithBot.length === 1 ? "channel" : "channels"}
+									</span>
+								</div>
+								<div className="flex flex-wrap gap-1.5">
+									{channelsWithBot.map((channel) => (
+										<ChannelPill key={channel.id} channel={channel} />
+									))}
+								</div>
+							</div>
+						)}
+
+						{/* Channels without bot access */}
+						{channelsWithoutBot.length > 0 && (
+							<div className="p-3">
+								<div className="flex items-center gap-2 mb-2">
+									<AlertCircle className="size-3.5 text-amber-500" />
+									<span className="text-[11px] font-medium">Add bot to index</span>
+									<span className="text-[10px] text-muted-foreground">
+										{channelsWithoutBot.length}{" "}
+										{channelsWithoutBot.length === 1 ? "channel" : "channels"}
+									</span>
+								</div>
+								<div className="flex flex-wrap gap-1.5">
+									{channelsWithoutBot.map((channel) => (
+										<ChannelPill key={channel.id} channel={channel} />
+									))}
+								</div>
+							</div>
+						)}
+					</div>
+				)}
+			</div>
+		</div>
+	);
+};
+
+interface ChannelPillProps {
+	channel: SlackChannel;
+}
+
+const ChannelPill: FC<ChannelPillProps> = ({ channel }) => {
+	return (
+		<div className="inline-flex items-center gap-1 px-2 py-1 rounded-md text-[11px] font-medium bg-slate-400/10 dark:bg-white/10 hover:bg-slate-400/20 dark:hover:bg-white/20 transition-colors">
+			{channel.is_private ? (
+				<Lock className="size-2.5 text-muted-foreground" />
+			) : (
+				<Hash className="size-2.5 text-muted-foreground" />
+			)}
+			<span>{channel.name}</span>
 		</div>
 	);
 };

surfsense_web/components/assistant-ui/connector-popup/hooks/use-indexing-connectors.ts

@@ -5,6 +5,44 @@ import type { SearchSourceConnector } from "@/contracts/types/connector.types";
 import type { InboxItem } from "@/contracts/types/inbox.types";
 import { isConnectorIndexingMetadata } from "@/contracts/types/inbox.types";
 
+/**
+ * Timeout thresholds for stuck task detection
+ *
+ * These align with the backend Celery configuration:
+ * - HARD_TIMEOUT: 8 hours (task_time_limit=28800 in Celery)
+ *   Any task running longer than this is definitely dead.
+ *
+ * - STALE_THRESHOLD: 15 minutes without notification updates
+ *   If heartbeats are being sent every 30s, missing 15+ minutes of updates
+ *   indicates the task has likely crashed or the worker is down.
+ */
+const HARD_TIMEOUT_MS = 8 * 60 * 60 * 1000; // 8 hours in milliseconds
+const STALE_THRESHOLD_MS = 15 * 60 * 1000; // 15 minutes in milliseconds
+
+/**
+ * Check if a notification is stale (no updates for too long)
+ * @param updatedAt - ISO timestamp of last notification update
+ * @returns true if the notification hasn't been updated recently
+ */
+function isNotificationStale(updatedAt: string | null | undefined): boolean {
+	if (!updatedAt) return false;
+	const lastUpdate = new Date(updatedAt).getTime();
+	const now = Date.now();
+	return now - lastUpdate > STALE_THRESHOLD_MS;
+}
+
+/**
+ * Check if a task has exceeded the hard timeout (definitely dead)
+ * @param startedAt - ISO timestamp when the task started
+ * @returns true if the task has been running longer than the hard limit
+ */
+function isTaskTimedOut(startedAt: string | null | undefined): boolean {
+	if (!startedAt) return false;
+	const startTime = new Date(startedAt).getTime();
+	const now = Date.now();
+	return now - startTime > HARD_TIMEOUT_MS;
+}
+
 /**
  * Hook to track which connectors are currently indexing using local state.
  *
@@ -13,6 +51,8 @@ import { isConnectorIndexingMetadata } from "@/contracts/types/inbox.types";
  * 2. Detecting in_progress notifications from Electric SQL to restore state after remounts
  * 3. Clearing indexing state when notifications become completed or failed
  * 4. Clearing indexing state when Electric SQL detects last_indexed_at changed
+ * 5. Detecting stale/stuck tasks that haven't updated in 15+ minutes
+ * 6. Detecting hard timeout (8h) - tasks that definitely cannot still be running
  *
  * The actual `last_indexed_at` value comes from Electric SQL/PGlite, not local state.
  */
@@ -57,6 +97,7 @@ export function useIndexingConnectors(
 
 	// Detect notification status changes and update indexing state accordingly
 	// This restores spinner state after component remounts and handles all status transitions
+	// Also detects stale/stuck tasks that haven't been updated in a while
 	useEffect(() => {
 		if (!inboxItems || inboxItems.length === 0) return;
 
@@ -71,11 +112,26 @@ export function useIndexingConnectors(
 				const metadata = isConnectorIndexingMetadata(item.metadata) ? item.metadata : null;
 				if (!metadata) continue;
 
-				// If status is "in_progress", add connector to indexing set
+				// If status is "in_progress", check if it's actually still running
 				if (metadata.status === "in_progress") {
-					if (!newIndexingIds.has(metadata.connector_id)) {
+					// Check for hard timeout (8h) - task is definitely dead
-						newIndexingIds.add(metadata.connector_id);
+					const timedOut = isTaskTimedOut(metadata.started_at);
-						hasChanges = true;
+
+					// Check for stale notification (15min without updates) - task likely crashed
+					const stale = isNotificationStale(item.updated_at);
+
+					if (timedOut || stale) {
+						// Task is stuck - don't show as indexing
+						if (newIndexingIds.has(metadata.connector_id)) {
+							newIndexingIds.delete(metadata.connector_id);
+							hasChanges = true;
+						}
+					} else {
+						// Task appears to be genuinely running
+						if (!newIndexingIds.has(metadata.connector_id)) {
+							newIndexingIds.add(metadata.connector_id);
+							hasChanges = true;
+						}
 					}
 				}
 				// If status is "completed" or "failed", remove connector from indexing set

surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx

@@ -1,6 +1,5 @@
 "use client";
 
-import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns";
 import { ArrowLeft, Plus, Server } from "lucide-react";
 import type { FC } from "react";
 import { Button } from "@/components/ui/button";
@@ -8,6 +7,7 @@ import { Spinner } from "@/components/ui/spinner";
 import { EnumConnectorName } from "@/contracts/enums/connector";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
 import type { SearchSourceConnector } from "@/contracts/types/connector.types";
+import { formatRelativeDate } from "@/lib/format-date";
 import { cn } from "@/lib/utils";
 import { useConnectorStatus } from "../hooks/use-connector-status";
 import { getConnectorDisplayName } from "../tabs/all-connectors-tab";
@@ -32,38 +32,6 @@ function isIndexableConnector(connectorType: string): boolean {
 	return !nonIndexableTypes.includes(connectorType);
 }
 
-/**
- * Format last indexed date with contextual messages
- */
-function formatLastIndexedDate(dateString: string): string {
-	const date = new Date(dateString);
-	const now = new Date();
-	const minutesAgo = differenceInMinutes(now, date);
-	const daysAgo = differenceInDays(now, date);
-
-	if (minutesAgo < 1) {
-		return "Just now";
-	}
-
-	if (minutesAgo < 60) {
-		return `${minutesAgo} ${minutesAgo === 1 ? "minute" : "minutes"} ago`;
-	}
-
-	if (isToday(date)) {
-		return `Today at ${format(date, "h:mm a")}`;
-	}
-
-	if (isYesterday(date)) {
-		return `Yesterday at ${format(date, "h:mm a")}`;
-	}
-
-	if (daysAgo < 7) {
-		return `${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago`;
-	}
-
-	return format(date, "MMM d, yyyy");
-}
-
 export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
 	connectorType,
 	connectorTitle,
@@ -215,7 +183,7 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
 											<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap truncate">
 												{isIndexableConnector(connector.connector_type)
 													? connector.last_indexed_at
-														? `Last indexed: ${formatLastIndexedDate(connector.last_indexed_at)}`
+														? `Last indexed: ${formatRelativeDate(connector.last_indexed_at)}`
 														: "Never indexed"
 													: "Active"}
 											</p>

surfsense_web/components/layout/ui/sidebar/Sidebar.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { FolderOpen, MessageSquare, PenSquare } from "lucide-react";
+import { FolderOpen, PenSquare } from "lucide-react";
 import { useTranslations } from "next-intl";
 import { Button } from "@/components/ui/button";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
@@ -122,33 +122,15 @@ export function Sidebar({
 
 			{/* Chat sections - fills available space */}
 			{isCollapsed ? (
-				<div className="flex-1 flex flex-col items-center gap-2 py-2 w-[60px]">
+				<div className="flex-1 w-[60px]" />
-					{(chats.length > 0 || sharedChats.length > 0) && (
-						<Tooltip>
-							<TooltipTrigger asChild>
-								<Button
-									variant="ghost"
-									size="icon"
-									className="h-10 w-10"
-									onClick={() => onToggleCollapse?.()}
-								>
-									<MessageSquare className="h-4 w-4" />
-									<span className="sr-only">{t("chats")}</span>
-								</Button>
-							</TooltipTrigger>
-							<TooltipContent side="right">
-								{t("chats")} ({chats.length + sharedChats.length})
-							</TooltipContent>
-						</Tooltip>
-					)}
-				</div>
 			) : (
 				<div className="flex-1 flex flex-col gap-1 py-2 w-[240px] min-h-0 overflow-hidden">
-					{/* Shared Chats Section - takes half the space */}
+					{/* Shared Chats Section - takes only space needed, max 50% */}
 					<SidebarSection
 						title={t("shared_chats")}
 						defaultOpen={true}
-						fillHeight={true}
+						fillHeight={false}
+						className="shrink-0 max-h-[50%] flex flex-col"
 						action={
 							onViewAllSharedChats ? (
 								<Tooltip>
@@ -170,9 +152,9 @@ export function Sidebar({
 						}
 					>
 						{sharedChats.length > 0 ? (
-							<div className="relative flex-1 min-h-0">
+							<div className="relative min-h-0 flex-1">
 								<div
-									className={`flex flex-col gap-0.5 h-full overflow-y-auto scrollbar-thin scrollbar-thumb-muted-foreground/20 scrollbar-track-transparent ${sharedChats.length > 4 ? "pb-8" : ""}`}
+									className={`flex flex-col gap-0.5 max-h-full overflow-y-auto scrollbar-thin scrollbar-thumb-muted-foreground/20 scrollbar-track-transparent ${sharedChats.length > 4 ? "pb-8" : ""}`}
 								>
 									{sharedChats.slice(0, 20).map((chat) => (
 										<ChatListItem
@@ -196,7 +178,7 @@ export function Sidebar({
 						)}
 					</SidebarSection>
 
-					{/* Private Chats Section - takes half the space */}
+					{/* Private Chats Section - fills remaining space */}
 					<SidebarSection
 						title={t("chats")}
 						defaultOpen={true}

surfsense_web/components/layout/ui/sidebar/SidebarSection.tsx

@@ -30,7 +30,12 @@ export function SidebarSection({
 		<Collapsible
 			open={isOpen}
 			onOpenChange={setIsOpen}
-			className={cn("overflow-hidden", fillHeight && "flex flex-col flex-1 min-h-0", className)}
+			className={cn(
+				"overflow-hidden",
+				fillHeight && "flex flex-col min-h-0",
+				fillHeight && isOpen && "flex-1",
+				className
+			)}
 		>
 			<div className="flex items-center group/section shrink-0">
 				<CollapsibleTrigger className="flex flex-1 items-center gap-1.5 px-2 py-1.5 text-xs font-medium text-muted-foreground hover:text-foreground transition-colors min-w-0">
@@ -56,12 +61,8 @@ export function SidebarSection({
 				)}
 			</div>
 
-			<CollapsibleContent
+			<CollapsibleContent className={cn("overflow-hidden flex-1 flex flex-col min-h-0")}>
-				className={cn("overflow-hidden", fillHeight && "flex-1 flex flex-col min-h-0")}
+				<div className={cn("px-2 pb-2 flex-1 flex flex-col min-h-0 overflow-hidden")}>
-			>
-				<div
-					className={cn("px-2 pb-2", fillHeight && "flex-1 flex flex-col min-h-0 overflow-hidden")}
-				>
 					{children}
 				</div>
 			</CollapsibleContent>

surfsense_web/components/new-chat/chat-share-button.tsx

@@ -2,10 +2,10 @@
 
 import { useQueryClient } from "@tanstack/react-query";
 import { useAtomValue, useSetAtom } from "jotai";
-import { Globe, Link2, User, Users } from "lucide-react";
+import { Globe, User, Users } from "lucide-react";
 import { useCallback, useState } from "react";
 import { toast } from "sonner";
-import { togglePublicShareMutationAtom } from "@/atoms/chat/chat-thread-mutation.atoms";
+import { createSnapshotMutationAtom } from "@/atoms/chat/chat-thread-mutation.atoms";
 import { currentThreadAtom, setThreadVisibilityAtom } from "@/atoms/chat/current-thread.atom";
 import { Button } from "@/components/ui/button";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
@@ -49,19 +49,15 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS
 
 	// Use Jotai atom for visibility (single source of truth)
 	const currentThreadState = useAtomValue(currentThreadAtom);
-	const setCurrentThreadState = useSetAtom(currentThreadAtom);
 	const setThreadVisibility = useSetAtom(setThreadVisibilityAtom);
 
-	// Public share mutation
+	// Snapshot creation mutation
-	const { mutateAsync: togglePublicShare, isPending: isTogglingPublic } = useAtomValue(
+	const { mutateAsync: createSnapshot, isPending: isCreatingSnapshot } = useAtomValue(
-		togglePublicShareMutationAtom
+		createSnapshotMutationAtom
 	);
 
 	// Use Jotai visibility if available (synced from chat page), otherwise fall back to thread prop
 	const currentVisibility = currentThreadState.visibility ?? thread?.visibility ?? "PRIVATE";
-	const isPublicEnabled =
-		currentThreadState.publicShareEnabled ?? thread?.public_share_enabled ?? false;
-	const publicShareToken = currentThreadState.publicShareToken ?? null;
 
 	const handleVisibilityChange = useCallback(
 		async (newVisibility: ChatVisibility) => {
@@ -96,45 +92,24 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS
 		[thread, currentVisibility, onVisibilityChange, queryClient, setThreadVisibility]
 	);
 
-	const handlePublicShareToggle = useCallback(async () => {
+	const handleCreatePublicLink = useCallback(async () => {
 		if (!thread) return;
 
 		try {
-			const response = await togglePublicShare({
+			await createSnapshot({ thread_id: thread.id });
-				thread_id: thread.id,
+			setOpen(false);
-				enabled: !isPublicEnabled,
-			});
-
-			// Update atom state with response
-			setCurrentThreadState((prev) => ({
-				...prev,
-				publicShareEnabled: response.enabled,
-				publicShareToken: response.share_token,
-			}));
 		} catch (error) {
-			console.error("Failed to toggle public share:", error);
+			console.error("Failed to create public link:", error);
 		}
-	}, [thread, isPublicEnabled, togglePublicShare, setCurrentThreadState]);
+	}, [thread, createSnapshot]);
-
-	const handleCopyPublicLink = useCallback(async () => {
-		if (!publicShareToken) return;
-
-		const publicUrl = `${window.location.origin}/public/${publicShareToken}`;
-		await navigator.clipboard.writeText(publicUrl);
-		toast.success("Public link copied to clipboard");
-	}, [publicShareToken]);
 
 	// Don't show if no thread (new chat that hasn't been created yet)
 	if (!thread) {
 		return null;
 	}
 
-	const CurrentIcon = isPublicEnabled ? Globe : currentVisibility === "PRIVATE" ? User : Users;
+	const CurrentIcon = currentVisibility === "PRIVATE" ? User : Users;
-	const buttonLabel = isPublicEnabled
+	const buttonLabel = currentVisibility === "PRIVATE" ? "Private" : "Shared";
-		? "Public"
-		: currentVisibility === "PRIVATE"
-			? "Private"
-			: "Shared";
 
 	return (
 		<Popover open={open} onOpenChange={setOpen}>
@@ -211,67 +186,31 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS
 					{/* Divider */}
 					<div className="border-t border-border my-1" />
 
-					{/* Public Share Option */}
+					{/* Public Link Option */}
 					<button
 						type="button"
-						onClick={handlePublicShareToggle}
+						onClick={handleCreatePublicLink}
-						disabled={isTogglingPublic}
+						disabled={isCreatingSnapshot}
 						className={cn(
 							"w-full flex items-center gap-2.5 px-2.5 py-2 rounded-md transition-all",
 							"hover:bg-accent/50 cursor-pointer",
 							"focus:outline-none",
-							"disabled:opacity-50 disabled:cursor-not-allowed",
+							"disabled:opacity-50 disabled:cursor-not-allowed"
-							isPublicEnabled && "bg-accent/80"
 						)}
 					>
-						<div
+						<div className="size-7 rounded-md shrink-0 grid place-items-center bg-muted">
-							className={cn(
+							<Globe className="size-4 block text-muted-foreground" />
-								"size-7 rounded-md shrink-0 grid place-items-center",
-								isPublicEnabled ? "bg-primary/10" : "bg-muted"
-							)}
-						>
-							<Globe
-								className={cn(
-									"size-4 block",
-									isPublicEnabled ? "text-primary" : "text-muted-foreground"
-								)}
-							/>
 						</div>
 						<div className="flex-1 text-left min-w-0">
 							<div className="flex items-center gap-1.5">
-								<span className={cn("text-sm font-medium", isPublicEnabled && "text-primary")}>
+								<span className="text-sm font-medium">
-									Public
+									{isCreatingSnapshot ? "Creating link..." : "Create public link"}
 								</span>
-								{isPublicEnabled && (
-									<span className="text-xs bg-primary/10 text-primary px-1.5 py-0.5 rounded">
-										ON
-									</span>
-								)}
 							</div>
 							<p className="text-xs text-muted-foreground mt-0.5 leading-snug">
-								Anyone with the link can read
+								Creates a shareable snapshot of this chat
 							</p>
 						</div>
-						{isPublicEnabled && publicShareToken && (
-							<div
-								role="button"
-								tabIndex={0}
-								onClick={(e) => {
-									e.stopPropagation();
-									handleCopyPublicLink();
-								}}
-								onKeyDown={(e) => {
-									if (e.key === "Enter" || e.key === " ") {
-										e.stopPropagation();
-										handleCopyPublicLink();
-									}
-								}}
-								className="shrink-0 p-1.5 rounded-md hover:bg-muted transition-colors cursor-pointer"
-								title="Copy public link"
-							>
-								<Link2 className="size-4 text-muted-foreground" />
-							</div>
-						)}
 					</button>
 				</div>
 			</PopoverContent>

surfsense_web/components/public-chat/public-chat-footer.tsx

@@ -26,7 +26,7 @@ export function PublicChatFooter({ shareToken }: PublicChatFooterProps) {
 				share_token: shareToken,
 			});
 
-			// Redirect to the new chat page (content will be loaded there)
+			// Redirect to the new chat page with cloned content
 			router.push(`/dashboard/${response.search_space_id}/new-chat/${response.thread_id}`);
 		} catch (error) {
 			const message = error instanceof Error ? error.message : "Failed to copy chat";

surfsense_web/components/tool-ui/generate-podcast.tsx

@@ -2,6 +2,7 @@
 
 import { makeAssistantToolUI } from "@assistant-ui/react";
 import { AlertCircleIcon, MicIcon } from "lucide-react";
+import { useParams, usePathname } from "next/navigation";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { z } from "zod";
 import { Audio } from "@/components/tool-ui/audio";
@@ -172,9 +173,6 @@ function AudioLoadingState({ title }: { title: string }) {
 	);
 }
 
-/**
- * Podcast Player Component - Fetches audio and transcript with authentication
- */
 function PodcastPlayer({
 	podcastId,
 	title,
@@ -186,6 +184,11 @@ function PodcastPlayer({
 	description: string;
 	durationMs?: number;
 }) {
+	const params = useParams();
+	const pathname = usePathname();
+	const isPublicRoute = pathname?.startsWith("/public/");
+	const shareToken = isPublicRoute && typeof params?.token === "string" ? params.token : null;
+
 	const [audioSrc, setAudioSrc] = useState<string | null>(null);
 	const [transcript, setTranscript] = useState<PodcastTranscriptEntry[] | null>(null);
 	const [isLoading, setIsLoading] = useState(true);
@@ -217,30 +220,46 @@ function PodcastPlayer({
 			const timeoutId = setTimeout(() => controller.abort(), 60000); // 60s timeout
 
 			try {
-				// Fetch audio blob and podcast details in parallel
+				let audioBlob: Blob;
-				const [audioResponse, rawPodcastDetails] = await Promise.all([
+				let rawPodcastDetails: unknown = null;
-					authenticatedFetch(
-						`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/podcasts/${podcastId}/audio`,
-						{ method: "GET", signal: controller.signal }
-					),
-					baseApiService.get<unknown>(`/api/v1/podcasts/${podcastId}`),
-				]);
 
-				if (!audioResponse.ok) {
+				if (shareToken) {
-					throw new Error(`Failed to load audio: ${audioResponse.status}`);
+					// Public view - use public endpoints (baseApiService handles no-auth for /api/v1/public/)
+					const [blob, details] = await Promise.all([
+						baseApiService.getBlob(`/api/v1/public/${shareToken}/podcasts/${podcastId}/stream`),
+						baseApiService.get(`/api/v1/public/${shareToken}/podcasts/${podcastId}`),
+					]);
+					audioBlob = blob;
+					rawPodcastDetails = details;
+				} else {
+					// Authenticated view - fetch audio and details in parallel
+					const [audioResponse, details] = await Promise.all([
+						authenticatedFetch(
+							`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/podcasts/${podcastId}/audio`,
+							{ method: "GET", signal: controller.signal }
+						),
+						baseApiService.get<unknown>(`/api/v1/podcasts/${podcastId}`),
+					]);
+
+					if (!audioResponse.ok) {
+						throw new Error(`Failed to load audio: ${audioResponse.status}`);
+					}
+
+					audioBlob = await audioResponse.blob();
+					rawPodcastDetails = details;
 				}
 
-				const audioBlob = await audioResponse.blob();
-
 				// Create object URL from blob
 				const objectUrl = URL.createObjectURL(audioBlob);
 				objectUrlRef.current = objectUrl;
 				setAudioSrc(objectUrl);
 
 				// Parse and validate podcast details, then set transcript
-				const podcastDetails = parsePodcastDetails(rawPodcastDetails);
+				if (rawPodcastDetails) {
-				if (podcastDetails.podcast_transcript) {
+					const podcastDetails = parsePodcastDetails(rawPodcastDetails);
-					setTranscript(podcastDetails.podcast_transcript);
+					if (podcastDetails.podcast_transcript) {
+						setTranscript(podcastDetails.podcast_transcript);
+					}
 				}
 			} finally {
 				clearTimeout(timeoutId);
@@ -255,7 +274,7 @@ function PodcastPlayer({
 		} finally {
 			setIsLoading(false);
 		}
-	}, [podcastId]);
+	}, [podcastId, shareToken]);
 
 	// Load podcast when component mounts
 	useEffect(() => {

surfsense_web/contracts/types/chat-threads.types.ts

@@ -1,19 +1,53 @@
 import { z } from "zod";
 
 /**
- * Toggle public share
+ * Snapshot info
  */
-export const togglePublicShareRequest = z.object({
+export const snapshotInfo = z.object({
-	thread_id: z.number(),
+	id: z.number(),
-	enabled: z.boolean(),
+	share_token: z.string(),
+	public_url: z.string(),
+	created_at: z.string(),
+	message_count: z.number(),
 });
 
-export const togglePublicShareResponse = z.object({
+/**
-	enabled: z.boolean(),
+ * Create snapshot
-	public_url: z.string().nullable(),
+ */
-	share_token: z.string().nullable(),
+export const createSnapshotRequest = z.object({
+	thread_id: z.number(),
+});
+
+export const createSnapshotResponse = z.object({
+	snapshot_id: z.number(),
+	share_token: z.string(),
+	public_url: z.string(),
+	is_new: z.boolean(),
+});
+
+/**
+ * List snapshots
+ */
+export const listSnapshotsRequest = z.object({
+	thread_id: z.number(),
+});
+
+export const listSnapshotsResponse = z.object({
+	snapshots: z.array(snapshotInfo),
+});
+
+/**
+ * Delete snapshot
+ */
+export const deleteSnapshotRequest = z.object({
+	thread_id: z.number(),
+	snapshot_id: z.number(),
 });
 
 // Type exports
-export type TogglePublicShareRequest = z.infer<typeof togglePublicShareRequest>;
+export type SnapshotInfo = z.infer<typeof snapshotInfo>;
-export type TogglePublicShareResponse = z.infer<typeof togglePublicShareResponse>;
+export type CreateSnapshotRequest = z.infer<typeof createSnapshotRequest>;
+export type CreateSnapshotResponse = z.infer<typeof createSnapshotResponse>;
+export type ListSnapshotsRequest = z.infer<typeof listSnapshotsRequest>;
+export type ListSnapshotsResponse = z.infer<typeof listSnapshotsResponse>;
+export type DeleteSnapshotRequest = z.infer<typeof deleteSnapshotRequest>;

surfsense_web/contracts/types/connector.types.ts

@@ -203,6 +203,46 @@ export const listGoogleDriveFoldersResponse = z.object({
 	items: z.array(googleDriveItem),
 });
 
+/**
+ * Slack channel with bot membership status
+ */
+export const slackChannel = z.object({
+	id: z.string(),
+	name: z.string(),
+	is_private: z.boolean(),
+	is_member: z.boolean(),
+});
+
+/**
+ * List Slack channels
+ */
+export const listSlackChannelsRequest = z.object({
+	connector_id: z.number(),
+});
+
+export const listSlackChannelsResponse = z.array(slackChannel);
+
+/**
+ * Discord channel with indexing permission info
+ */
+export const discordChannel = z.object({
+	id: z.string(),
+	name: z.string(),
+	type: z.enum(["text", "announcement"]),
+	position: z.number(),
+	category_id: z.string().nullable().optional(),
+	can_index: z.boolean(),
+});
+
+/**
+ * List Discord channels
+ */
+export const listDiscordChannelsRequest = z.object({
+	connector_id: z.number(),
+});
+
+export const listDiscordChannelsResponse = z.array(discordChannel);
+
 // Inferred types
 export type SearchSourceConnectorType = z.infer<typeof searchSourceConnectorTypeEnum>;
 export type SearchSourceConnector = z.infer<typeof searchSourceConnector>;
@@ -223,3 +263,9 @@ export type ListGitHubRepositoriesResponse = z.infer<typeof listGitHubRepositori
 export type ListGoogleDriveFoldersRequest = z.infer<typeof listGoogleDriveFoldersRequest>;
 export type ListGoogleDriveFoldersResponse = z.infer<typeof listGoogleDriveFoldersResponse>;
 export type GoogleDriveItem = z.infer<typeof googleDriveItem>;
+export type SlackChannel = z.infer<typeof slackChannel>;
+export type ListSlackChannelsRequest = z.infer<typeof listSlackChannelsRequest>;
+export type ListSlackChannelsResponse = z.infer<typeof listSlackChannelsResponse>;
+export type DiscordChannel = z.infer<typeof discordChannel>;
+export type ListDiscordChannelsRequest = z.infer<typeof listDiscordChannelsRequest>;
+export type ListDiscordChannelsResponse = z.infer<typeof listDiscordChannelsResponse>;

surfsense_web/contracts/types/public-chat.types.ts

@@ -39,7 +39,7 @@ export const getPublicChatResponse = z.object({
 });
 
 /**
- * Clone public chat (init)
+ * Clone public chat
  */
 export const clonePublicChatRequest = z.object({
 	share_token: z.string(),
@@ -48,19 +48,6 @@ export const clonePublicChatRequest = z.object({
 export const clonePublicChatResponse = z.object({
 	thread_id: z.number(),
 	search_space_id: z.number(),
-	share_token: z.string(),
-});
-
-/**
- * Complete clone
- */
-export const completeCloneRequest = z.object({
-	thread_id: z.number(),
-});
-
-export const completeCloneResponse = z.object({
-	status: z.string(),
-	message_count: z.number(),
 });
 
 // Type exports
@@ -71,5 +58,3 @@ export type GetPublicChatRequest = z.infer<typeof getPublicChatRequest>;
 export type GetPublicChatResponse = z.infer<typeof getPublicChatResponse>;
 export type ClonePublicChatRequest = z.infer<typeof clonePublicChatRequest>;
 export type ClonePublicChatResponse = z.infer<typeof clonePublicChatResponse>;
-export type CompleteCloneRequest = z.infer<typeof completeCloneRequest>;
-export type CompleteCloneResponse = z.infer<typeof completeCloneResponse>;

surfsense_web/lib/apis/base-api.service.ts

@@ -26,7 +26,7 @@ class BaseApiService {
 	noAuthEndpoints: string[] = ["/auth/jwt/login", "/auth/register", "/auth/refresh"];
 
 	// Prefixes that don't require auth (checked with startsWith)
-	noAuthPrefixes: string[] = ["/api/v1/public/", "/api/v1/podcasts/"];
+	noAuthPrefixes: string[] = ["/api/v1/public/"];
 
 	// Use a getter to always read fresh token from localStorage
 	// This ensures the token is always up-to-date after login/logout

surfsense_web/lib/apis/chat-threads-api.service.ts

@@ -1,31 +1,66 @@
 import {
-	type TogglePublicShareRequest,
+	type CreateSnapshotRequest,
-	type TogglePublicShareResponse,
+	type CreateSnapshotResponse,
-	togglePublicShareRequest,
+	createSnapshotRequest,
-	togglePublicShareResponse,
+	createSnapshotResponse,
+	type DeleteSnapshotRequest,
+	deleteSnapshotRequest,
+	type ListSnapshotsRequest,
+	type ListSnapshotsResponse,
+	listSnapshotsRequest,
+	listSnapshotsResponse,
 } from "@/contracts/types/chat-threads.types";
 import { ValidationError } from "../error";
 import { baseApiService } from "./base-api.service";
 
 class ChatThreadsApiService {
 	/**
-	 * Toggle public sharing for a thread.
+	 * Create a public snapshot for a thread.
-	 * Requires authentication.
 	 */
-	togglePublicShare = async (
+	createSnapshot = async (request: CreateSnapshotRequest): Promise<CreateSnapshotResponse> => {
-		request: TogglePublicShareRequest
+		const parsed = createSnapshotRequest.safeParse(request);
-	): Promise<TogglePublicShareResponse> => {
-		const parsed = togglePublicShareRequest.safeParse(request);
 
 		if (!parsed.success) {
 			const errorMessage = parsed.error.issues.map((issue) => issue.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}
 
-		return baseApiService.patch(
+		return baseApiService.post(
-			`/api/v1/threads/${parsed.data.thread_id}/public-share`,
+			`/api/v1/threads/${parsed.data.thread_id}/snapshots`,
-			togglePublicShareResponse,
+			createSnapshotResponse
-			{ body: { enabled: parsed.data.enabled } }
+		);
+	};
+
+	/**
+	 * List all snapshots for a thread.
+	 */
+	listSnapshots = async (request: ListSnapshotsRequest): Promise<ListSnapshotsResponse> => {
+		const parsed = listSnapshotsRequest.safeParse(request);
+
+		if (!parsed.success) {
+			const errorMessage = parsed.error.issues.map((issue) => issue.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		return baseApiService.get(
+			`/api/v1/threads/${parsed.data.thread_id}/snapshots`,
+			listSnapshotsResponse
+		);
+	};
+
+	/**
+	 * Delete a specific snapshot.
+	 */
+	deleteSnapshot = async (request: DeleteSnapshotRequest): Promise<void> => {
+		const parsed = deleteSnapshotRequest.safeParse(request);
+
+		if (!parsed.success) {
+			const errorMessage = parsed.error.issues.map((issue) => issue.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		await baseApiService.delete(
+			`/api/v1/threads/${parsed.data.thread_id}/snapshots/${parsed.data.snapshot_id}`
 		);
 	};
 }

surfsense_web/lib/apis/connectors-api.service.ts

@@ -5,6 +5,7 @@ import {
 	type DeleteConnectorRequest,
 	deleteConnectorRequest,
 	deleteConnectorResponse,
+	type DiscordChannel,
 	type GetConnectorRequest,
 	type GetConnectorsRequest,
 	getConnectorRequest,
@@ -16,10 +17,13 @@ import {
 	indexConnectorResponse,
 	type ListGitHubRepositoriesRequest,
 	type ListGoogleDriveFoldersRequest,
+	listDiscordChannelsResponse,
 	listGitHubRepositoriesRequest,
 	listGitHubRepositoriesResponse,
 	listGoogleDriveFoldersRequest,
 	listGoogleDriveFoldersResponse,
+	listSlackChannelsResponse,
+	type SlackChannel,
 	type UpdateConnectorRequest,
 	updateConnectorRequest,
 	updateConnectorResponse,
@@ -335,6 +339,36 @@ class ConnectorsApiService {
 			}
 		);
 	};
+
+	// =============================================================================
+	// Slack Connector Methods
+	// =============================================================================
+
+	/**
+	 * Get Slack channels with bot membership status
+	 */
+	getSlackChannels = async (connectorId: number) => {
+		return baseApiService.get(
+			`/api/v1/slack/connector/${connectorId}/channels`,
+			listSlackChannelsResponse
+		);
+	};
+
+	// =============================================================================
+	// Discord Connector Methods
+	// =============================================================================
+
+	/**
+	 * Get Discord text channels for a connector
+	 */
+	getDiscordChannels = async (connectorId: number) => {
+		return baseApiService.get(
+			`/api/v1/discord/connector/${connectorId}/channels`,
+			listDiscordChannelsResponse
+		);
+	};
 }
 
+export type { SlackChannel, DiscordChannel };
+
 export const connectorsApiService = new ConnectorsApiService();

surfsense_web/lib/apis/public-chat-api.service.ts

@@ -1,12 +1,8 @@
 import {
 	type ClonePublicChatRequest,
 	type ClonePublicChatResponse,
-	type CompleteCloneRequest,
-	type CompleteCloneResponse,
 	clonePublicChatRequest,
 	clonePublicChatResponse,
-	completeCloneRequest,
-	completeCloneResponse,
 	type GetPublicChatRequest,
 	type GetPublicChatResponse,
 	getPublicChatRequest,
@@ -18,7 +14,6 @@ import { baseApiService } from "./base-api.service";
 class PublicChatApiService {
 	/**
 	 * Get a public chat by share token.
-	 * No authentication required.
 	 */
 	getPublicChat = async (request: GetPublicChatRequest): Promise<GetPublicChatResponse> => {
 		const parsed = getPublicChatRequest.safeParse(request);
@@ -33,8 +28,6 @@ class PublicChatApiService {
 
 	/**
 	 * Clone a public chat to the user's account.
-	 * Creates an empty thread and returns thread_id for redirect.
-	 * Requires authentication.
 	 */
 	clonePublicChat = async (request: ClonePublicChatRequest): Promise<ClonePublicChatResponse> => {
 		const parsed = clonePublicChatRequest.safeParse(request);
@@ -49,25 +42,6 @@ class PublicChatApiService {
 			clonePublicChatResponse
 		);
 	};
-
-	/**
-	 * Complete the clone by copying messages and podcasts.
-	 * Called from the chat page after redirect.
-	 * Requires authentication.
-	 */
-	completeClone = async (request: CompleteCloneRequest): Promise<CompleteCloneResponse> => {
-		const parsed = completeCloneRequest.safeParse(request);
-
-		if (!parsed.success) {
-			const errorMessage = parsed.error.issues.map((issue) => issue.message).join(", ");
-			throw new ValidationError(`Invalid request: ${errorMessage}`);
-		}
-
-		return baseApiService.post(
-			`/api/v1/threads/${parsed.data.thread_id}/complete-clone`,
-			completeCloneResponse
-		);
-	};
 }
 
 export const publicChatApiService = new PublicChatApiService();

surfsense_web/lib/chat/thread-persistence.ts

@@ -24,9 +24,6 @@ export interface ThreadRecord {
 	created_at: string;
 	updated_at: string;
 	has_comments?: boolean;
-	public_share_enabled?: boolean;
-	public_share_token?: string | null;
-	clone_pending?: boolean;
 }
 
 export interface MessageRecord {

surfsense_web/lib/electric/client.ts

@@ -55,7 +55,8 @@ const pendingSyncs = new Map<string, Promise<SyncHandle>>();
 // Version for sync state - increment this to force fresh sync when Electric config changes
 // v2: user-specific database architecture
 // v3: consistent cutoff date for sync+queries, visibility refresh support
-const SYNC_VERSION = 3;
+// v4: heartbeat-based stale notification detection with updated_at tracking
+const SYNC_VERSION = 4;
 
 // Database name prefix for identifying SurfSense databases
 const DB_PREFIX = "surfsense-";

surfsense_web/lib/format-date.ts

@@ -0,0 +1,24 @@
+import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns";
+
+/**
+ * Format a date string as a human-readable relative time
+ * - < 1 min: "Just now"
+ * - < 60 min: "15m ago"
+ * - Today: "Today, 2:30 PM"
+ * - Yesterday: "Yesterday, 2:30 PM"
+ * - < 7 days: "3d ago"
+ * - Older: "Jan 15, 2026"
+ */
+export function formatRelativeDate(dateString: string): string {
+	const date = new Date(dateString);
+	const now = new Date();
+	const minutesAgo = differenceInMinutes(now, date);
+	const daysAgo = differenceInDays(now, date);
+
+	if (minutesAgo < 1) return "Just now";
+	if (minutesAgo < 60) return `${minutesAgo}m ago`;
+	if (isToday(date)) return `Today, ${format(date, "h:mm a")}`;
+	if (isYesterday(date)) return `Yesterday, ${format(date, "h:mm a")}`;
+	if (daysAgo < 7) return `${daysAgo}d ago`;
+	return format(date, "MMM d, yyyy");
+}