feat: fixed migrations

- Migrated data from the old 'chats' table to 'new_chat_threads' and 'new_chat_messages'.
- Dropped the 'chats' table and removed the 'chattype' enum as part of the migration process.
- Updated the migration script to truncate thread titles to 500 characters to comply with database constraints.
- Adjusted the downgrade function to reflect changes in the migration process.

surfsense_backend/alembic/versions/49_migrate_old_chats_to_new_chat.py

@@ -6,9 +6,8 @@ Create Date: 2025-12-21
 
 This migration:
 1. Migrates data from old 'chats' table to 'new_chat_threads' and 'new_chat_messages'
-2. Drops the 'podcasts' table (podcast data is not migrated as per user request)
-3. Drops the 'chats' table
-4. Removes the 'chattype' enum
+2. Drops the 'chats' table
+3. Removes the 'chattype' enum
 """
 
 import json
@@ -92,7 +91,11 @@ def upgrade() -> None:
                 print(f"[Migration 49] Skipping empty chat {chat_id}")
                 continue
 
-            # Create new thread
+            # Create new thread - truncate title to 500 chars (VARCHAR(500) limit)
+            thread_title = title or "Migrated Chat"
+            if len(thread_title) > 500:
+                thread_title = thread_title[:497] + "..."
+            
             result = connection.execute(
                 sa.text("""
                     INSERT INTO new_chat_threads 
@@ -101,7 +104,7 @@ def upgrade() -> None:
                     RETURNING id
                 """),
                 {
-                    "title": title or "Migrated Chat",
+                    "title": thread_title,
                     "search_space_id": search_space_id,
                     "created_at": created_at,
                 },
@@ -162,11 +165,7 @@ def upgrade() -> None:
 
     print(f"[Migration 49] Successfully migrated {migrated_count} chats")
 
-    # Drop podcasts table (FK references chats, so drop first)
-    print("[Migration 49] Dropping podcasts table...")
-    op.drop_table("podcasts")
-
-    # Drop chats table
+    # Drop chats table (podcasts table was already updated to remove chat_id FK)
     print("[Migration 49] Dropping chats table...")
     op.drop_table("chats")
 
@@ -178,7 +177,7 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    """Recreate old tables (data cannot be restored)."""
+    """Recreate old chats table (data cannot be restored)."""
     # Recreate chattype enum
     op.execute(
         sa.text("""
@@ -209,32 +208,4 @@ def downgrade() -> None:
         ),
     )
 
-    # Recreate podcasts table
-    op.create_table(
-        "podcasts",
-        sa.Column("id", sa.Integer(), primary_key=True, index=True),
-        sa.Column("title", sa.String(), nullable=False, index=True),
-        sa.Column("podcast_transcript", sa.JSON(), nullable=False, server_default="{}"),
-        sa.Column("file_location", sa.String(500), nullable=False, server_default=""),
-        sa.Column(
-            "chat_id",
-            sa.Integer(),
-            sa.ForeignKey("chats.id", ondelete="CASCADE"),
-            nullable=True,
-        ),
-        sa.Column("chat_state_version", sa.BigInteger(), nullable=True),
-        sa.Column(
-            "search_space_id",
-            sa.Integer(),
-            sa.ForeignKey("searchspaces.id", ondelete="CASCADE"),
-            nullable=False,
-        ),
-        sa.Column(
-            "created_at",
-            sa.TIMESTAMP(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-    )
-
-    print("[Migration 49 Downgrade] Tables recreated (data not restored)")
+    print("[Migration 49 Downgrade] Chats table recreated (data not restored)")

surfsense_backend/alembic/versions/53_cleanup_old_llm_configs.py

@@ -6,8 +6,8 @@ Create Date: 2024-12-22
 
 This migration:
 1. Migrates data from old llm_configs table to new_llm_configs (preserving user configs)
-2. Drops the old llm_configs table (no longer used)
-3. Removes the is_default column from new_llm_configs (roles now determine which config to use)
+2. Updates searchspaces to point to migrated configs
+3. Drops the old llm_configs table (no longer used)
 """
 
 from alembic import op
@@ -47,7 +47,6 @@ def upgrade():
                     system_instructions,
                     use_default_system_instructions,
                     citations_enabled,
-                    is_default,
                     search_space_id,
                     created_at
                 )
@@ -59,11 +58,10 @@ def upgrade():
                     lc.model_name,
                     lc.api_key,
                     lc.api_base,
-                    COALESCE(lc.litellm_params, '{}'::jsonb),
+                    COALESCE(lc.litellm_params::json, '{}'::json),
                     '' as system_instructions,  -- Use defaults
                     TRUE as use_default_system_instructions,
                     TRUE as citations_enabled,
-                    FALSE as is_default,
                     lc.search_space_id,
                     COALESCE(lc.created_at, NOW())
                 FROM llm_configs lc
@@ -130,23 +128,7 @@ def upgrade():
         """
     )
 
-    # STEP 3: Drop the is_default column from new_llm_configs
-    # (role assignments now determine which config to use)
-    op.execute(
-        """
-        DO $$
-        BEGIN
-            IF EXISTS (
-                SELECT 1 FROM information_schema.columns
-                WHERE table_name = 'new_llm_configs' AND column_name = 'is_default'
-            ) THEN
-                ALTER TABLE new_llm_configs DROP COLUMN is_default;
-            END IF;
-        END$$;
-        """
-    )
-
-    # STEP 4: Drop the old llm_configs table (data has been migrated)
+    # STEP 3: Drop the old llm_configs table (data has been migrated)
     op.execute("DROP TABLE IF EXISTS llm_configs CASCADE")
 
 
@@ -213,7 +195,7 @@ def downgrade():
                     nlc.api_key,
                     nlc.api_base,
                     'English' as language,  -- Default language
-                    COALESCE(nlc.litellm_params, '{}'::jsonb),
+                    COALESCE(nlc.litellm_params::jsonb, '{}'::jsonb),
                     nlc.search_space_id,
                     nlc.created_at
                 FROM new_llm_configs nlc
@@ -228,17 +210,3 @@ def downgrade():
         """
     )
 
-    # Add back the is_default column to new_llm_configs
-    op.execute(
-        """
-        DO $$
-        BEGIN
-            IF NOT EXISTS (
-                SELECT 1 FROM information_schema.columns
-                WHERE table_name = 'new_llm_configs' AND column_name = 'is_default'
-            ) THEN
-                ALTER TABLE new_llm_configs ADD COLUMN is_default BOOLEAN NOT NULL DEFAULT FALSE;
-            END IF;
-        END$$;
-        """
-    )