apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-08 07:12:39 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'upstream/dev' into feat/replace-logs

Browse source
This commit is contained in:
Anish Sarkar 2026-01-15 03:07:20 +05:30
commit 2e0f742000
47 changed files with 2365 additions and 700 deletions
Download patch file Download diff file Expand all files Collapse all files

54
surfsense_backend/alembic/versions/0_initial_schema.py Normal file
View file

@ -0,0 +1,54 @@
"""Initial schema setup
Revision ID: 0
Revises: None
Creates all tables from SQLAlchemy models. Idempotent - safe to run on existing databases.
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "0"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
from app.db import Base
connection = op.get_bind()
# Create tables
op.execute(sa.text("CREATE EXTENSION IF NOT EXISTS vector"))
Base.metadata.create_all(bind=connection)
# Set up indexes
op.execute(
sa.text(
"CREATE INDEX IF NOT EXISTS document_vector_index ON documents USING hnsw (embedding public.vector_cosine_ops)"
)
)
op.execute(
sa.text(
"CREATE INDEX IF NOT EXISTS document_search_index ON documents USING gin (to_tsvector('english', content))"
)
)
op.execute(
sa.text(
"CREATE INDEX IF NOT EXISTS chucks_vector_index ON chunks USING hnsw (embedding public.vector_cosine_ops)"
)
)
op.execute(
sa.text(
"CREATE INDEX IF NOT EXISTS chucks_search_index ON chunks USING gin (to_tsvector('english', content))"
)
)
def downgrade() -> None:
pass

34
surfsense_backend/alembic/versions/10_update_chattype_enum_to_qna_report_structure.py
View file

@ -6,6 +6,8 @@ Revises: 9
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
@ -18,9 +20,37 @@ depends_on: str | Sequence[str] | None = None
CHAT_TYPE_ENUM = "chattype"
def enum_exists(enum_name: str) -> bool:
"""Check if an enum type exists in the database."""
conn = op.get_bind()
result = conn.execute(
sa.text(
"SELECT EXISTS (SELECT 1 FROM pg_type WHERE typname = :enum_name)"
),
{"enum_name": enum_name},
)
return result.scalar()
def table_exists(table_name: str) -> bool:
"""Check if a table exists in the database."""
conn = op.get_bind()
result = conn.execute(
sa.text(
"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = :table_name)"
),
{"table_name": table_name},
)
return result.scalar()
def upgrade() -> None:
"""Upgrade schema - replace ChatType enum values with new QNA/REPORT structure."""
# Skip if chats table or chattype enum doesn't exist (fresh database)
if not table_exists("chats") or not enum_exists(CHAT_TYPE_ENUM):
return
# Old enum name for temporary storage
old_enum_name = f"{CHAT_TYPE_ENUM}_old"
@ -72,6 +102,10 @@ def upgrade() -> None:
def downgrade() -> None:
"""Downgrade schema - revert ChatType enum to old GENERAL/DEEP/DEEPER/DEEPEST structure."""
# Skip if chats table or chattype enum doesn't exist
if not table_exists("chats") or not enum_exists(CHAT_TYPE_ENUM):
return
# Old enum name for temporary storage
old_enum_name = f"{CHAT_TYPE_ENUM}_old"

28
surfsense_backend/alembic/versions/1_add_github_connector_enum.py
View file

@ -7,22 +7,36 @@ Revises:
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# Import pgvector if needed for other types, though not for this ENUM change
# import pgvector
# revision identifiers, used by Alembic.
revision: str = "1"
down_revision: str | None = None
down_revision: str | None = "0"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def enum_exists(enum_name: str) -> bool:
"""Check if an enum type exists in the database."""
conn = op.get_bind()
result = conn.execute(
sa.text(
"SELECT EXISTS (SELECT 1 FROM pg_type WHERE typname = :enum_name)"
),
{"enum_name": enum_name},
)
return result.scalar()
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# Skip if the enum doesn't exist (fresh DB after downgrade - create_db_and_tables will handle it)
if not enum_exists("searchsourceconnectortype"):
return
# Manually add the command to add the enum value
# Note: It's generally better to let autogenerate handle this, but we're bypassing it
op.execute(
@ -51,6 +65,10 @@ END$$;
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# Skip if the enum doesn't exist
if not enum_exists("searchsourceconnectortype"):
return
# Downgrading removal of an enum value is complex and potentially dangerous
# if the value is in use. Often omitted or requires manual SQL based on context.
# For now, we'll just pass. If you needed to reverse this, you'd likely

18
surfsense_backend/alembic/versions/24_fix_null_chat_types.py
View file

@ -7,6 +7,8 @@ Revises: 23
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
@ -16,11 +18,27 @@ branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def table_exists(table_name: str) -> bool:
"""Check if a table exists in the database."""
conn = op.get_bind()
result = conn.execute(
sa.text(
"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = :table_name)"
),
{"table_name": table_name},
)
return result.scalar()
def upgrade() -> None:
"""
Fix any chats with NULL type values by setting them to QNA.
This handles edge cases from previous migrations where type values were not properly migrated.
"""
# Skip if chats table doesn't exist (fresh database)
if not table_exists("chats"):
return
# Update any NULL type values to QNA (the default chat type)
op.execute(
"""

69
surfsense_backend/alembic/versions/34_add_podcast_staleness_detection.py
View file

@ -10,6 +10,8 @@ Revises: 33
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers
@ -19,42 +21,59 @@ branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def table_exists(table_name: str) -> bool:
"""Check if a table exists in the database."""
conn = op.get_bind()
result = conn.execute(
sa.text(
"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = :table_name)"
),
{"table_name": table_name},
)
return result.scalar()
def upgrade() -> None:
"""Add columns only if they don't already exist (safe for re-runs)."""
# Add 'state_version' column to chats table (default 1)
op.execute("""
ALTER TABLE chats
ADD COLUMN IF NOT EXISTS state_version BIGINT DEFAULT 1 NOT NULL
""")
# Skip if chats table doesn't exist (fresh database)
if table_exists("chats"):
op.execute("""
ALTER TABLE chats
ADD COLUMN IF NOT EXISTS state_version BIGINT DEFAULT 1 NOT NULL
""")
# Add 'chat_state_version' column to podcasts table
op.execute("""
ALTER TABLE podcasts
ADD COLUMN IF NOT EXISTS chat_state_version BIGINT
""")
if table_exists("podcasts"):
op.execute("""
ALTER TABLE podcasts
ADD COLUMN IF NOT EXISTS chat_state_version BIGINT
""")
# Add 'chat_id' column to podcasts table
op.execute("""
ALTER TABLE podcasts
ADD COLUMN IF NOT EXISTS chat_id INTEGER
""")
# Add 'chat_id' column to podcasts table
op.execute("""
ALTER TABLE podcasts
ADD COLUMN IF NOT EXISTS chat_id INTEGER
""")
def downgrade() -> None:
"""Remove columns only if they exist."""
op.execute("""
ALTER TABLE podcasts
DROP COLUMN IF EXISTS chat_state_version
""")
if table_exists("podcasts"):
op.execute("""
ALTER TABLE podcasts
DROP COLUMN IF EXISTS chat_state_version
""")
op.execute("""
ALTER TABLE podcasts
DROP COLUMN IF EXISTS chat_id
""")
op.execute("""
ALTER TABLE podcasts
DROP COLUMN IF EXISTS chat_id
""")
op.execute("""
ALTER TABLE chats
DROP COLUMN IF EXISTS state_version
""")
if table_exists("chats"):
op.execute("""
ALTER TABLE chats
DROP COLUMN IF EXISTS state_version
""")

80
surfsense_backend/alembic/versions/49_migrate_old_chats_to_new_chat.py
View file

@ -62,8 +62,25 @@ def parse_timestamp(ts, fallback):
return fallback
def table_exists(table_name: str) -> bool:
"""Check if a table exists in the database."""
conn = op.get_bind()
result = conn.execute(
sa.text(
"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = :table_name)"
),
{"table_name": table_name},
)
return result.scalar()
def upgrade() -> None:
"""Migrate old chats to new_chat_threads and remove old tables."""
# Skip if chats table doesn't exist (fresh database)
if not table_exists("chats"):
print("[Migration 49] Chats table does not exist, skipping migration")
return
connection = op.get_bind()
# Get all old chats
@ -176,36 +193,49 @@ def upgrade() -> None:
print("[Migration 49] Migration complete!")
def enum_exists(enum_name: str) -> bool:
"""Check if an enum type exists in the database."""
conn = op.get_bind()
result = conn.execute(
sa.text(
"SELECT EXISTS (SELECT 1 FROM pg_type WHERE typname = :enum_name)"
),
{"enum_name": enum_name},
)
return result.scalar()
def downgrade() -> None:
"""Recreate old chats table (data cannot be restored)."""
# Recreate chattype enum
# Skip if chats table already exists
if table_exists("chats"):
print("[Migration 49 Downgrade] Chats table already exists, skipping")
return
# Recreate chattype enum if it doesn't exist
if not enum_exists("chattype"):
op.execute(
sa.text("""
CREATE TYPE chattype AS ENUM ('QNA')
""")
)
# Recreate chats table using raw SQL to avoid SQLAlchemy trying to create the enum
op.execute(
sa.text("""
CREATE TYPE chattype AS ENUM ('QNA')
CREATE TABLE chats (
id SERIAL PRIMARY KEY,
type chattype NOT NULL,
title VARCHAR NOT NULL,
initial_connectors VARCHAR[],
messages JSON NOT NULL,
state_version BIGINT NOT NULL DEFAULT 1,
search_space_id INTEGER NOT NULL REFERENCES searchspaces(id) ON DELETE CASCADE,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
)
""")
)
# Recreate chats table
op.create_table(
"chats",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("type", sa.Enum("QNA", name="chattype"), nullable=False),
sa.Column("title", sa.String(), nullable=False, index=True),
sa.Column("initial_connectors", sa.ARRAY(sa.String()), nullable=True),
sa.Column("messages", sa.JSON(), nullable=False),
sa.Column("state_version", sa.BigInteger(), nullable=False, default=1),
sa.Column(
"search_space_id",
sa.Integer(),
sa.ForeignKey("searchspaces.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"created_at",
sa.TIMESTAMP(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
)
op.execute(sa.text("CREATE INDEX ix_chats_id ON chats (id)"))
op.execute(sa.text("CREATE INDEX ix_chats_title ON chats (title)"))
print("[Migration 49 Downgrade] Chats table recreated (data not restored)")

50
surfsense_backend/alembic/versions/5263aa4e7f94_allow_multiple_connectors_with_unique_.py Normal file
View file

@ -0,0 +1,50 @@
"""allow_multiple_connectors_with_unique_names
Revision ID: 5263aa4e7f94
Revises: a1b2c3d4e5f6
Create Date: 2026-01-13 12:23:31.481643
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = '5263aa4e7f94'
down_revision: str | None = 'a1b2c3d4e5f6'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Upgrade schema."""
# Drop the old unique constraint
op.drop_constraint(
'uq_searchspace_user_connector_type',
'search_source_connectors',
type_='unique'
)
# Create new unique constraint that includes name
op.create_unique_constraint(
'uq_searchspace_user_connector_type_name',
'search_source_connectors',
['search_space_id', 'user_id', 'connector_type', 'name']
)
def downgrade() -> None:
"""Downgrade schema."""
# Drop the new constraint
op.drop_constraint(
'uq_searchspace_user_connector_type_name',
'search_source_connectors',
type_='unique'
)
# Restore the old constraint
op.create_unique_constraint(
'uq_searchspace_user_connector_type',
'search_source_connectors',
['search_space_id', 'user_id', 'connector_type']
)

16
surfsense_backend/alembic/versions/52_rename_llm_preference_columns.py
View file

@ -39,7 +39,7 @@ def upgrade():
"""
)
# Rename columns (only if they exist with old names)
# Rename columns (only if source exists and target doesn't already exist)
op.execute(
"""
DO $$
@ -47,6 +47,9 @@ def upgrade():
IF EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'searchspaces' AND column_name = 'fast_llm_id'
) AND NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'searchspaces' AND column_name = 'agent_llm_id'
) THEN
ALTER TABLE searchspaces RENAME COLUMN fast_llm_id TO agent_llm_id;
END IF;
@ -61,6 +64,9 @@ def upgrade():
IF EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'searchspaces' AND column_name = 'long_context_llm_id'
) AND NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'searchspaces' AND column_name = 'document_summary_llm_id'
) THEN
ALTER TABLE searchspaces RENAME COLUMN long_context_llm_id TO document_summary_llm_id;
END IF;
@ -100,7 +106,7 @@ def downgrade():
"""
)
# Rename columns back
# Rename columns back (only if source exists and target doesn't already exist)
op.execute(
"""
DO $$
@ -108,6 +114,9 @@ def downgrade():
IF EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'searchspaces' AND column_name = 'agent_llm_id'
) AND NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'searchspaces' AND column_name = 'fast_llm_id'
) THEN
ALTER TABLE searchspaces RENAME COLUMN agent_llm_id TO fast_llm_id;
END IF;
@ -122,6 +131,9 @@ def downgrade():
IF EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'searchspaces' AND column_name = 'document_summary_llm_id'
) AND NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'searchspaces' AND column_name = 'long_context_llm_id'
) THEN
ALTER TABLE searchspaces RENAME COLUMN document_summary_llm_id TO long_context_llm_id;
END IF;

24
surfsense_backend/alembic/versions/55_rename_google_drive_connector_to_file.py
View file

@ -60,14 +60,28 @@ def downgrade() -> None:
connection = op.get_bind()
connection.execute(
# Only update if the target enum value exists (it won't on fresh databases)
result = connection.execute(
text(
"""
UPDATE documents
SET document_type = 'GOOGLE_DRIVE_CONNECTOR'
WHERE document_type = 'GOOGLE_DRIVE_FILE';
SELECT EXISTS (
SELECT 1 FROM pg_type t
JOIN pg_enum e ON t.oid = e.enumtypid
WHERE t.typname = 'documenttype' AND e.enumlabel = 'GOOGLE_DRIVE_CONNECTOR'
);
"""
)
)
enum_exists = result.scalar()
connection.commit()
if enum_exists:
connection.execute(
text(
"""
UPDATE documents
SET document_type = 'GOOGLE_DRIVE_CONNECTOR'
WHERE document_type = 'GOOGLE_DRIVE_FILE';
"""
)
)
connection.commit()

104
surfsense_backend/alembic/versions/5_remove_title_char_limit.py
View file

@ -18,59 +18,77 @@ branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# Alter Chat table
op.alter_column(
"chats",
"title",
existing_type=sa.String(200),
type_=sa.String(),
existing_nullable=False,
def table_exists(table_name: str) -> bool:
"""Check if a table exists in the database."""
conn = op.get_bind()
result = conn.execute(
sa.text(
"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = :table_name)"
),
{"table_name": table_name},
)
return result.scalar()
def upgrade() -> None:
# Alter Chat table (may not exist on fresh databases, removed in migration 49)
if table_exists("chats"):
op.alter_column(
"chats",
"title",
existing_type=sa.String(200),
type_=sa.String(),
existing_nullable=False,
)
# Alter Document table
op.alter_column(
"documents",
"title",
existing_type=sa.String(200),
type_=sa.String(),
existing_nullable=False,
)
if table_exists("documents"):
op.alter_column(
"documents",
"title",
existing_type=sa.String(200),
type_=sa.String(),
existing_nullable=False,
)
# Alter Podcast table
op.alter_column(
"podcasts",
"title",
existing_type=sa.String(200),
type_=sa.String(),
existing_nullable=False,
)
if table_exists("podcasts"):
op.alter_column(
"podcasts",
"title",
existing_type=sa.String(200),
type_=sa.String(),
existing_nullable=False,
)
def downgrade() -> None:
# Revert Chat table
op.alter_column(
"chats",
"title",
existing_type=sa.String(),
type_=sa.String(200),
existing_nullable=False,
)
if table_exists("chats"):
op.alter_column(
"chats",
"title",
existing_type=sa.String(),
type_=sa.String(200),
existing_nullable=False,
)
# Revert Document table
op.alter_column(
"documents",
"title",
existing_type=sa.String(),
type_=sa.String(200),
existing_nullable=False,
)
if table_exists("documents"):
op.alter_column(
"documents",
"title",
existing_type=sa.String(),
type_=sa.String(200),
existing_nullable=False,
)
# Revert Podcast table
op.alter_column(
"podcasts",
"title",
existing_type=sa.String(),
type_=sa.String(200),
existing_nullable=False,
)
if table_exists("podcasts"):
op.alter_column(
"podcasts",
"title",
existing_type=sa.String(),
type_=sa.String(200),
existing_nullable=False,
)

72
surfsense_backend/alembic/versions/62_add_user_profile_columns.py Normal file
View file

@ -0,0 +1,72 @@
"""Add display_name and avatar_url columns to user table
This migration adds:
- display_name column for user's full name from OAuth
- avatar_url column for user's profile picture URL from OAuth
Revision ID: 62
Revises: 61
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "62"
down_revision: str | None = "61"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add display_name and avatar_url columns to user table."""
# Add display_name column (nullable for existing users)
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'user' AND column_name = 'display_name'
) THEN
ALTER TABLE "user"
ADD COLUMN display_name VARCHAR;
END IF;
END$$;
"""
)
# Add avatar_url column (nullable for existing users)
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'user' AND column_name = 'avatar_url'
) THEN
ALTER TABLE "user"
ADD COLUMN avatar_url VARCHAR;
END IF;
END$$;
"""
)
def downgrade() -> None:
"""Remove display_name and avatar_url columns from user table."""
op.execute(
"""
ALTER TABLE "user"
DROP COLUMN IF EXISTS avatar_url;
"""
)
op.execute(
"""
ALTER TABLE "user"
DROP COLUMN IF EXISTS display_name;
"""
)

47
surfsense_backend/alembic/versions/63_add_message_author_id.py Normal file
View file

@ -0,0 +1,47 @@
"""Add author_id column to new_chat_messages table
Revision ID: 63
Revises: 62
"""
from collections.abc import Sequence
from alembic import op
revision: str = "63"
down_revision: str | None = "62"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add author_id column to new_chat_messages table."""
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'new_chat_messages' AND column_name = 'author_id'
) THEN
ALTER TABLE new_chat_messages
ADD COLUMN author_id UUID REFERENCES "user"(id) ON DELETE SET NULL;
CREATE INDEX IF NOT EXISTS ix_new_chat_messages_author_id
ON new_chat_messages(author_id);
END IF;
END$$;
"""
)
def downgrade() -> None:
"""Remove author_id column from new_chat_messages table."""
op.execute(
"""
DROP INDEX IF EXISTS ix_new_chat_messages_author_id;
ALTER TABLE new_chat_messages
DROP COLUMN IF EXISTS author_id;
"""
)

37
surfsense_backend/alembic/versions/a1b2c3d4e5f6_add_mcp_connector_type.py Normal file
View file

@ -0,0 +1,37 @@
"""Add MCP connector type
Revision ID: a1b2c3d4e5f6
Revises: 61
Create Date: 2026-01-09 15:19:51.827647
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = 'a1b2c3d4e5f6'
down_revision: str | None = '61'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add MCP_CONNECTOR to SearchSourceConnectorType enum."""
# Add new enum value using raw SQL
op.execute(
"""
ALTER TYPE searchsourceconnectortype ADD VALUE IF NOT EXISTS 'MCP_CONNECTOR';
"""
)
def downgrade() -> None:
"""Remove MCP_CONNECTOR from SearchSourceConnectorType enum."""
# Note: PostgreSQL does not support removing enum values directly.
# To downgrade, you would need to:
# 1. Create a new enum without MCP_CONNECTOR
# 2. Alter the column to use the new enum
# 3. Drop the old enum
# This is left as a manual operation if needed.
pass