From 1b77269da64e4e44bbee38eb186a9e4e23f200c2 Mon Sep 17 00:00:00 2001 From: Swayam Date: Sun, 5 Oct 2025 14:30:03 +0530 Subject: [PATCH 1/2] feat: add interface params --- ..._add_inference_params_to_searchspaces_2.py | 32 +++++++++++++ ...b4_add_inference_params_to_searchspaces.py | 46 +++++++++++++++++++ ..._add_inference_params_to_searchspaces_1.py | 32 +++++++++++++ surfsense_backend/app/db.py | 10 ++++ surfsense_backend/app/schemas/search_space.py | 13 +++++- 5 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/alembic/versions/2c3ace3296c0_add_inference_params_to_searchspaces_2.py create mode 100644 surfsense_backend/alembic/versions/4bad510c6eb4_add_inference_params_to_searchspaces.py create mode 100644 surfsense_backend/alembic/versions/9bc8b50514d0_add_inference_params_to_searchspaces_1.py diff --git a/surfsense_backend/alembic/versions/2c3ace3296c0_add_inference_params_to_searchspaces_2.py b/surfsense_backend/alembic/versions/2c3ace3296c0_add_inference_params_to_searchspaces_2.py new file mode 100644 index 000000000..6564fd098 --- /dev/null +++ b/surfsense_backend/alembic/versions/2c3ace3296c0_add_inference_params_to_searchspaces_2.py @@ -0,0 +1,32 @@ +"""add inference_params to searchspaces 2 + +Revision ID: 2c3ace3296c0 +Revises: 9bc8b50514d0 +Create Date: 2025-10-05 12:56:00.739719 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '2c3ace3296c0' +down_revision: Union[str, None] = '9bc8b50514d0' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/surfsense_backend/alembic/versions/4bad510c6eb4_add_inference_params_to_searchspaces.py b/surfsense_backend/alembic/versions/4bad510c6eb4_add_inference_params_to_searchspaces.py new file mode 100644 index 000000000..fec40edcd --- /dev/null +++ b/surfsense_backend/alembic/versions/4bad510c6eb4_add_inference_params_to_searchspaces.py @@ -0,0 +1,46 @@ +"""add inference_params to searchspaces + +Revision ID: 4bad510c6eb4 +Revises: 21 +Create Date: 2025-10-05 10:19:08.790345 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '4bad510c6eb4' +down_revision: Union[str, None] = '21' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('chats', 'type', + existing_type=postgresql.ENUM('QNA', 'REPORT_GENERAL', 'REPORT_DEEP', 'REPORT_DEEPER', name='chattype'), + nullable=False) + op.drop_index(op.f('chucks_search_index'), table_name='chunks', postgresql_using='gin') + op.drop_index(op.f('chucks_vector_index'), table_name='chunks', postgresql_using='hnsw') + op.drop_index(op.f('document_search_index'), table_name='documents', postgresql_using='gin') + op.drop_index(op.f('document_vector_index'), table_name='documents', postgresql_using='hnsw') + op.add_column('searchspaces', sa.Column('inference_params', sa.JSON(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('searchspaces', 'inference_params') + op.create_index(op.f('document_vector_index'), 'documents', ['embedding'], unique=False, postgresql_using='hnsw') + op.create_index(op.f('document_search_index'), 'documents', [sa.literal_column("to_tsvector('english'::regconfig, content)")], unique=False, postgresql_using='gin') + op.create_index(op.f('chucks_vector_index'), 'chunks', ['embedding'], unique=False, postgresql_using='hnsw') + op.create_index(op.f('chucks_search_index'), 'chunks', [sa.literal_column("to_tsvector('english'::regconfig, content)")], unique=False, postgresql_using='gin') + op.alter_column('chats', 'type', + existing_type=postgresql.ENUM('QNA', 'REPORT_GENERAL', 'REPORT_DEEP', 'REPORT_DEEPER', name='chattype'), + nullable=True) + # ### end Alembic commands ### diff --git a/surfsense_backend/alembic/versions/9bc8b50514d0_add_inference_params_to_searchspaces_1.py b/surfsense_backend/alembic/versions/9bc8b50514d0_add_inference_params_to_searchspaces_1.py new file mode 100644 index 000000000..8f7674510 --- /dev/null +++ b/surfsense_backend/alembic/versions/9bc8b50514d0_add_inference_params_to_searchspaces_1.py @@ -0,0 +1,32 @@ +"""add inference_params to searchspaces 1 + +Revision ID: 9bc8b50514d0 +Revises: 4bad510c6eb4 +Create Date: 2025-10-05 12:31:02.448057 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '9bc8b50514d0' +down_revision: Union[str, None] = '4bad510c6eb4' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 1d33d4324..26c9ec4c2 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -27,6 +27,8 @@ from app.config import config from app.retriver.chunks_hybrid_search import ChucksHybridSearchRetriever from app.retriver.documents_hybrid_search import DocumentHybridSearchRetriever +from sqlalchemy.ext.mutable import MutableDict + if config.AUTH_TYPE == "GOOGLE": from fastapi_users.db import SQLAlchemyBaseOAuthAccountTableUUID @@ -204,6 +206,14 @@ class SearchSpace(BaseModel, TimestampMixin): name = Column(String(100), nullable=False, index=True) description = Column(String(500), nullable=True) + inference_params = Column( + MutableDict.as_mutable(JSON), + nullable=True, + server_default=text( + '\'{"temperature":0.0,"max_tokens":0,"top_k":0,"top_p":0}\'::json' + ) + ) + user_id = Column( UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False ) diff --git a/surfsense_backend/app/schemas/search_space.py b/surfsense_backend/app/schemas/search_space.py index 00bfdc0f6..09fe6195b 100644 --- a/surfsense_backend/app/schemas/search_space.py +++ b/surfsense_backend/app/schemas/search_space.py @@ -1,14 +1,21 @@ import uuid from datetime import datetime -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field from .base import IDModel, TimestampModel +class InferenceParams(BaseModel): + temperature: float | None = Field(None, ge=0.0, le=2.0) + max_tokens: int | None = Field(None, ge=0) + top_k: int | None = Field(None, ge=0) + top_p: int | None = Field(None, ge=0, le=1) + class SearchSpaceBase(BaseModel): name: str description: str | None = None + inference_params: InferenceParams | None = None class SearchSpaceCreate(SearchSpaceBase): @@ -16,7 +23,9 @@ class SearchSpaceCreate(SearchSpaceBase): class SearchSpaceUpdate(SearchSpaceBase): - pass + name: str | None = None + description: str | None = None + inference_params: InferenceParams | None = None class SearchSpaceRead(SearchSpaceBase, IDModel, TimestampModel): From f1a6f186a8b68df9811a298211a7cfc5db41a1e0 Mon Sep 17 00:00:00 2001 From: Swayam Date: Sun, 5 Oct 2025 16:12:04 +0530 Subject: [PATCH 2/2] feat: add interface params minor changes --- ..._add_inference_params_to_searchspaces_3.py | 38 +++++++++++++++++++ ..._add_inference_params_to_searchspaces_4.py | 32 ++++++++++++++++ surfsense_backend/app/db.py | 10 +---- surfsense_backend/app/schemas/search_space.py | 2 +- 4 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 surfsense_backend/alembic/versions/b86972495f7d_add_inference_params_to_searchspaces_3.py create mode 100644 surfsense_backend/alembic/versions/d17628273e13_add_inference_params_to_searchspaces_4.py diff --git a/surfsense_backend/alembic/versions/b86972495f7d_add_inference_params_to_searchspaces_3.py b/surfsense_backend/alembic/versions/b86972495f7d_add_inference_params_to_searchspaces_3.py new file mode 100644 index 000000000..99ff8ce0f --- /dev/null +++ b/surfsense_backend/alembic/versions/b86972495f7d_add_inference_params_to_searchspaces_3.py @@ -0,0 +1,38 @@ +"""add inference_params to searchspaces 3 + +Revision ID: b86972495f7d +Revises: 2c3ace3296c0 +Create Date: 2025-10-05 15:05:06.391825 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'b86972495f7d' +down_revision: Union[str, None] = '2c3ace3296c0' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('chucks_search_index'), table_name='chunks', postgresql_using='gin') + op.drop_index(op.f('chucks_vector_index'), table_name='chunks', postgresql_using='hnsw') + op.drop_index(op.f('document_search_index'), table_name='documents', postgresql_using='gin') + op.drop_index(op.f('document_vector_index'), table_name='documents', postgresql_using='hnsw') + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_index(op.f('document_vector_index'), 'documents', ['embedding'], unique=False, postgresql_using='hnsw') + op.create_index(op.f('document_search_index'), 'documents', [sa.literal_column("to_tsvector('english'::regconfig, content)")], unique=False, postgresql_using='gin') + op.create_index(op.f('chucks_vector_index'), 'chunks', ['embedding'], unique=False, postgresql_using='hnsw') + op.create_index(op.f('chucks_search_index'), 'chunks', [sa.literal_column("to_tsvector('english'::regconfig, content)")], unique=False, postgresql_using='gin') + # ### end Alembic commands ### diff --git a/surfsense_backend/alembic/versions/d17628273e13_add_inference_params_to_searchspaces_4.py b/surfsense_backend/alembic/versions/d17628273e13_add_inference_params_to_searchspaces_4.py new file mode 100644 index 000000000..74d1b8fbf --- /dev/null +++ b/surfsense_backend/alembic/versions/d17628273e13_add_inference_params_to_searchspaces_4.py @@ -0,0 +1,32 @@ +"""add inference_params to searchspaces 4 + +Revision ID: d17628273e13 +Revises: b86972495f7d +Create Date: 2025-10-05 15:30:12.651638 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'd17628273e13' +down_revision: Union[str, None] = 'b86972495f7d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 26c9ec4c2..54af73239 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -27,8 +27,6 @@ from app.config import config from app.retriver.chunks_hybrid_search import ChucksHybridSearchRetriever from app.retriver.documents_hybrid_search import DocumentHybridSearchRetriever -from sqlalchemy.ext.mutable import MutableDict - if config.AUTH_TYPE == "GOOGLE": from fastapi_users.db import SQLAlchemyBaseOAuthAccountTableUUID @@ -206,13 +204,7 @@ class SearchSpace(BaseModel, TimestampMixin): name = Column(String(100), nullable=False, index=True) description = Column(String(500), nullable=True) - inference_params = Column( - MutableDict.as_mutable(JSON), - nullable=True, - server_default=text( - '\'{"temperature":0.0,"max_tokens":0,"top_k":0,"top_p":0}\'::json' - ) - ) + inference_params = Column(JSON, nullable=True) user_id = Column( UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False diff --git a/surfsense_backend/app/schemas/search_space.py b/surfsense_backend/app/schemas/search_space.py index 09fe6195b..3dd8da4e8 100644 --- a/surfsense_backend/app/schemas/search_space.py +++ b/surfsense_backend/app/schemas/search_space.py @@ -9,7 +9,7 @@ class InferenceParams(BaseModel): temperature: float | None = Field(None, ge=0.0, le=2.0) max_tokens: int | None = Field(None, ge=0) top_k: int | None = Field(None, ge=0) - top_p: int | None = Field(None, ge=0, le=1) + top_p: float | None = Field(None, ge=0.0, le=1.0) class SearchSpaceBase(BaseModel):