diff --git a/surfsense_backend/app/routes/airtable_add_connector_route.py b/surfsense_backend/app/routes/airtable_add_connector_route.py index d5959cf16..fa124f1c2 100644 --- a/surfsense_backend/app/routes/airtable_add_connector_route.py +++ b/surfsense_backend/app/routes/airtable_add_connector_route.py @@ -69,7 +69,7 @@ def generate_pkce_pair() -> tuple[str, str]: return code_verifier, code_challenge -@router.get("/auth/airtable/connector/add/") +@router.get("/auth/airtable/connector/add") async def connect_airtable(space_id: int, user: User = Depends(current_active_user)): """ Initiate Airtable OAuth flow. @@ -131,7 +131,7 @@ async def connect_airtable(space_id: int, user: User = Depends(current_active_us ) from e -@router.get("/auth/airtable/connector/callback/") +@router.get("/auth/airtable/connector/callback") async def airtable_callback( request: Request, code: str, diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index e003dc260..f77171167 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -130,7 +130,7 @@ async def handle_chat_data( return response -@router.post("/chats/", response_model=ChatRead) +@router.post("/chats", response_model=ChatRead) async def create_chat( chat: ChatCreate, session: AsyncSession = Depends(get_async_session), @@ -164,7 +164,7 @@ async def create_chat( ) from None -@router.get("/chats/", response_model=list[ChatReadWithoutMessages]) +@router.get("/chats", response_model=list[ChatReadWithoutMessages]) async def read_chats( skip: int = 0, limit: int = 100, diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index be2f1c8c7..344a2503d 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -38,7 +38,7 @@ os.environ["UNSTRUCTURED_HAS_PATCHED_LOOP"] = "1" router = APIRouter() -@router.post("/documents/") +@router.post("/documents") async def create_documents( request: DocumentsCreate, session: AsyncSession = Depends(get_async_session), @@ -147,7 +147,7 @@ async def create_documents_file_upload( ) from e -@router.get("/documents/", response_model=PaginatedResponse[DocumentRead]) +@router.get("/documents", response_model=PaginatedResponse[DocumentRead]) async def read_documents( skip: int | None = None, page: int | None = None, @@ -248,7 +248,7 @@ async def read_documents( ) from e -@router.get("/documents/search/", response_model=PaginatedResponse[DocumentRead]) +@router.get("/documents/search", response_model=PaginatedResponse[DocumentRead]) async def search_documents( title: str, skip: int | None = None, @@ -353,6 +353,103 @@ async def search_documents( ) from e +@router.get("/documents/type-counts") +async def get_document_type_counts( + search_space_id: int | None = None, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + """ + Get counts of documents by type for the current user. + + Args: + search_space_id: If provided, restrict counts to a specific search space. + session: Database session (injected). + user: Current authenticated user (injected). + + Returns: + Dict mapping document types to their counts. + """ + try: + from sqlalchemy import func + + query = ( + select(Document.document_type, func.count(Document.id)) + .join(SearchSpace) + .filter(SearchSpace.user_id == user.id) + .group_by(Document.document_type) + ) + + if search_space_id is not None: + query = query.filter(Document.search_space_id == search_space_id) + + result = await session.execute(query) + type_counts = dict(result.all()) + + return type_counts + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to fetch document type counts: {e!s}" + ) from e + + +@router.get("/documents/by-chunk/{chunk_id}", response_model=DocumentWithChunksRead) +async def get_document_by_chunk_id( + chunk_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + """ + Retrieves a document based on a chunk ID, including all its chunks ordered by creation time. + The document's embedding and chunk embeddings are excluded from the response. + """ + try: + # First, get the chunk and verify it exists + chunk_result = await session.execute(select(Chunk).filter(Chunk.id == chunk_id)) + chunk = chunk_result.scalars().first() + + if not chunk: + raise HTTPException( + status_code=404, detail=f"Chunk with id {chunk_id} not found" + ) + + # Get the associated document and verify ownership + document_result = await session.execute( + select(Document) + .options(selectinload(Document.chunks)) + .join(SearchSpace) + .filter(Document.id == chunk.document_id, SearchSpace.user_id == user.id) + ) + document = document_result.scalars().first() + + if not document: + raise HTTPException( + status_code=404, + detail="Document not found or you don't have access to it", + ) + + # Sort chunks by creation time + sorted_chunks = sorted(document.chunks, key=lambda x: x.created_at) + + # Return the document with its chunks + return DocumentWithChunksRead( + id=document.id, + title=document.title, + document_type=document.document_type, + document_metadata=document.document_metadata, + content=document.content, + created_at=document.created_at, + search_space_id=document.search_space_id, + chunks=sorted_chunks, + ) + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to retrieve document: {e!s}" + ) from e + + @router.get("/documents/{document_id}", response_model=DocumentRead) async def read_document( document_id: int, @@ -464,100 +561,3 @@ async def delete_document( raise HTTPException( status_code=500, detail=f"Failed to delete document: {e!s}" ) from e - - -@router.get("/documents/type-counts/") -async def get_document_type_counts( - search_space_id: int | None = None, - session: AsyncSession = Depends(get_async_session), - user: User = Depends(current_active_user), -): - """ - Get counts of documents by type for the current user. - - Args: - search_space_id: If provided, restrict counts to a specific search space. - session: Database session (injected). - user: Current authenticated user (injected). - - Returns: - Dict mapping document types to their counts. - """ - try: - from sqlalchemy import func - - query = ( - select(Document.document_type, func.count(Document.id)) - .join(SearchSpace) - .filter(SearchSpace.user_id == user.id) - .group_by(Document.document_type) - ) - - if search_space_id is not None: - query = query.filter(Document.search_space_id == search_space_id) - - result = await session.execute(query) - type_counts = dict(result.all()) - - return type_counts - except Exception as e: - raise HTTPException( - status_code=500, detail=f"Failed to fetch document type counts: {e!s}" - ) from e - - -@router.get("/documents/by-chunk/{chunk_id}", response_model=DocumentWithChunksRead) -async def get_document_by_chunk_id( - chunk_id: int, - session: AsyncSession = Depends(get_async_session), - user: User = Depends(current_active_user), -): - """ - Retrieves a document based on a chunk ID, including all its chunks ordered by creation time. - The document's embedding and chunk embeddings are excluded from the response. - """ - try: - # First, get the chunk and verify it exists - chunk_result = await session.execute(select(Chunk).filter(Chunk.id == chunk_id)) - chunk = chunk_result.scalars().first() - - if not chunk: - raise HTTPException( - status_code=404, detail=f"Chunk with id {chunk_id} not found" - ) - - # Get the associated document and verify ownership - document_result = await session.execute( - select(Document) - .options(selectinload(Document.chunks)) - .join(SearchSpace) - .filter(Document.id == chunk.document_id, SearchSpace.user_id == user.id) - ) - document = document_result.scalars().first() - - if not document: - raise HTTPException( - status_code=404, - detail="Document not found or you don't have access to it", - ) - - # Sort chunks by creation time - sorted_chunks = sorted(document.chunks, key=lambda x: x.created_at) - - # Return the document with its chunks - return DocumentWithChunksRead( - id=document.id, - title=document.title, - document_type=document.document_type, - document_metadata=document.document_metadata, - content=document.content, - created_at=document.created_at, - search_space_id=document.search_space_id, - chunks=sorted_chunks, - ) - except HTTPException: - raise - except Exception as e: - raise HTTPException( - status_code=500, detail=f"Failed to retrieve document: {e!s}" - ) from e diff --git a/surfsense_backend/app/routes/google_calendar_add_connector_route.py b/surfsense_backend/app/routes/google_calendar_add_connector_route.py index e1356809b..fa4ef5466 100644 --- a/surfsense_backend/app/routes/google_calendar_add_connector_route.py +++ b/surfsense_backend/app/routes/google_calendar_add_connector_route.py @@ -53,7 +53,7 @@ def get_google_flow(): ) from e -@router.get("/auth/google/calendar/connector/add/") +@router.get("/auth/google/calendar/connector/add") async def connect_calendar(space_id: int, user: User = Depends(current_active_user)): try: if not space_id: @@ -83,7 +83,7 @@ async def connect_calendar(space_id: int, user: User = Depends(current_active_us ) from e -@router.get("/auth/google/calendar/connector/callback/") +@router.get("/auth/google/calendar/connector/callback") async def calendar_callback( request: Request, code: str, diff --git a/surfsense_backend/app/routes/google_gmail_add_connector_route.py b/surfsense_backend/app/routes/google_gmail_add_connector_route.py index 59badd071..6d37da244 100644 --- a/surfsense_backend/app/routes/google_gmail_add_connector_route.py +++ b/surfsense_backend/app/routes/google_gmail_add_connector_route.py @@ -52,7 +52,7 @@ def get_google_flow(): return flow -@router.get("/auth/google/gmail/connector/add/") +@router.get("/auth/google/gmail/connector/add") async def connect_gmail(space_id: int, user: User = Depends(current_active_user)): try: if not space_id: @@ -82,7 +82,7 @@ async def connect_gmail(space_id: int, user: User = Depends(current_active_user) ) from e -@router.get("/auth/google/gmail/connector/callback/") +@router.get("/auth/google/gmail/connector/callback") async def gmail_callback( request: Request, code: str, diff --git a/surfsense_backend/app/routes/llm_config_routes.py b/surfsense_backend/app/routes/llm_config_routes.py index ec8ea5846..be33632a6 100644 --- a/surfsense_backend/app/routes/llm_config_routes.py +++ b/surfsense_backend/app/routes/llm_config_routes.py @@ -87,7 +87,7 @@ class LLMPreferencesRead(BaseModel): strategic_llm: LLMConfigRead | None = None -@router.post("/llm-configs/", response_model=LLMConfigRead) +@router.post("/llm-configs", response_model=LLMConfigRead) async def create_llm_config( llm_config: LLMConfigCreate, session: AsyncSession = Depends(get_async_session), @@ -112,7 +112,7 @@ async def create_llm_config( ) from e -@router.get("/llm-configs/", response_model=list[LLMConfigRead]) +@router.get("/llm-configs", response_model=list[LLMConfigRead]) async def read_llm_configs( search_space_id: int, skip: int = 0, diff --git a/surfsense_backend/app/routes/logs_routes.py b/surfsense_backend/app/routes/logs_routes.py index cdcb03479..d9dd997ce 100644 --- a/surfsense_backend/app/routes/logs_routes.py +++ b/surfsense_backend/app/routes/logs_routes.py @@ -13,7 +13,7 @@ from app.utils.check_ownership import check_ownership router = APIRouter() -@router.post("/logs/", response_model=LogRead) +@router.post("/logs", response_model=LogRead) async def create_log( log: LogCreate, session: AsyncSession = Depends(get_async_session), @@ -38,7 +38,7 @@ async def create_log( ) from e -@router.get("/logs/", response_model=list[LogRead]) +@router.get("/logs", response_model=list[LogRead]) async def read_logs( skip: int = 0, limit: int = 100, diff --git a/surfsense_backend/app/routes/podcasts_routes.py b/surfsense_backend/app/routes/podcasts_routes.py index bd66c60b8..e37bdd190 100644 --- a/surfsense_backend/app/routes/podcasts_routes.py +++ b/surfsense_backend/app/routes/podcasts_routes.py @@ -21,7 +21,7 @@ from app.utils.check_ownership import check_ownership router = APIRouter() -@router.post("/podcasts/", response_model=PodcastRead) +@router.post("/podcasts", response_model=PodcastRead) async def create_podcast( podcast: PodcastCreate, session: AsyncSession = Depends(get_async_session), @@ -54,7 +54,7 @@ async def create_podcast( ) from None -@router.get("/podcasts/", response_model=list[PodcastRead]) +@router.get("/podcasts", response_model=list[PodcastRead]) async def read_podcasts( skip: int = 0, limit: int = 100, @@ -171,7 +171,7 @@ async def generate_chat_podcast_with_new_session( logging.error(f"Error generating podcast from chat: {e!s}") -@router.post("/podcasts/generate/") +@router.post("/podcasts/generate") async def generate_podcast( request: PodcastGenerateRequest, session: AsyncSession = Depends(get_async_session), diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index bd24efd49..4e62035ff 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -70,7 +70,7 @@ class GitHubPATRequest(BaseModel): # --- New Endpoint to list GitHub Repositories --- -@router.post("/github/repositories/", response_model=list[dict[str, Any]]) +@router.post("/github/repositories", response_model=list[dict[str, Any]]) async def list_github_repositories( pat_request: GitHubPATRequest, user: User = Depends(current_active_user), # Ensure the user is logged in @@ -96,7 +96,7 @@ async def list_github_repositories( ) from e -@router.post("/search-source-connectors/", response_model=SearchSourceConnectorRead) +@router.post("/search-source-connectors", response_model=SearchSourceConnectorRead) async def create_search_source_connector( connector: SearchSourceConnectorCreate, search_space_id: int = Query( @@ -189,9 +189,7 @@ async def create_search_source_connector( ) from e -@router.get( - "/search-source-connectors/", response_model=list[SearchSourceConnectorRead] -) +@router.get("/search-source-connectors", response_model=list[SearchSourceConnectorRead]) async def read_search_source_connectors( skip: int = 0, limit: int = 100, diff --git a/surfsense_backend/app/routes/search_spaces_routes.py b/surfsense_backend/app/routes/search_spaces_routes.py index dc7f69a14..e336178ce 100644 --- a/surfsense_backend/app/routes/search_spaces_routes.py +++ b/surfsense_backend/app/routes/search_spaces_routes.py @@ -10,7 +10,7 @@ from app.utils.check_ownership import check_ownership router = APIRouter() -@router.post("/searchspaces/", response_model=SearchSpaceRead) +@router.post("/searchspaces", response_model=SearchSpaceRead) async def create_search_space( search_space: SearchSpaceCreate, session: AsyncSession = Depends(get_async_session), @@ -31,7 +31,7 @@ async def create_search_space( ) from e -@router.get("/searchspaces/", response_model=list[SearchSpaceRead]) +@router.get("/searchspaces", response_model=list[SearchSpaceRead]) async def read_search_spaces( skip: int = 0, limit: int = 200, diff --git a/surfsense_browser_extension/background/messages/savedata.ts b/surfsense_browser_extension/background/messages/savedata.ts index bd422af19..8719eb365 100644 --- a/surfsense_browser_extension/background/messages/savedata.ts +++ b/surfsense_browser_extension/background/messages/savedata.ts @@ -131,7 +131,7 @@ const handler: PlasmoMessaging.MessageHandler = async (req, res) => { }; const response = await fetch( - `${process.env.PLASMO_PUBLIC_BACKEND_URL}/api/v1/documents/`, + `${process.env.PLASMO_PUBLIC_BACKEND_URL}/api/v1/documents`, requestOptions ); const resp = await response.json(); diff --git a/surfsense_browser_extension/background/messages/savesnapshot.ts b/surfsense_browser_extension/background/messages/savesnapshot.ts index 0f3f0c0a9..8ab60b9fa 100644 --- a/surfsense_browser_extension/background/messages/savesnapshot.ts +++ b/surfsense_browser_extension/background/messages/savesnapshot.ts @@ -123,7 +123,7 @@ const handler: PlasmoMessaging.MessageHandler = async (req, res) => { }; const response = await fetch( - `${process.env.PLASMO_PUBLIC_BACKEND_URL}/api/v1/documents/`, + `${process.env.PLASMO_PUBLIC_BACKEND_URL}/api/v1/documents`, requestOptions ); const resp = await response.json(); diff --git a/surfsense_browser_extension/routes/pages/HomePage.tsx b/surfsense_browser_extension/routes/pages/HomePage.tsx index d1366a8df..362c64056 100644 --- a/surfsense_browser_extension/routes/pages/HomePage.tsx +++ b/surfsense_browser_extension/routes/pages/HomePage.tsx @@ -47,7 +47,7 @@ const HomePage = () => { const token = await storage.get("token"); try { const response = await fetch( - `${process.env.PLASMO_PUBLIC_BACKEND_URL}/api/v1/searchspaces/`, + `${process.env.PLASMO_PUBLIC_BACKEND_URL}/api/v1/searchspaces`, { headers: { Authorization: `Bearer ${token}`, diff --git a/surfsense_web/app/dashboard/[search_space_id]/chats/chats-client.tsx b/surfsense_web/app/dashboard/[search_space_id]/chats/chats-client.tsx index 44a2846bb..cb58acf27 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/chats/chats-client.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/chats/chats-client.tsx @@ -140,7 +140,7 @@ export default function ChatsPageClient({ searchSpaceId }: ChatsPageClientProps) // Fetch all chats for this search space const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chats/?search_space_id=${searchSpaceId}`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chats?search_space_id=${searchSpaceId}`, { headers: { Authorization: `Bearer ${token}`, @@ -285,7 +285,7 @@ export default function ChatsPageClient({ searchSpaceId }: ChatsPageClientProps) }; const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/podcasts/generate/`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/podcasts/generate`, { method: "POST", headers: { diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/baidu-search-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/baidu-search-api/page.tsx index 6624b8214..3e9f4898e 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/baidu-search-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/baidu-search-api/page.tsx @@ -102,6 +102,9 @@ export default function BaiduSearchApiPage() { config, is_indexable: false, last_indexed_at: null, + periodic_indexing_enabled: false, + indexing_frequency_minutes: null, + next_scheduled_at: null, }, parseInt(searchSpaceId) ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx index cc86b1680..2d5f6954c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx @@ -67,6 +67,9 @@ export default function ClickUpConnectorPage() { CLICKUP_API_TOKEN: values.api_token, }, last_indexed_at: null, + periodic_indexing_enabled: false, + indexing_frequency_minutes: null, + next_scheduled_at: null, }; await createConnector(connectorData, parseInt(searchSpaceId)); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx index 361afd5ff..c625f8900 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx @@ -88,6 +88,9 @@ export default function ConfluenceConnectorPage() { }, is_indexable: true, last_indexed_at: null, + periodic_indexing_enabled: false, + indexing_frequency_minutes: null, + next_scheduled_at: null, }, parseInt(searchSpaceId) ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx index 573190945..1daa6bcd0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx @@ -82,6 +82,9 @@ export default function DiscordConnectorPage() { }, is_indexable: true, last_indexed_at: null, + periodic_indexing_enabled: false, + indexing_frequency_minutes: null, + next_scheduled_at: null, }, parseInt(searchSpaceId) ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx index ec0c39e3e..e417995ed 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx @@ -177,7 +177,10 @@ export default function ElasticsearchConnectorPage() { name: values.name, connector_type: EnumConnectorName.ELASTICSEARCH_CONNECTOR, is_indexable: true, - search_space_id: searchSpaceIdNum, + last_indexed_at: null, + periodic_indexing_enabled: false, + indexing_frequency_minutes: null, + next_scheduled_at: null, config, }; @@ -464,7 +467,7 @@ export default function ElasticsearchConnectorPage() {