From 738e23b51ab20eb2b82887409e03b8c23514d2d1 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 13 Jan 2026 01:15:33 +0200 Subject: [PATCH] Add SurfSense docs to documents table --- .../app/routes/surfsense_docs_routes.py | 82 +++++++++- .../app/schemas/surfsense_docs.py | 15 ++ .../(manage)/components/DocumentsFilters.tsx | 2 +- .../components/DocumentsTableShell.tsx | 24 ++- .../(manage)/components/RowActions.tsx | 69 ++++---- .../documents/(manage)/page.tsx | 153 +++++++++++++++--- .../contracts/enums/connectorIcons.tsx | 3 + .../contracts/types/document.types.ts | 23 +++ .../lib/apis/documents-api.service.ts | 26 +++ 9 files changed, 338 insertions(+), 59 deletions(-) diff --git a/surfsense_backend/app/routes/surfsense_docs_routes.py b/surfsense_backend/app/routes/surfsense_docs_routes.py index a2de65568..e1713e8a3 100644 --- a/surfsense_backend/app/routes/surfsense_docs_routes.py +++ b/surfsense_backend/app/routes/surfsense_docs_routes.py @@ -7,7 +7,7 @@ on a [citation:doc-XXX] link. """ from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload @@ -17,8 +17,10 @@ from app.db import ( User, get_async_session, ) +from app.schemas import PaginatedResponse from app.schemas.surfsense_docs import ( SurfsenseDocsChunkRead, + SurfsenseDocsDocumentRead, SurfsenseDocsDocumentWithChunksRead, ) from app.users import current_active_user @@ -87,3 +89,81 @@ async def get_surfsense_doc_by_chunk_id( status_code=500, detail=f"Failed to retrieve Surfsense documentation: {e!s}", ) from e + + +@router.get( + "/surfsense-docs", + response_model=PaginatedResponse[SurfsenseDocsDocumentRead], +) +async def list_surfsense_docs( + page: int = 0, + page_size: int = 50, + title: str | None = None, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + """ + List all Surfsense documentation documents. + + Args: + page: Zero-based page index. + page_size: Number of items per page (default: 50). + title: Optional title filter (case-insensitive substring match). + session: Database session (injected). + user: Current authenticated user (injected). + + Returns: + PaginatedResponse[SurfsenseDocsDocumentRead]: Paginated list of Surfsense docs. + """ + try: + # Base query + query = select(SurfsenseDocsDocument) + count_query = select(func.count()).select_from(SurfsenseDocsDocument) + + # Filter by title if provided + if title and title.strip(): + query = query.filter(SurfsenseDocsDocument.title.ilike(f"%{title}%")) + count_query = count_query.filter( + SurfsenseDocsDocument.title.ilike(f"%{title}%") + ) + + # Get total count + total_result = await session.execute(count_query) + total = total_result.scalar() or 0 + + # Calculate offset + offset = page * page_size + + # Get paginated results + result = await session.execute( + query.order_by(SurfsenseDocsDocument.title).offset(offset).limit(page_size) + ) + docs = result.scalars().all() + + # Convert to response format + items = [ + SurfsenseDocsDocumentRead( + id=doc.id, + title=doc.title, + source=doc.source, + content=doc.content, + created_at=doc.created_at, + updated_at=doc.updated_at, + ) + for doc in docs + ] + + has_more = (offset + len(items)) < total + + return PaginatedResponse( + items=items, + total=total, + page=page, + page_size=page_size, + has_more=has_more, + ) + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Failed to list Surfsense documentation: {e!s}", + ) from e diff --git a/surfsense_backend/app/schemas/surfsense_docs.py b/surfsense_backend/app/schemas/surfsense_docs.py index c6029320f..ce32c0ef8 100644 --- a/surfsense_backend/app/schemas/surfsense_docs.py +++ b/surfsense_backend/app/schemas/surfsense_docs.py @@ -2,6 +2,8 @@ Schemas for Surfsense documentation. """ +from datetime import datetime + from pydantic import BaseModel, ConfigDict @@ -14,6 +16,19 @@ class SurfsenseDocsChunkRead(BaseModel): model_config = ConfigDict(from_attributes=True) +class SurfsenseDocsDocumentRead(BaseModel): + """Schema for a Surfsense docs document (without chunks).""" + + id: int + title: str + source: str + content: str + created_at: datetime | None = None + updated_at: datetime | None = None + + model_config = ConfigDict(from_attributes=True) + + class SurfsenseDocsDocumentWithChunksRead(BaseModel): """Schema for a Surfsense docs document with its chunks.""" diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx index 4adb5414c..67413d6f0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx @@ -47,7 +47,7 @@ export function DocumentsFilters({ columnVisibility, onToggleColumn, }: { - typeCounts: Record; + typeCounts: Partial>; selectedIds: Set; onSearch: (v: string) => void; searchValue: string; diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx index 94c0626e6..566e103ac 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx @@ -79,17 +79,25 @@ export function DocumentsTableShell({ [documents, sortKey, sortDesc] ); - const allSelectedOnPage = sorted.length > 0 && sorted.every((d) => selectedIds.has(d.id)); - const someSelectedOnPage = sorted.some((d) => selectedIds.has(d.id)) && !allSelectedOnPage; + // Filter out SURFSENSE_DOCS for selection purposes + const selectableDocs = React.useMemo( + () => sorted.filter((d) => d.document_type !== "SURFSENSE_DOCS"), + [sorted] + ); + + const allSelectedOnPage = + selectableDocs.length > 0 && selectableDocs.every((d) => selectedIds.has(d.id)); + const someSelectedOnPage = + selectableDocs.some((d) => selectedIds.has(d.id)) && !allSelectedOnPage; const toggleAll = (checked: boolean) => { const next = new Set(selectedIds); if (checked) - sorted.forEach((d) => { + selectableDocs.forEach((d) => { next.add(d.id); }); else - sorted.forEach((d) => { + selectableDocs.forEach((d) => { next.delete(d.id); }); setSelectedIds(next); @@ -230,9 +238,10 @@ export function DocumentsTableShell({ const icon = getDocumentTypeIcon(doc.document_type); const title = doc.title; const truncatedTitle = title.length > 30 ? `${title.slice(0, 30)}...` : title; + const isSurfsenseDoc = doc.document_type === "SURFSENSE_DOCS"; return ( toggleOne(doc.id, !!v)} + checked={selectedIds.has(doc.id) && !isSurfsenseDoc} + onCheckedChange={(v) => !isSurfsenseDoc && toggleOne(doc.id, !!v)} + disabled={isSurfsenseDoc} aria-label="Select row" /> diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx index 2fe9ab3e8..d277a84ee 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx @@ -28,6 +28,9 @@ import type { Document } from "./types"; // Only FILE and NOTE document types can be edited const EDITABLE_DOCUMENT_TYPES = ["FILE", "NOTE"] as const; +// SURFSENSE_DOCS are system-managed and cannot be deleted +const NON_DELETABLE_DOCUMENT_TYPES = ["SURFSENSE_DOCS"] as const; + export function RowActions({ document, deleteDocument, @@ -48,6 +51,10 @@ export function RowActions({ document.document_type as (typeof EDITABLE_DOCUMENT_TYPES)[number] ); + const isDeletable = !NON_DELETABLE_DOCUMENT_TYPES.includes( + document.document_type as (typeof NON_DELETABLE_DOCUMENT_TYPES)[number] + ); + const handleDelete = async () => { setIsDeleting(true); try { @@ -120,29 +127,31 @@ export function RowActions({ - - - - - - - -

Delete

- - + + + + +

Delete

+ + + )} {/* Mobile Actions Dropdown */} @@ -165,13 +174,15 @@ export function RowActions({ Metadata - setIsDeleteOpen(true)} - className="text-destructive focus:text-destructive" - > - - Delete - + {isDeletable && ( + setIsDeleteOpen(true)} + className="text-destructive focus:text-destructive" + > + + Delete + + )} diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 019f5796a..368f0f654 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -19,7 +19,7 @@ import { DocumentsFilters } from "./components/DocumentsFilters"; import { DocumentsTableShell, type SortKey } from "./components/DocumentsTableShell"; import { PaginationControls } from "./components/PaginationControls"; import { ProcessingIndicator } from "./components/ProcessingIndicator"; -import type { ColumnVisibility } from "./components/types"; +import type { ColumnVisibility, Document } from "./components/types"; function useDebounced(value: T, delay = 250) { const [debounced, setDebounced] = useState(value); @@ -50,33 +50,43 @@ export default function DocumentsTable() { const [sortKey, setSortKey] = useState("title"); const [sortDesc, setSortDesc] = useState(false); const [selectedIds, setSelectedIds] = useState>(new Set()); - const { data: typeCounts } = useAtomValue(documentTypeCountsAtom); + const { data: rawTypeCounts } = useAtomValue(documentTypeCountsAtom); const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom); - // Build query parameters for fetching documents + // Filter out SURFSENSE_DOCS from active types for regular documents API + const regularDocumentTypes = useMemo( + () => activeTypes.filter((t) => t !== "SURFSENSE_DOCS"), + [activeTypes] + ); + + // Check if only SURFSENSE_DOCS is selected (skip regular docs query) + const onlySurfsenseDocsSelected = + activeTypes.length === 1 && activeTypes[0] === "SURFSENSE_DOCS"; + + // Build query parameters for fetching documents (excluding SURFSENSE_DOCS type) const queryParams = useMemo( () => ({ search_space_id: searchSpaceId, page: pageIndex, page_size: pageSize, - ...(activeTypes.length > 0 && { document_types: activeTypes }), + ...(regularDocumentTypes.length > 0 && { document_types: regularDocumentTypes }), }), - [searchSpaceId, pageIndex, pageSize, activeTypes] + [searchSpaceId, pageIndex, pageSize, regularDocumentTypes] ); - // Build search query parameters + // Build search query parameters (excluding SURFSENSE_DOCS type) const searchQueryParams = useMemo( () => ({ search_space_id: searchSpaceId, page: pageIndex, page_size: pageSize, title: debouncedSearch.trim(), - ...(activeTypes.length > 0 && { document_types: activeTypes }), + ...(regularDocumentTypes.length > 0 && { document_types: regularDocumentTypes }), }), - [searchSpaceId, pageIndex, pageSize, activeTypes, debouncedSearch] + [searchSpaceId, pageIndex, pageSize, regularDocumentTypes, debouncedSearch] ); - // Use query for fetching documents + // Use query for fetching documents (disabled when only SURFSENSE_DOCS is selected) const { data: documentsResponse, isLoading: isDocumentsLoading, @@ -86,10 +96,10 @@ export default function DocumentsTable() { queryKey: cacheKeys.documents.globalQueryParams(queryParams), queryFn: () => documentsApiService.getDocuments({ queryParams }), staleTime: 3 * 60 * 1000, // 3 minutes - enabled: !!searchSpaceId && !debouncedSearch.trim(), + enabled: !!searchSpaceId && !debouncedSearch.trim() && !onlySurfsenseDocsSelected, }); - // Use query for searching documents + // Use query for searching documents (disabled when only SURFSENSE_DOCS is selected) const { data: searchResponse, isLoading: isSearchLoading, @@ -99,16 +109,109 @@ export default function DocumentsTable() { queryKey: cacheKeys.documents.globalQueryParams(searchQueryParams), queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }), staleTime: 3 * 60 * 1000, // 3 minutes - enabled: !!searchSpaceId && !!debouncedSearch.trim(), + enabled: !!searchSpaceId && !!debouncedSearch.trim() && !onlySurfsenseDocsSelected, }); + // Determine if we should show SurfSense docs (when no type filter or SURFSENSE_DOCS is selected) + const showSurfsenseDocs = + activeTypes.length === 0 || activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum); + + // Use query for fetching SurfSense docs + const { + data: surfsenseDocsResponse, + isLoading: isSurfsenseDocsLoading, + refetch: refetchSurfsenseDocs, + } = useQuery({ + queryKey: ["surfsense-docs", debouncedSearch, pageIndex, pageSize], + queryFn: () => + documentsApiService.getSurfsenseDocs({ + page: pageIndex, + page_size: pageSize, + title: debouncedSearch.trim() || undefined, + }), + staleTime: 3 * 60 * 1000, // 3 minutes + enabled: showSurfsenseDocs, + }); + + // Transform SurfSense docs to match the Document type + const surfsenseDocsAsDocuments: Document[] = useMemo(() => { + if (!surfsenseDocsResponse?.items) return []; + return surfsenseDocsResponse.items.map((doc) => ({ + id: doc.id, + title: doc.title, + document_type: "SURFSENSE_DOCS", + document_metadata: { source: doc.source }, + content: doc.content, + created_at: doc.created_at || doc.updated_at || new Date().toISOString(), + search_space_id: -1, // Special value for global docs + })); + }, [surfsenseDocsResponse]); + + // Merge type counts with SURFSENSE_DOCS count + const typeCounts = useMemo(() => { + const counts = { ...(rawTypeCounts || {}) }; + if (surfsenseDocsResponse?.total) { + counts.SURFSENSE_DOCS = surfsenseDocsResponse.total; + } + return counts; + }, [rawTypeCounts, surfsenseDocsResponse?.total]); + // Extract documents and total based on search state - const documents = debouncedSearch.trim() + const regularDocuments = debouncedSearch.trim() ? searchResponse?.items || [] : documentsResponse?.items || []; - const total = debouncedSearch.trim() ? searchResponse?.total || 0 : documentsResponse?.total || 0; - const loading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading; - const error = debouncedSearch.trim() ? searchError : documentsError; + const regularTotal = debouncedSearch.trim() + ? searchResponse?.total || 0 + : documentsResponse?.total || 0; + + // Merge regular documents with SurfSense docs + const documents = useMemo(() => { + // If filtering by type and not including SURFSENSE_DOCS, only show regular docs + if (activeTypes.length > 0 && !activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum)) { + return regularDocuments; + } + // If filtering only by SURFSENSE_DOCS, only show surfsense docs + if (activeTypes.length === 1 && activeTypes[0] === "SURFSENSE_DOCS") { + return surfsenseDocsAsDocuments; + } + // Otherwise, merge both (surfsense docs first) + return [...surfsenseDocsAsDocuments, ...regularDocuments]; + }, [regularDocuments, surfsenseDocsAsDocuments, activeTypes]); + + const total = useMemo(() => { + if (activeTypes.length > 0 && !activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum)) { + return regularTotal; + } + if (activeTypes.length === 1 && activeTypes[0] === "SURFSENSE_DOCS") { + return surfsenseDocsResponse?.total || 0; + } + return regularTotal + (surfsenseDocsResponse?.total || 0); + }, [regularTotal, surfsenseDocsResponse?.total, activeTypes]); + + const loading = useMemo(() => { + // If only SURFSENSE_DOCS selected, only check surfsense loading + if (onlySurfsenseDocsSelected) { + return isSurfsenseDocsLoading; + } + // Otherwise check both regular docs and surfsense docs loading + const regularLoading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading; + return regularLoading || (showSurfsenseDocs && isSurfsenseDocsLoading); + }, [ + onlySurfsenseDocsSelected, + isSurfsenseDocsLoading, + debouncedSearch, + isSearchLoading, + isDocumentsLoading, + showSurfsenseDocs, + ]); + + const error = useMemo(() => { + // If only SURFSENSE_DOCS selected, no regular docs errors + if (onlySurfsenseDocsSelected) { + return null; + } + return debouncedSearch.trim() ? searchError : documentsError; + }, [onlySurfsenseDocsSelected, debouncedSearch, searchError, documentsError]); // Display server-filtered results directly const displayDocs = documents || []; @@ -131,16 +234,24 @@ export default function DocumentsTable() { if (isRefreshing) return; setIsRefreshing(true); try { - if (debouncedSearch.trim()) { - await refetchSearch(); - } else { - await refetchDocuments(); + const refetchPromises: Promise[] = []; + // Only refetch regular documents if not in "only surfsense docs" mode + if (!onlySurfsenseDocsSelected) { + if (debouncedSearch.trim()) { + refetchPromises.push(refetchSearch()); + } else { + refetchPromises.push(refetchDocuments()); + } } + if (showSurfsenseDocs) { + refetchPromises.push(refetchSurfsenseDocs()); + } + await Promise.all(refetchPromises); toast.success(t("refresh_success") || "Documents refreshed"); } finally { setIsRefreshing(false); } - }, [debouncedSearch, refetchSearch, refetchDocuments, t, isRefreshing]); + }, [debouncedSearch, refetchSearch, refetchDocuments, refetchSurfsenseDocs, showSurfsenseDocs, onlySurfsenseDocsSelected, t, isRefreshing]); // Set up smart polling for active tasks - only polls when tasks are in progress const { summary } = useLogsSummary(searchSpaceId, 24, { diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index befe132f9..e2d667a16 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -1,5 +1,6 @@ import { IconLinkPlus, IconUsersGroup } from "@tabler/icons-react"; import { + BookOpen, File, FileText, Globe, @@ -86,6 +87,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas return ; case "EXTENSION": return ; + case "SURFSENSE_DOCS": + return ; case "DEEP": return ; case "DEEPER": diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts index 757c6aeb4..c167d4cbf 100644 --- a/surfsense_web/contracts/types/document.types.ts +++ b/surfsense_web/contracts/types/document.types.ts @@ -22,6 +22,7 @@ export const documentTypeEnum = z.enum([ "LINEAR_CONNECTOR", "NOTE", "CIRCLEBACK", + "SURFSENSE_DOCS", ]); export const document = z.object({ @@ -183,6 +184,26 @@ export const getSurfsenseDocsByChunkRequest = z.object({ export const getSurfsenseDocsByChunkResponse = surfsenseDocsDocumentWithChunks; +/** + * List Surfsense docs + */ +export const getSurfsenseDocsRequest = z.object({ + page: z.number().optional(), + page_size: z.number().optional(), + title: z.string().optional(), +}); + +export const getSurfsenseDocsResponse = z.object({ + items: z.array(surfsenseDocsDocument.extend({ + created_at: z.string().nullable().optional(), + updated_at: z.string().nullable().optional(), + })), + total: z.number(), + page: z.number(), + page_size: z.number(), + has_more: z.boolean(), +}); + /** * Update document */ @@ -227,3 +248,5 @@ export type SurfsenseDocsDocument = z.infer; export type SurfsenseDocsDocumentWithChunks = z.infer; export type GetSurfsenseDocsByChunkRequest = z.infer; export type GetSurfsenseDocsByChunkResponse = z.infer; +export type GetSurfsenseDocsRequest = z.infer; +export type GetSurfsenseDocsResponse = z.infer; diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts index 2e7d18e44..91ab154d6 100644 --- a/surfsense_web/lib/apis/documents-api.service.ts +++ b/surfsense_web/lib/apis/documents-api.service.ts @@ -9,6 +9,7 @@ import { type GetDocumentRequest, type GetDocumentsRequest, type GetDocumentTypeCountsRequest, + type GetSurfsenseDocsRequest, getDocumentByChunkRequest, getDocumentByChunkResponse, getDocumentRequest, @@ -18,6 +19,7 @@ import { getDocumentTypeCountsRequest, getDocumentTypeCountsResponse, getSurfsenseDocsByChunkResponse, + getSurfsenseDocsResponse, type SearchDocumentsRequest, searchDocumentsRequest, searchDocumentsResponse, @@ -221,6 +223,30 @@ class DocumentsApiService { ); }; + /** + * List all Surfsense documentation documents + */ + getSurfsenseDocs = async (request: GetSurfsenseDocsRequest = {}) => { + const queryParams = new URLSearchParams(); + + if (request.page !== undefined) { + queryParams.set("page", String(request.page)); + } + if (request.page_size !== undefined) { + queryParams.set("page_size", String(request.page_size)); + } + if (request.title) { + queryParams.set("title", request.title); + } + + const queryString = queryParams.toString(); + const url = queryString + ? `/api/v1/surfsense-docs?${queryString}` + : "/api/v1/surfsense-docs"; + + return baseApiService.get(url, getSurfsenseDocsResponse); + }; + /** * Update a document */