Add SurfSense docs to documents table

surfsense_backend/app/routes/surfsense_docs_routes.py

@@ -7,7 +7,7 @@ on a [citation:doc-XXX] link.
 """
 
 from fastapi import APIRouter, Depends, HTTPException
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import selectinload
 
@@ -17,8 +17,10 @@ from app.db import (
     User,
     get_async_session,
 )
+from app.schemas import PaginatedResponse
 from app.schemas.surfsense_docs import (
     SurfsenseDocsChunkRead,
+    SurfsenseDocsDocumentRead,
     SurfsenseDocsDocumentWithChunksRead,
 )
 from app.users import current_active_user
@@ -87,3 +89,81 @@ async def get_surfsense_doc_by_chunk_id(
             status_code=500,
             detail=f"Failed to retrieve Surfsense documentation: {e!s}",
         ) from e
+
+
+@router.get(
+    "/surfsense-docs",
+    response_model=PaginatedResponse[SurfsenseDocsDocumentRead],
+)
+async def list_surfsense_docs(
+    page: int = 0,
+    page_size: int = 50,
+    title: str | None = None,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    """
+    List all Surfsense documentation documents.
+
+    Args:
+        page: Zero-based page index.
+        page_size: Number of items per page (default: 50).
+        title: Optional title filter (case-insensitive substring match).
+        session: Database session (injected).
+        user: Current authenticated user (injected).
+
+    Returns:
+        PaginatedResponse[SurfsenseDocsDocumentRead]: Paginated list of Surfsense docs.
+    """
+    try:
+        # Base query
+        query = select(SurfsenseDocsDocument)
+        count_query = select(func.count()).select_from(SurfsenseDocsDocument)
+
+        # Filter by title if provided
+        if title and title.strip():
+            query = query.filter(SurfsenseDocsDocument.title.ilike(f"%{title}%"))
+            count_query = count_query.filter(
+                SurfsenseDocsDocument.title.ilike(f"%{title}%")
+            )
+
+        # Get total count
+        total_result = await session.execute(count_query)
+        total = total_result.scalar() or 0
+
+        # Calculate offset
+        offset = page * page_size
+
+        # Get paginated results
+        result = await session.execute(
+            query.order_by(SurfsenseDocsDocument.title).offset(offset).limit(page_size)
+        )
+        docs = result.scalars().all()
+
+        # Convert to response format
+        items = [
+            SurfsenseDocsDocumentRead(
+                id=doc.id,
+                title=doc.title,
+                source=doc.source,
+                content=doc.content,
+                created_at=doc.created_at,
+                updated_at=doc.updated_at,
+            )
+            for doc in docs
+        ]
+
+        has_more = (offset + len(items)) < total
+
+        return PaginatedResponse(
+            items=items,
+            total=total,
+            page=page,
+            page_size=page_size,
+            has_more=has_more,
+        )
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to list Surfsense documentation: {e!s}",
+        ) from e

surfsense_backend/app/schemas/surfsense_docs.py

@@ -2,6 +2,8 @@
 Schemas for Surfsense documentation.
 """
 
+from datetime import datetime
+
 from pydantic import BaseModel, ConfigDict
 
 
@@ -14,6 +16,19 @@ class SurfsenseDocsChunkRead(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
 
+class SurfsenseDocsDocumentRead(BaseModel):
+    """Schema for a Surfsense docs document (without chunks)."""
+
+    id: int
+    title: str
+    source: str
+    content: str
+    created_at: datetime | None = None
+    updated_at: datetime | None = None
+
+    model_config = ConfigDict(from_attributes=True)
+
+
 class SurfsenseDocsDocumentWithChunksRead(BaseModel):
     """Schema for a Surfsense docs document with its chunks."""
 

surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx

@@ -47,7 +47,7 @@ export function DocumentsFilters({
 	columnVisibility,
 	onToggleColumn,
 }: {
-	typeCounts: Record<DocumentTypeEnum, number>;
+	typeCounts: Partial<Record<DocumentTypeEnum, number>>;
 	selectedIds: Set<number>;
 	onSearch: (v: string) => void;
 	searchValue: string;

surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx

@@ -79,17 +79,25 @@ export function DocumentsTableShell({
 		[documents, sortKey, sortDesc]
 	);
 
-	const allSelectedOnPage = sorted.length > 0 && sorted.every((d) => selectedIds.has(d.id));
+	// Filter out SURFSENSE_DOCS for selection purposes
-	const someSelectedOnPage = sorted.some((d) => selectedIds.has(d.id)) && !allSelectedOnPage;
+	const selectableDocs = React.useMemo(
+		() => sorted.filter((d) => d.document_type !== "SURFSENSE_DOCS"),
+		[sorted]
+	);
+
+	const allSelectedOnPage =
+		selectableDocs.length > 0 && selectableDocs.every((d) => selectedIds.has(d.id));
+	const someSelectedOnPage =
+		selectableDocs.some((d) => selectedIds.has(d.id)) && !allSelectedOnPage;
 
 	const toggleAll = (checked: boolean) => {
 		const next = new Set(selectedIds);
 		if (checked)
-			sorted.forEach((d) => {
+			selectableDocs.forEach((d) => {
 				next.add(d.id);
 			});
 		else
-			sorted.forEach((d) => {
+			selectableDocs.forEach((d) => {
 				next.delete(d.id);
 			});
 		setSelectedIds(next);
@@ -230,9 +238,10 @@ export function DocumentsTableShell({
 									const icon = getDocumentTypeIcon(doc.document_type);
 									const title = doc.title;
 									const truncatedTitle = title.length > 30 ? `${title.slice(0, 30)}...` : title;
+									const isSurfsenseDoc = doc.document_type === "SURFSENSE_DOCS";
 									return (
 										<motion.tr
-											key={doc.id}
+											key={`${doc.document_type}-${doc.id}`}
 											initial={{ opacity: 0, y: 10 }}
 											animate={{
 												opacity: 1,
@@ -249,8 +258,9 @@ export function DocumentsTableShell({
 										>
 											<TableCell className="px-4 py-3">
 												<Checkbox
-													checked={selectedIds.has(doc.id)}
+													checked={selectedIds.has(doc.id) && !isSurfsenseDoc}
-													onCheckedChange={(v) => toggleOne(doc.id, !!v)}
+													onCheckedChange={(v) => !isSurfsenseDoc && toggleOne(doc.id, !!v)}
+													disabled={isSurfsenseDoc}
 													aria-label="Select row"
 												/>
 											</TableCell>

surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx

@@ -28,6 +28,9 @@ import type { Document } from "./types";
 // Only FILE and NOTE document types can be edited
 const EDITABLE_DOCUMENT_TYPES = ["FILE", "NOTE"] as const;
 
+// SURFSENSE_DOCS are system-managed and cannot be deleted
+const NON_DELETABLE_DOCUMENT_TYPES = ["SURFSENSE_DOCS"] as const;
+
 export function RowActions({
 	document,
 	deleteDocument,
@@ -48,6 +51,10 @@ export function RowActions({
 		document.document_type as (typeof EDITABLE_DOCUMENT_TYPES)[number]
 	);
 
+	const isDeletable = !NON_DELETABLE_DOCUMENT_TYPES.includes(
+		document.document_type as (typeof NON_DELETABLE_DOCUMENT_TYPES)[number]
+	);
+
 	const handleDelete = async () => {
 		setIsDeleting(true);
 		try {
@@ -120,6 +127,7 @@ export function RowActions({
 					</TooltipContent>
 				</Tooltip>
 
+				{isDeletable && (
 					<Tooltip>
 						<TooltipTrigger asChild>
 							<motion.div
@@ -143,6 +151,7 @@ export function RowActions({
 							<p>Delete</p>
 						</TooltipContent>
 					</Tooltip>
+				)}
 			</div>
 
 			{/* Mobile Actions Dropdown */}
@@ -165,6 +174,7 @@ export function RowActions({
 							<FileText className="mr-2 h-4 w-4" />
 							<span>Metadata</span>
 						</DropdownMenuItem>
+						{isDeletable && (
 							<DropdownMenuItem
 								onClick={() => setIsDeleteOpen(true)}
 								className="text-destructive focus:text-destructive"
@@ -172,6 +182,7 @@ export function RowActions({
 								<Trash2 className="mr-2 h-4 w-4" />
 								<span>Delete</span>
 							</DropdownMenuItem>
+						)}
 					</DropdownMenuContent>
 				</DropdownMenu>
 			</div>

surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx

@@ -19,7 +19,7 @@ import { DocumentsFilters } from "./components/DocumentsFilters";
 import { DocumentsTableShell, type SortKey } from "./components/DocumentsTableShell";
 import { PaginationControls } from "./components/PaginationControls";
 import { ProcessingIndicator } from "./components/ProcessingIndicator";
-import type { ColumnVisibility } from "./components/types";
+import type { ColumnVisibility, Document } from "./components/types";
 
 function useDebounced<T>(value: T, delay = 250) {
 	const [debounced, setDebounced] = useState(value);
@@ -50,33 +50,43 @@ export default function DocumentsTable() {
 	const [sortKey, setSortKey] = useState<SortKey>("title");
 	const [sortDesc, setSortDesc] = useState(false);
 	const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
-	const { data: typeCounts } = useAtomValue(documentTypeCountsAtom);
+	const { data: rawTypeCounts } = useAtomValue(documentTypeCountsAtom);
 	const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom);
 
-	// Build query parameters for fetching documents
+	// Filter out SURFSENSE_DOCS from active types for regular documents API
+	const regularDocumentTypes = useMemo(
+		() => activeTypes.filter((t) => t !== "SURFSENSE_DOCS"),
+		[activeTypes]
+	);
+
+	// Check if only SURFSENSE_DOCS is selected (skip regular docs query)
+	const onlySurfsenseDocsSelected =
+		activeTypes.length === 1 && activeTypes[0] === "SURFSENSE_DOCS";
+
+	// Build query parameters for fetching documents (excluding SURFSENSE_DOCS type)
 	const queryParams = useMemo(
 		() => ({
 			search_space_id: searchSpaceId,
 			page: pageIndex,
 			page_size: pageSize,
-			...(activeTypes.length > 0 && { document_types: activeTypes }),
+			...(regularDocumentTypes.length > 0 && { document_types: regularDocumentTypes }),
 		}),
-		[searchSpaceId, pageIndex, pageSize, activeTypes]
+		[searchSpaceId, pageIndex, pageSize, regularDocumentTypes]
 	);
 
-	// Build search query parameters
+	// Build search query parameters (excluding SURFSENSE_DOCS type)
 	const searchQueryParams = useMemo(
 		() => ({
 			search_space_id: searchSpaceId,
 			page: pageIndex,
 			page_size: pageSize,
 			title: debouncedSearch.trim(),
-			...(activeTypes.length > 0 && { document_types: activeTypes }),
+			...(regularDocumentTypes.length > 0 && { document_types: regularDocumentTypes }),
 		}),
-		[searchSpaceId, pageIndex, pageSize, activeTypes, debouncedSearch]
+		[searchSpaceId, pageIndex, pageSize, regularDocumentTypes, debouncedSearch]
 	);
 
-	// Use query for fetching documents
+	// Use query for fetching documents (disabled when only SURFSENSE_DOCS is selected)
 	const {
 		data: documentsResponse,
 		isLoading: isDocumentsLoading,
@@ -86,10 +96,10 @@ export default function DocumentsTable() {
 		queryKey: cacheKeys.documents.globalQueryParams(queryParams),
 		queryFn: () => documentsApiService.getDocuments({ queryParams }),
 		staleTime: 3 * 60 * 1000, // 3 minutes
-		enabled: !!searchSpaceId && !debouncedSearch.trim(),
+		enabled: !!searchSpaceId && !debouncedSearch.trim() && !onlySurfsenseDocsSelected,
 	});
 
-	// Use query for searching documents
+	// Use query for searching documents (disabled when only SURFSENSE_DOCS is selected)
 	const {
 		data: searchResponse,
 		isLoading: isSearchLoading,
@@ -99,16 +109,109 @@ export default function DocumentsTable() {
 		queryKey: cacheKeys.documents.globalQueryParams(searchQueryParams),
 		queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }),
 		staleTime: 3 * 60 * 1000, // 3 minutes
-		enabled: !!searchSpaceId && !!debouncedSearch.trim(),
+		enabled: !!searchSpaceId && !!debouncedSearch.trim() && !onlySurfsenseDocsSelected,
 	});
 
+	// Determine if we should show SurfSense docs (when no type filter or SURFSENSE_DOCS is selected)
+	const showSurfsenseDocs =
+		activeTypes.length === 0 || activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum);
+
+	// Use query for fetching SurfSense docs
+	const {
+		data: surfsenseDocsResponse,
+		isLoading: isSurfsenseDocsLoading,
+		refetch: refetchSurfsenseDocs,
+	} = useQuery({
+		queryKey: ["surfsense-docs", debouncedSearch, pageIndex, pageSize],
+		queryFn: () =>
+			documentsApiService.getSurfsenseDocs({
+				page: pageIndex,
+				page_size: pageSize,
+				title: debouncedSearch.trim() || undefined,
+			}),
+		staleTime: 3 * 60 * 1000, // 3 minutes
+		enabled: showSurfsenseDocs,
+	});
+
+	// Transform SurfSense docs to match the Document type
+	const surfsenseDocsAsDocuments: Document[] = useMemo(() => {
+		if (!surfsenseDocsResponse?.items) return [];
+		return surfsenseDocsResponse.items.map((doc) => ({
+			id: doc.id,
+			title: doc.title,
+			document_type: "SURFSENSE_DOCS",
+			document_metadata: { source: doc.source },
+			content: doc.content,
+			created_at: doc.created_at || doc.updated_at || new Date().toISOString(),
+			search_space_id: -1, // Special value for global docs
+		}));
+	}, [surfsenseDocsResponse]);
+
+	// Merge type counts with SURFSENSE_DOCS count
+	const typeCounts = useMemo(() => {
+		const counts = { ...(rawTypeCounts || {}) };
+		if (surfsenseDocsResponse?.total) {
+			counts.SURFSENSE_DOCS = surfsenseDocsResponse.total;
+		}
+		return counts;
+	}, [rawTypeCounts, surfsenseDocsResponse?.total]);
+
 	// Extract documents and total based on search state
-	const documents = debouncedSearch.trim()
+	const regularDocuments = debouncedSearch.trim()
 		? searchResponse?.items || []
 		: documentsResponse?.items || [];
-	const total = debouncedSearch.trim() ? searchResponse?.total || 0 : documentsResponse?.total || 0;
+	const regularTotal = debouncedSearch.trim()
-	const loading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading;
+		? searchResponse?.total || 0
-	const error = debouncedSearch.trim() ? searchError : documentsError;
+		: documentsResponse?.total || 0;
+
+	// Merge regular documents with SurfSense docs
+	const documents = useMemo(() => {
+		// If filtering by type and not including SURFSENSE_DOCS, only show regular docs
+		if (activeTypes.length > 0 && !activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum)) {
+			return regularDocuments;
+		}
+		// If filtering only by SURFSENSE_DOCS, only show surfsense docs
+		if (activeTypes.length === 1 && activeTypes[0] === "SURFSENSE_DOCS") {
+			return surfsenseDocsAsDocuments;
+		}
+		// Otherwise, merge both (surfsense docs first)
+		return [...surfsenseDocsAsDocuments, ...regularDocuments];
+	}, [regularDocuments, surfsenseDocsAsDocuments, activeTypes]);
+
+	const total = useMemo(() => {
+		if (activeTypes.length > 0 && !activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum)) {
+			return regularTotal;
+		}
+		if (activeTypes.length === 1 && activeTypes[0] === "SURFSENSE_DOCS") {
+			return surfsenseDocsResponse?.total || 0;
+		}
+		return regularTotal + (surfsenseDocsResponse?.total || 0);
+	}, [regularTotal, surfsenseDocsResponse?.total, activeTypes]);
+
+	const loading = useMemo(() => {
+		// If only SURFSENSE_DOCS selected, only check surfsense loading
+		if (onlySurfsenseDocsSelected) {
+			return isSurfsenseDocsLoading;
+		}
+		// Otherwise check both regular docs and surfsense docs loading
+		const regularLoading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading;
+		return regularLoading || (showSurfsenseDocs && isSurfsenseDocsLoading);
+	}, [
+		onlySurfsenseDocsSelected,
+		isSurfsenseDocsLoading,
+		debouncedSearch,
+		isSearchLoading,
+		isDocumentsLoading,
+		showSurfsenseDocs,
+	]);
+
+	const error = useMemo(() => {
+		// If only SURFSENSE_DOCS selected, no regular docs errors
+		if (onlySurfsenseDocsSelected) {
+			return null;
+		}
+		return debouncedSearch.trim() ? searchError : documentsError;
+	}, [onlySurfsenseDocsSelected, debouncedSearch, searchError, documentsError]);
 
 	// Display server-filtered results directly
 	const displayDocs = documents || [];
@@ -131,16 +234,24 @@ export default function DocumentsTable() {
 		if (isRefreshing) return;
 		setIsRefreshing(true);
 		try {
+			const refetchPromises: Promise<unknown>[] = [];
+			// Only refetch regular documents if not in "only surfsense docs" mode
+			if (!onlySurfsenseDocsSelected) {
 				if (debouncedSearch.trim()) {
-				await refetchSearch();
+					refetchPromises.push(refetchSearch());
 				} else {
-				await refetchDocuments();
+					refetchPromises.push(refetchDocuments());
 				}
+			}
+			if (showSurfsenseDocs) {
+				refetchPromises.push(refetchSurfsenseDocs());
+			}
+			await Promise.all(refetchPromises);
 			toast.success(t("refresh_success") || "Documents refreshed");
 		} finally {
 			setIsRefreshing(false);
 		}
-	}, [debouncedSearch, refetchSearch, refetchDocuments, t, isRefreshing]);
+	}, [debouncedSearch, refetchSearch, refetchDocuments, refetchSurfsenseDocs, showSurfsenseDocs, onlySurfsenseDocsSelected, t, isRefreshing]);
 
 	// Set up smart polling for active tasks - only polls when tasks are in progress
 	const { summary } = useLogsSummary(searchSpaceId, 24, {

surfsense_web/contracts/enums/connectorIcons.tsx

@@ -1,5 +1,6 @@
 import { IconLinkPlus, IconUsersGroup } from "@tabler/icons-react";
 import {
+	BookOpen,
 	File,
 	FileText,
 	Globe,
@@ -86,6 +87,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas
 			return <FileText {...iconProps} />;
 		case "EXTENSION":
 			return <Webhook {...iconProps} />;
+		case "SURFSENSE_DOCS":
+			return <BookOpen {...iconProps} />;
 		case "DEEP":
 			return <Sparkles {...iconProps} />;
 		case "DEEPER":

surfsense_web/contracts/types/document.types.ts

@@ -22,6 +22,7 @@ export const documentTypeEnum = z.enum([
 	"LINEAR_CONNECTOR",
 	"NOTE",
 	"CIRCLEBACK",
+	"SURFSENSE_DOCS",
 ]);
 
 export const document = z.object({
@@ -183,6 +184,26 @@ export const getSurfsenseDocsByChunkRequest = z.object({
 
 export const getSurfsenseDocsByChunkResponse = surfsenseDocsDocumentWithChunks;
 
+/**
+ * List Surfsense docs
+ */
+export const getSurfsenseDocsRequest = z.object({
+	page: z.number().optional(),
+	page_size: z.number().optional(),
+	title: z.string().optional(),
+});
+
+export const getSurfsenseDocsResponse = z.object({
+	items: z.array(surfsenseDocsDocument.extend({
+		created_at: z.string().nullable().optional(),
+		updated_at: z.string().nullable().optional(),
+	})),
+	total: z.number(),
+	page: z.number(),
+	page_size: z.number(),
+	has_more: z.boolean(),
+});
+
 /**
  * Update document
  */
@@ -227,3 +248,5 @@ export type SurfsenseDocsDocument = z.infer<typeof surfsenseDocsDocument>;
 export type SurfsenseDocsDocumentWithChunks = z.infer<typeof surfsenseDocsDocumentWithChunks>;
 export type GetSurfsenseDocsByChunkRequest = z.infer<typeof getSurfsenseDocsByChunkRequest>;
 export type GetSurfsenseDocsByChunkResponse = z.infer<typeof getSurfsenseDocsByChunkResponse>;
+export type GetSurfsenseDocsRequest = z.infer<typeof getSurfsenseDocsRequest>;
+export type GetSurfsenseDocsResponse = z.infer<typeof getSurfsenseDocsResponse>;

surfsense_web/lib/apis/documents-api.service.ts

@@ -9,6 +9,7 @@ import {
 	type GetDocumentRequest,
 	type GetDocumentsRequest,
 	type GetDocumentTypeCountsRequest,
+	type GetSurfsenseDocsRequest,
 	getDocumentByChunkRequest,
 	getDocumentByChunkResponse,
 	getDocumentRequest,
@@ -18,6 +19,7 @@ import {
 	getDocumentTypeCountsRequest,
 	getDocumentTypeCountsResponse,
 	getSurfsenseDocsByChunkResponse,
+	getSurfsenseDocsResponse,
 	type SearchDocumentsRequest,
 	searchDocumentsRequest,
 	searchDocumentsResponse,
@@ -221,6 +223,30 @@ class DocumentsApiService {
 		);
 	};
 
+	/**
+	 * List all Surfsense documentation documents
+	 */
+	getSurfsenseDocs = async (request: GetSurfsenseDocsRequest = {}) => {
+		const queryParams = new URLSearchParams();
+
+		if (request.page !== undefined) {
+			queryParams.set("page", String(request.page));
+		}
+		if (request.page_size !== undefined) {
+			queryParams.set("page_size", String(request.page_size));
+		}
+		if (request.title) {
+			queryParams.set("title", request.title);
+		}
+
+		const queryString = queryParams.toString();
+		const url = queryString
+			? `/api/v1/surfsense-docs?${queryString}`
+			: "/api/v1/surfsense-docs";
+
+		return baseApiService.get(url, getSurfsenseDocsResponse);
+	};
+
 	/**
 	 * Update a document
 	 */