apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-04-25 00:36:31 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add SurfSense docs to documents table

Browse source
This commit is contained in:
CREDO23 2026-01-13 01:15:33 +02:00
parent 4ace7d09a0
commit 738e23b51a
9 changed files with 338 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

82
surfsense_backend/app/routes/surfsense_docs_routes.py
View file

@ -7,7 +7,7 @@ on a [citation:doc-XXX] link.
"""
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
@ -17,8 +17,10 @@ from app.db import (
User,
get_async_session,
)
from app.schemas import PaginatedResponse
from app.schemas.surfsense_docs import (
SurfsenseDocsChunkRead,
SurfsenseDocsDocumentRead,
SurfsenseDocsDocumentWithChunksRead,
)
from app.users import current_active_user
@ -87,3 +89,81 @@ async def get_surfsense_doc_by_chunk_id(
status_code=500,
detail=f"Failed to retrieve Surfsense documentation: {e!s}",
) from e
@router.get(
"/surfsense-docs",
response_model=PaginatedResponse[SurfsenseDocsDocumentRead],
)
async def list_surfsense_docs(
page: int = 0,
page_size: int = 50,
title: str | None = None,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
List all Surfsense documentation documents.
Args:
page: Zero-based page index.
page_size: Number of items per page (default: 50).
title: Optional title filter (case-insensitive substring match).
session: Database session (injected).
user: Current authenticated user (injected).
Returns:
PaginatedResponse[SurfsenseDocsDocumentRead]: Paginated list of Surfsense docs.
"""
try:
# Base query
query = select(SurfsenseDocsDocument)
count_query = select(func.count()).select_from(SurfsenseDocsDocument)
# Filter by title if provided
if title and title.strip():
query = query.filter(SurfsenseDocsDocument.title.ilike(f"%{title}%"))
count_query = count_query.filter(
SurfsenseDocsDocument.title.ilike(f"%{title}%")
)
# Get total count
total_result = await session.execute(count_query)
total = total_result.scalar() or 0
# Calculate offset
offset = page * page_size
# Get paginated results
result = await session.execute(
query.order_by(SurfsenseDocsDocument.title).offset(offset).limit(page_size)
)
docs = result.scalars().all()
# Convert to response format
items = [
SurfsenseDocsDocumentRead(
id=doc.id,
title=doc.title,
source=doc.source,
content=doc.content,
created_at=doc.created_at,
updated_at=doc.updated_at,
)
for doc in docs
]
has_more = (offset + len(items)) < total
return PaginatedResponse(
items=items,
total=total,
page=page,
page_size=page_size,
has_more=has_more,
)
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to list Surfsense documentation: {e!s}",
) from e

15
surfsense_backend/app/schemas/surfsense_docs.py
View file

@ -2,6 +2,8 @@
Schemas for Surfsense documentation.
"""
from datetime import datetime
from pydantic import BaseModel, ConfigDict
@ -14,6 +16,19 @@ class SurfsenseDocsChunkRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
class SurfsenseDocsDocumentRead(BaseModel):
"""Schema for a Surfsense docs document (without chunks)."""
id: int
title: str
source: str
content: str
created_at: datetime | None = None
updated_at: datetime | None = None
model_config = ConfigDict(from_attributes=True)
class SurfsenseDocsDocumentWithChunksRead(BaseModel):
"""Schema for a Surfsense docs document with its chunks."""

2
surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx
View file

@ -47,7 +47,7 @@ export function DocumentsFilters({
columnVisibility,
onToggleColumn,
}: {
typeCounts: Record<DocumentTypeEnum, number>;
typeCounts: Partial<Record<DocumentTypeEnum, number>>;
selectedIds: Set<number>;
onSearch: (v: string) => void;
searchValue: string;

24
surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx
View file

@ -79,17 +79,25 @@ export function DocumentsTableShell({
[documents, sortKey, sortDesc]
);
const allSelectedOnPage = sorted.length > 0 && sorted.every((d) => selectedIds.has(d.id));
const someSelectedOnPage = sorted.some((d) => selectedIds.has(d.id)) && !allSelectedOnPage;
// Filter out SURFSENSE_DOCS for selection purposes
const selectableDocs = React.useMemo(
() => sorted.filter((d) => d.document_type !== "SURFSENSE_DOCS"),
[sorted]
);
const allSelectedOnPage =
selectableDocs.length > 0 && selectableDocs.every((d) => selectedIds.has(d.id));
const someSelectedOnPage =
selectableDocs.some((d) => selectedIds.has(d.id)) && !allSelectedOnPage;
const toggleAll = (checked: boolean) => {
const next = new Set(selectedIds);
if (checked)
sorted.forEach((d) => {
selectableDocs.forEach((d) => {
next.add(d.id);
});
else
sorted.forEach((d) => {
selectableDocs.forEach((d) => {
next.delete(d.id);
});
setSelectedIds(next);
@ -230,9 +238,10 @@ export function DocumentsTableShell({
const icon = getDocumentTypeIcon(doc.document_type);
const title = doc.title;
const truncatedTitle = title.length > 30 ? `${title.slice(0, 30)}...` : title;
const isSurfsenseDoc = doc.document_type === "SURFSENSE_DOCS";
return (
<motion.tr
key={doc.id}
key={`${doc.document_type}-${doc.id}`}
initial={{ opacity: 0, y: 10 }}
animate={{
opacity: 1,
@ -249,8 +258,9 @@ export function DocumentsTableShell({
>
<TableCell className="px-4 py-3">
<Checkbox
checked={selectedIds.has(doc.id)}
onCheckedChange={(v) => toggleOne(doc.id, !!v)}
checked={selectedIds.has(doc.id) && !isSurfsenseDoc}
onCheckedChange={(v) => !isSurfsenseDoc && toggleOne(doc.id, !!v)}
disabled={isSurfsenseDoc}
aria-label="Select row"
/>
</TableCell>

69
surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/RowActions.tsx
View file

@ -28,6 +28,9 @@ import type { Document } from "./types";
// Only FILE and NOTE document types can be edited
const EDITABLE_DOCUMENT_TYPES = ["FILE", "NOTE"] as const;
// SURFSENSE_DOCS are system-managed and cannot be deleted
const NON_DELETABLE_DOCUMENT_TYPES = ["SURFSENSE_DOCS"] as const;
export function RowActions({
document,
deleteDocument,
@ -48,6 +51,10 @@ export function RowActions({
document.document_type as (typeof EDITABLE_DOCUMENT_TYPES)[number]
);
const isDeletable = !NON_DELETABLE_DOCUMENT_TYPES.includes(
document.document_type as (typeof NON_DELETABLE_DOCUMENT_TYPES)[number]
);
const handleDelete = async () => {
setIsDeleting(true);
try {
@ -120,29 +127,31 @@ export function RowActions({
</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<motion.div
whileHover={{ scale: 1.1 }}
whileTap={{ scale: 0.95 }}
transition={{ type: "spring", stiffness: 400, damping: 17 }}
>
<Button
variant="ghost"
size="icon"
className="h-8 w-8 text-muted-foreground hover:text-destructive hover:bg-destructive/10"
onClick={() => setIsDeleteOpen(true)}
disabled={isDeleting}
{isDeletable && (
<Tooltip>
<TooltipTrigger asChild>
<motion.div
whileHover={{ scale: 1.1 }}
whileTap={{ scale: 0.95 }}
transition={{ type: "spring", stiffness: 400, damping: 17 }}
>
<Trash2 className="h-4 w-4" />
<span className="sr-only">Delete</span>
</Button>
</motion.div>
</TooltipTrigger>
<TooltipContent side="top">
<p>Delete</p>
</TooltipContent>
</Tooltip>
<Button
variant="ghost"
size="icon"
className="h-8 w-8 text-muted-foreground hover:text-destructive hover:bg-destructive/10"
onClick={() => setIsDeleteOpen(true)}
disabled={isDeleting}
>
<Trash2 className="h-4 w-4" />
<span className="sr-only">Delete</span>
</Button>
</motion.div>
</TooltipTrigger>
<TooltipContent side="top">
<p>Delete</p>
</TooltipContent>
</Tooltip>
)}
</div>
{/* Mobile Actions Dropdown */}
@ -165,13 +174,15 @@ export function RowActions({
<FileText className="mr-2 h-4 w-4" />
<span>Metadata</span>
</DropdownMenuItem>
<DropdownMenuItem
onClick={() => setIsDeleteOpen(true)}
className="text-destructive focus:text-destructive"
>
<Trash2 className="mr-2 h-4 w-4" />
<span>Delete</span>
</DropdownMenuItem>
{isDeletable && (
<DropdownMenuItem
onClick={() => setIsDeleteOpen(true)}
className="text-destructive focus:text-destructive"
>
<Trash2 className="mr-2 h-4 w-4" />
<span>Delete</span>
</DropdownMenuItem>
)}
</DropdownMenuContent>
</DropdownMenu>
</div>

153
surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
View file

@ -19,7 +19,7 @@ import { DocumentsFilters } from "./components/DocumentsFilters";
import { DocumentsTableShell, type SortKey } from "./components/DocumentsTableShell";
import { PaginationControls } from "./components/PaginationControls";
import { ProcessingIndicator } from "./components/ProcessingIndicator";
import type { ColumnVisibility } from "./components/types";
import type { ColumnVisibility, Document } from "./components/types";
function useDebounced<T>(value: T, delay = 250) {
const [debounced, setDebounced] = useState(value);
@ -50,33 +50,43 @@ export default function DocumentsTable() {
const [sortKey, setSortKey] = useState<SortKey>("title");
const [sortDesc, setSortDesc] = useState(false);
const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
const { data: typeCounts } = useAtomValue(documentTypeCountsAtom);
const { data: rawTypeCounts } = useAtomValue(documentTypeCountsAtom);
const { mutateAsync: deleteDocumentMutation } = useAtomValue(deleteDocumentMutationAtom);
// Build query parameters for fetching documents
// Filter out SURFSENSE_DOCS from active types for regular documents API
const regularDocumentTypes = useMemo(
() => activeTypes.filter((t) => t !== "SURFSENSE_DOCS"),
[activeTypes]
);
// Check if only SURFSENSE_DOCS is selected (skip regular docs query)
const onlySurfsenseDocsSelected =
activeTypes.length === 1 && activeTypes[0] === "SURFSENSE_DOCS";
// Build query parameters for fetching documents (excluding SURFSENSE_DOCS type)
const queryParams = useMemo(
() => ({
search_space_id: searchSpaceId,
page: pageIndex,
page_size: pageSize,
...(activeTypes.length > 0 && { document_types: activeTypes }),
...(regularDocumentTypes.length > 0 && { document_types: regularDocumentTypes }),
}),
[searchSpaceId, pageIndex, pageSize, activeTypes]
[searchSpaceId, pageIndex, pageSize, regularDocumentTypes]
);
// Build search query parameters
// Build search query parameters (excluding SURFSENSE_DOCS type)
const searchQueryParams = useMemo(
() => ({
search_space_id: searchSpaceId,
page: pageIndex,
page_size: pageSize,
title: debouncedSearch.trim(),
...(activeTypes.length > 0 && { document_types: activeTypes }),
...(regularDocumentTypes.length > 0 && { document_types: regularDocumentTypes }),
}),
[searchSpaceId, pageIndex, pageSize, activeTypes, debouncedSearch]
[searchSpaceId, pageIndex, pageSize, regularDocumentTypes, debouncedSearch]
);
// Use query for fetching documents
// Use query for fetching documents (disabled when only SURFSENSE_DOCS is selected)
const {
data: documentsResponse,
isLoading: isDocumentsLoading,
@ -86,10 +96,10 @@ export default function DocumentsTable() {
queryKey: cacheKeys.documents.globalQueryParams(queryParams),
queryFn: () => documentsApiService.getDocuments({ queryParams }),
staleTime: 3 * 60 * 1000, // 3 minutes
enabled: !!searchSpaceId && !debouncedSearch.trim(),
enabled: !!searchSpaceId && !debouncedSearch.trim() && !onlySurfsenseDocsSelected,
});
// Use query for searching documents
// Use query for searching documents (disabled when only SURFSENSE_DOCS is selected)
const {
data: searchResponse,
isLoading: isSearchLoading,
@ -99,16 +109,109 @@ export default function DocumentsTable() {
queryKey: cacheKeys.documents.globalQueryParams(searchQueryParams),
queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }),
staleTime: 3 * 60 * 1000, // 3 minutes
enabled: !!searchSpaceId && !!debouncedSearch.trim(),
enabled: !!searchSpaceId && !!debouncedSearch.trim() && !onlySurfsenseDocsSelected,
});
// Determine if we should show SurfSense docs (when no type filter or SURFSENSE_DOCS is selected)
const showSurfsenseDocs =
activeTypes.length === 0 || activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum);
// Use query for fetching SurfSense docs
const {
data: surfsenseDocsResponse,
isLoading: isSurfsenseDocsLoading,
refetch: refetchSurfsenseDocs,
} = useQuery({
queryKey: ["surfsense-docs", debouncedSearch, pageIndex, pageSize],
queryFn: () =>
documentsApiService.getSurfsenseDocs({
page: pageIndex,
page_size: pageSize,
title: debouncedSearch.trim() || undefined,
}),
staleTime: 3 * 60 * 1000, // 3 minutes
enabled: showSurfsenseDocs,
});
// Transform SurfSense docs to match the Document type
const surfsenseDocsAsDocuments: Document[] = useMemo(() => {
if (!surfsenseDocsResponse?.items) return [];
return surfsenseDocsResponse.items.map((doc) => ({
id: doc.id,
title: doc.title,
document_type: "SURFSENSE_DOCS",
document_metadata: { source: doc.source },
content: doc.content,
created_at: doc.created_at || doc.updated_at || new Date().toISOString(),
search_space_id: -1, // Special value for global docs
}));
}, [surfsenseDocsResponse]);
// Merge type counts with SURFSENSE_DOCS count
const typeCounts = useMemo(() => {
const counts = { ...(rawTypeCounts || {}) };
if (surfsenseDocsResponse?.total) {
counts.SURFSENSE_DOCS = surfsenseDocsResponse.total;
}
return counts;
}, [rawTypeCounts, surfsenseDocsResponse?.total]);
// Extract documents and total based on search state
const documents = debouncedSearch.trim()
const regularDocuments = debouncedSearch.trim()
? searchResponse?.items || []
: documentsResponse?.items || [];
const total = debouncedSearch.trim() ? searchResponse?.total || 0 : documentsResponse?.total || 0;
const loading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading;
const error = debouncedSearch.trim() ? searchError : documentsError;
const regularTotal = debouncedSearch.trim()
? searchResponse?.total || 0
: documentsResponse?.total || 0;
// Merge regular documents with SurfSense docs
const documents = useMemo(() => {
// If filtering by type and not including SURFSENSE_DOCS, only show regular docs
if (activeTypes.length > 0 && !activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum)) {
return regularDocuments;
}
// If filtering only by SURFSENSE_DOCS, only show surfsense docs
if (activeTypes.length === 1 && activeTypes[0] === "SURFSENSE_DOCS") {
return surfsenseDocsAsDocuments;
}
// Otherwise, merge both (surfsense docs first)
return [...surfsenseDocsAsDocuments, ...regularDocuments];
}, [regularDocuments, surfsenseDocsAsDocuments, activeTypes]);
const total = useMemo(() => {
if (activeTypes.length > 0 && !activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum)) {
return regularTotal;
}
if (activeTypes.length === 1 && activeTypes[0] === "SURFSENSE_DOCS") {
return surfsenseDocsResponse?.total || 0;
}
return regularTotal + (surfsenseDocsResponse?.total || 0);
}, [regularTotal, surfsenseDocsResponse?.total, activeTypes]);
const loading = useMemo(() => {
// If only SURFSENSE_DOCS selected, only check surfsense loading
if (onlySurfsenseDocsSelected) {
return isSurfsenseDocsLoading;
}
// Otherwise check both regular docs and surfsense docs loading
const regularLoading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading;
return regularLoading || (showSurfsenseDocs && isSurfsenseDocsLoading);
}, [
onlySurfsenseDocsSelected,
isSurfsenseDocsLoading,
debouncedSearch,
isSearchLoading,
isDocumentsLoading,
showSurfsenseDocs,
]);
const error = useMemo(() => {
// If only SURFSENSE_DOCS selected, no regular docs errors
if (onlySurfsenseDocsSelected) {
return null;
}
return debouncedSearch.trim() ? searchError : documentsError;
}, [onlySurfsenseDocsSelected, debouncedSearch, searchError, documentsError]);
// Display server-filtered results directly
const displayDocs = documents || [];
@ -131,16 +234,24 @@ export default function DocumentsTable() {
if (isRefreshing) return;
setIsRefreshing(true);
try {
if (debouncedSearch.trim()) {
await refetchSearch();
} else {
await refetchDocuments();
const refetchPromises: Promise<unknown>[] = [];
// Only refetch regular documents if not in "only surfsense docs" mode
if (!onlySurfsenseDocsSelected) {
if (debouncedSearch.trim()) {
refetchPromises.push(refetchSearch());
} else {
refetchPromises.push(refetchDocuments());
}
}
if (showSurfsenseDocs) {
refetchPromises.push(refetchSurfsenseDocs());
}
await Promise.all(refetchPromises);
toast.success(t("refresh_success") || "Documents refreshed");
} finally {
setIsRefreshing(false);
}
}, [debouncedSearch, refetchSearch, refetchDocuments, t, isRefreshing]);
}, [debouncedSearch, refetchSearch, refetchDocuments, refetchSurfsenseDocs, showSurfsenseDocs, onlySurfsenseDocsSelected, t, isRefreshing]);
// Set up smart polling for active tasks - only polls when tasks are in progress
const { summary } = useLogsSummary(searchSpaceId, 24, {

3
surfsense_web/contracts/enums/connectorIcons.tsx
View file

@ -1,5 +1,6 @@
import { IconLinkPlus, IconUsersGroup } from "@tabler/icons-react";
import {
BookOpen,
File,
FileText,
Globe,
@ -86,6 +87,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas
return <FileText {...iconProps} />;
case "EXTENSION":
return <Webhook {...iconProps} />;
case "SURFSENSE_DOCS":
return <BookOpen {...iconProps} />;
case "DEEP":
return <Sparkles {...iconProps} />;
case "DEEPER":

23
surfsense_web/contracts/types/document.types.ts
View file

@ -22,6 +22,7 @@ export const documentTypeEnum = z.enum([
"LINEAR_CONNECTOR",
"NOTE",
"CIRCLEBACK",
"SURFSENSE_DOCS",
]);
export const document = z.object({
@ -183,6 +184,26 @@ export const getSurfsenseDocsByChunkRequest = z.object({
export const getSurfsenseDocsByChunkResponse = surfsenseDocsDocumentWithChunks;
/**
* List Surfsense docs
*/
export const getSurfsenseDocsRequest = z.object({
page: z.number().optional(),
page_size: z.number().optional(),
title: z.string().optional(),
});
export const getSurfsenseDocsResponse = z.object({
items: z.array(surfsenseDocsDocument.extend({
created_at: z.string().nullable().optional(),
updated_at: z.string().nullable().optional(),
})),
total: z.number(),
page: z.number(),
page_size: z.number(),
has_more: z.boolean(),
});
/**
* Update document
*/
@ -227,3 +248,5 @@ export type SurfsenseDocsDocument = z.infer<typeof surfsenseDocsDocument>;
export type SurfsenseDocsDocumentWithChunks = z.infer<typeof surfsenseDocsDocumentWithChunks>;
export type GetSurfsenseDocsByChunkRequest = z.infer<typeof getSurfsenseDocsByChunkRequest>;
export type GetSurfsenseDocsByChunkResponse = z.infer<typeof getSurfsenseDocsByChunkResponse>;
export type GetSurfsenseDocsRequest = z.infer<typeof getSurfsenseDocsRequest>;
export type GetSurfsenseDocsResponse = z.infer<typeof getSurfsenseDocsResponse>;

26
surfsense_web/lib/apis/documents-api.service.ts
View file

@ -9,6 +9,7 @@ import {
type GetDocumentRequest,
type GetDocumentsRequest,
type GetDocumentTypeCountsRequest,
type GetSurfsenseDocsRequest,
getDocumentByChunkRequest,
getDocumentByChunkResponse,
getDocumentRequest,
@ -18,6 +19,7 @@ import {
getDocumentTypeCountsRequest,
getDocumentTypeCountsResponse,
getSurfsenseDocsByChunkResponse,
getSurfsenseDocsResponse,
type SearchDocumentsRequest,
searchDocumentsRequest,
searchDocumentsResponse,
@ -221,6 +223,30 @@ class DocumentsApiService {
);
};
/**
* List all Surfsense documentation documents
*/
getSurfsenseDocs = async (request: GetSurfsenseDocsRequest = {}) => {
const queryParams = new URLSearchParams();
if (request.page !== undefined) {
queryParams.set("page", String(request.page));
}
if (request.page_size !== undefined) {
queryParams.set("page_size", String(request.page_size));
}
if (request.title) {
queryParams.set("title", request.title);
}
const queryString = queryParams.toString();
const url = queryString
? `/api/v1/surfsense-docs?${queryString}`
: "/api/v1/surfsense-docs";
return baseApiService.get(url, getSurfsenseDocsResponse);
};
/**
* Update a document
*/