diff --git a/.gitignore b/.gitignore index 342c0b258..cb6d28b4e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ ./surfsense_backend/podcasts/ .env node_modules/ -.ruff_cache/ \ No newline at end of file +.ruff_cache/ +.venv +.pnpm-store diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx index 978cdf219..a0f079e7d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx @@ -28,6 +28,7 @@ import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import type { ColumnVisibility } from "./types"; +import { DocumentTypeEnum } from "@/contracts/types/document.types"; const fadeInScale: Variants = { hidden: { opacity: 0, scale: 0.95 }, @@ -46,13 +47,13 @@ export function DocumentsFilters({ columnVisibility, onToggleColumn, }: { - typeCounts: Record; + typeCounts: Record; selectedIds: Set; onSearch: (v: string) => void; searchValue: string; onBulkDelete: () => Promise; - onToggleType: (type: string, checked: boolean) => void; - activeTypes: string[]; + onToggleType: (type: DocumentTypeEnum, checked: boolean) => void; + activeTypes: DocumentTypeEnum[]; columnVisibility: ColumnVisibility; onToggleColumn: (id: keyof ColumnVisibility, checked: boolean) => void; }) { @@ -61,7 +62,7 @@ export function DocumentsFilters({ const inputRef = useRef(null); const uniqueTypes = useMemo(() => { - return Object.keys(typeCountsRecord).sort(); + return Object.keys(typeCountsRecord).sort() as DocumentTypeEnum[]; }, [typeCountsRecord]); const typeCounts = useMemo(() => { @@ -156,7 +157,7 @@ export function DocumentsFilters({
Filters
- {uniqueTypes.map((value, i) => ( + {uniqueTypes.map((value : DocumentTypeEnum, i) => ( (value: T, delay = 250) { const [debounced, setDebounced] = useState(value); @@ -30,7 +35,7 @@ export default function DocumentsTable() { const [search, setSearch] = useState(""); const debouncedSearch = useDebounced(search, 250); - const [activeTypes, setActiveTypes] = useState([]); + const [activeTypes, setActiveTypes] = useState([]); const [columnVisibility, setColumnVisibility] = useState({ title: true, document_type: true, @@ -42,55 +47,67 @@ export default function DocumentsTable() { const [sortKey, setSortKey] = useState("title"); const [sortDesc, setSortDesc] = useState(false); const [selectedIds, setSelectedIds] = useState>(new Set()); - const [typeCounts, setTypeCounts] = useState>({}); + const {data: typeCounts} = useAtomValue(documentTypeCountsAtom) ; + const {mutateAsync : deleteDocumentMutation} = useAtomValue(deleteDocumentMutationAtom); - // Use server-side pagination, search, and filtering + // Build query parameters for fetching documents + const queryParams = useMemo( + () => ({ + search_space_id: searchSpaceId, + page: pageIndex, + page_size: pageSize, + ...(activeTypes.length > 0 && { document_types: activeTypes }), + }), + [searchSpaceId, pageIndex, pageSize, activeTypes] + ); + + // Build search query parameters + const searchQueryParams = useMemo( + () => ({ + search_space_id: searchSpaceId, + page: pageIndex, + page_size: pageSize, + title: debouncedSearch.trim(), + ...(activeTypes.length > 0 && { document_types: activeTypes }), + }), + [searchSpaceId, pageIndex, pageSize, activeTypes, debouncedSearch] + ); + + // Use query for fetching documents const { - documents, - total, - loading, - error, - fetchDocuments, - searchDocuments, - deleteDocument, - getDocumentTypeCounts, - } = useDocuments(searchSpaceId, { - page: pageIndex, - pageSize: pageSize, + data: documentsResponse, + isLoading: isDocumentsLoading, + refetch: refetchDocuments, + error : documentsError + } = useQuery({ + queryKey: cacheKeys.documents.globalQueryParams(queryParams), + queryFn: () => documentsApiService.getDocuments({ queryParams }), + staleTime: 3 * 60 * 1000, // 3 minutes + enabled: !!searchSpaceId && !debouncedSearch.trim(), }); - // Fetch document type counts on mount and when search space changes - useEffect(() => { - if (searchSpaceId && getDocumentTypeCounts) { - getDocumentTypeCounts().then(setTypeCounts); - } - }, [searchSpaceId, getDocumentTypeCounts]); + // Use query for searching documents + const { + data: searchResponse, + isLoading: isSearchLoading, + refetch: refetchSearch, + error: searchError + } = useQuery({ + queryKey: cacheKeys.documents.globalQueryParams(searchQueryParams), + queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }), + staleTime: 3 * 60 * 1000, // 3 minutes + enabled: !!searchSpaceId && !!debouncedSearch.trim(), + }); - // Refetch when pagination changes or when search/filters change - useEffect(() => { - if (searchSpaceId) { - if (debouncedSearch.trim()) { - // Use search endpoint if there's a search query - searchDocuments?.( - debouncedSearch, - pageIndex, - pageSize, - activeTypes.length > 0 ? activeTypes : undefined - ); - } else { - // Use regular fetch if no search - fetchDocuments?.(pageIndex, pageSize, activeTypes.length > 0 ? activeTypes : undefined); - } - } - }, [ - pageIndex, - pageSize, - debouncedSearch, - activeTypes, - searchSpaceId, - fetchDocuments, - searchDocuments, - ]); + // Extract documents and total based on search state + const documents = debouncedSearch.trim() + ? searchResponse?.items || [] + : documentsResponse?.items || []; + const total = debouncedSearch.trim() + ? searchResponse?.total || 0 + : documentsResponse?.total || 0; + const loading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading; + const error = debouncedSearch.trim() ? searchError : documentsError // Display server-filtered results directly const displayDocs = documents || []; @@ -98,7 +115,7 @@ export default function DocumentsTable() { const pageStart = pageIndex * pageSize; const pageEnd = Math.min(pageStart + pageSize, displayTotal); - const onToggleType = (type: string, checked: boolean) => { + const onToggleType = (type: DocumentTypeEnum, checked: boolean) => { setActiveTypes((prev) => (checked ? [...prev, type] : prev.filter((t) => t !== type))); setPageIndex(0); }; @@ -109,16 +126,25 @@ export default function DocumentsTable() { const refreshCurrentView = useCallback(async () => { if (debouncedSearch.trim()) { - await searchDocuments?.( - debouncedSearch, - pageIndex, - pageSize, - activeTypes.length > 0 ? activeTypes : undefined - ); + await refetchSearch(); } else { - await fetchDocuments?.(pageIndex, pageSize, activeTypes.length > 0 ? activeTypes : undefined); + await refetchDocuments(); } - }, [debouncedSearch, pageIndex, pageSize, activeTypes, searchDocuments, fetchDocuments]); + }, [debouncedSearch, refetchSearch, refetchDocuments]); + + // Create a delete function for single document deletion + const deleteDocument = useCallback( + async (id: number) => { + try { + await deleteDocumentMutation({ id }); + return true; + } catch (error) { + console.error("Failed to delete document:", error); + return false; + } + }, + [deleteDocumentMutation] + ); const onBulkDelete = async () => { if (selectedIds.size === 0) { @@ -126,7 +152,17 @@ export default function DocumentsTable() { return; } try { - const results = await Promise.all(Array.from(selectedIds).map((id) => deleteDocument?.(id))); + // Delete documents one by one using the mutation + const results = await Promise.all( + Array.from(selectedIds).map(async (id) => { + try { + await deleteDocumentMutation({ id }); + return true; + } catch { + return false; + } + }) + ); const okCount = results.filter((r) => r === true).length; if (okCount === selectedIds.size) toast.success(t("delete_success_count", { count: okCount })); @@ -159,7 +195,7 @@ export default function DocumentsTable() { className="w-full px-6 py-4" > deleteDocument?.(id) ?? Promise.resolve(false)} + deleteDocument={deleteDocument} sortKey={sortKey} sortDesc={sortDesc} onSortChange={(key) => { diff --git a/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx index 1a9a607fb..a9d8b9649 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx @@ -1,15 +1,15 @@ "use client"; import { type CreateMessage, type Message, useChat } from "@ai-sdk/react"; -import { useAtomValue } from "jotai"; +import { useAtom, useAtomValue } from "jotai"; import { useParams, useRouter } from "next/navigation"; import { useEffect, useMemo, useRef } from "react"; import { createChatMutationAtom, updateChatMutationAtom } from "@/atoms/chats/chat-mutation.atoms"; import { activeChatAtom } from "@/atoms/chats/chat-query.atoms"; import { activeChatIdAtom } from "@/atoms/chats/ui.atoms"; +import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; import ChatInterface from "@/components/chat/ChatInterface"; import { useChatState } from "@/hooks/use-chat"; -import { useDocumentTypes } from "@/hooks/use-document-types"; import type { Document } from "@/hooks/use-documents"; import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; @@ -46,7 +46,19 @@ export default function ResearcherPage() { }); // Fetch all available sources (document types + live search connectors) - const { documentTypes } = useDocumentTypes(Number(search_space_id)); + // Use the documentTypeCountsAtom for fetching document types + const [documentTypeCountsQuery] = useAtom(documentTypeCountsAtom); + const { data: documentTypeCountsData } = documentTypeCountsQuery; + + // Transform the response into the expected format + const documentTypes = useMemo(() => { + if (!documentTypeCountsData) return []; + return Object.entries(documentTypeCountsData).map(([type, count]) => ({ + type, + count, + })); + }, [documentTypeCountsData]); + const { connectors: searchConnectors } = useSearchSourceConnectors( false, Number(search_space_id) diff --git a/surfsense_web/atoms/documents/document-mutation.atoms.ts b/surfsense_web/atoms/documents/document-mutation.atoms.ts new file mode 100644 index 000000000..ec7bad237 --- /dev/null +++ b/surfsense_web/atoms/documents/document-mutation.atoms.ts @@ -0,0 +1,115 @@ +import { atomWithMutation } from "jotai-tanstack-query"; +import { toast } from "sonner"; +import { activeSearchSpaceIdAtom } from "@/atoms/seach-spaces/seach-space-queries.atom"; +import type { + CreateDocumentRequest, + DeleteDocumentRequest, + GetDocumentsResponse, + UpdateDocumentRequest, + UploadDocumentRequest, +} from "@/contracts/types/document.types"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queryClient } from "@/lib/query-client/client"; +import { globalDocumentsQueryParamsAtom } from "./ui.atoms"; + +export const createDocumentMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + const documentsQueryParams = get(globalDocumentsQueryParamsAtom); + + return { + mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams), + enabled: !!searchSpaceId, + mutationFn: async (request: CreateDocumentRequest) => { + return documentsApiService.createDocument(request); + }, + + onSuccess: () => { + toast.success("Document created successfully"); + queryClient.invalidateQueries({ + queryKey: cacheKeys.documents.globalQueryParams(documentsQueryParams), + }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined), + }); + }, + }; +}); + +export const uploadDocumentMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + const documentsQueryParams = get(globalDocumentsQueryParamsAtom); + + return { + mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams), + enabled: !!searchSpaceId, + mutationFn: async (request: UploadDocumentRequest) => { + return documentsApiService.uploadDocument(request); + }, + + onSuccess: () => { + toast.success("Files uploaded for processing"); + }, + }; +}); + +export const updateDocumentMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + const documentsQueryParams = get(globalDocumentsQueryParamsAtom); + + return { + mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams), + enabled: !!searchSpaceId, + mutationFn: async (request: UpdateDocumentRequest) => { + return documentsApiService.updateDocument(request); + }, + + onSuccess: (_, request: UpdateDocumentRequest) => { + toast.success("Document updated successfully"); + queryClient.invalidateQueries({ + queryKey: cacheKeys.documents.globalQueryParams(documentsQueryParams), + }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.documents.document(String(request.id)), + }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined), + }); + }, + }; +}); + +export const deleteDocumentMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + const authToken = localStorage.getItem("surfsense_bearer_token"); + const documentsQueryParams = get(globalDocumentsQueryParamsAtom); + + return { + mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams), + enabled: !!searchSpaceId && !!authToken, + mutationFn: async (request: DeleteDocumentRequest) => { + return documentsApiService.deleteDocument(request); + }, + + onSuccess: (_, request: DeleteDocumentRequest) => { + toast.success("Document deleted successfully"); + queryClient.setQueryData( + cacheKeys.documents.globalQueryParams(documentsQueryParams), + (oldData: GetDocumentsResponse | undefined) => { + if (!oldData) return oldData; + return { + ...oldData, + items: oldData.items.filter((doc) => doc.id !== request.id), + total: oldData.total - 1, + }; + } + ); + queryClient.invalidateQueries({ + queryKey: cacheKeys.documents.document(String(request.id)), + }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined), + }); + }, + }; +}); diff --git a/surfsense_web/atoms/documents/document-query.atoms.ts b/surfsense_web/atoms/documents/document-query.atoms.ts new file mode 100644 index 000000000..acdff9aaf --- /dev/null +++ b/surfsense_web/atoms/documents/document-query.atoms.ts @@ -0,0 +1,38 @@ +import { atomWithQuery } from "jotai-tanstack-query"; +import { activeSearchSpaceIdAtom } from "@/atoms/seach-spaces/seach-space-queries.atom"; +import type { SearchDocumentsRequest } from "@/contracts/types/document.types"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { globalDocumentsQueryParamsAtom } from "./ui.atoms"; + +export const documentsAtom = atomWithQuery((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + const queryParams = get(globalDocumentsQueryParamsAtom); + + return { + queryKey: cacheKeys.documents.globalQueryParams(queryParams), + enabled: !!searchSpaceId, + queryFn: async () => { + return documentsApiService.getDocuments({ + queryParams: queryParams, + }); + }, + }; +}); + +export const documentTypeCountsAtom = atomWithQuery((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined), + enabled: !!searchSpaceId, + staleTime: 10 * 60 * 1000, // 10 minutes + queryFn: async () => { + return documentsApiService.getDocumentTypeCounts({ + queryParams: { + search_space_id: searchSpaceId ?? undefined, + }, + }); + }, + }; +}); diff --git a/surfsense_web/atoms/documents/ui.atoms.ts b/surfsense_web/atoms/documents/ui.atoms.ts new file mode 100644 index 000000000..33740e9c7 --- /dev/null +++ b/surfsense_web/atoms/documents/ui.atoms.ts @@ -0,0 +1,7 @@ +import { atom } from "jotai"; +import type { GetDocumentsRequest } from "@/contracts/types/document.types"; + +export const globalDocumentsQueryParamsAtom = atom({ + page_size: 10, + page: 0, +}); diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx index 7a76c4d56..4e63048d1 100644 --- a/surfsense_web/components/chat/ChatInputGroup.tsx +++ b/surfsense_web/components/chat/ChatInputGroup.tsx @@ -3,7 +3,9 @@ import { ChatInput } from "@llamaindex/chat-ui"; import { Brain, Check, FolderOpen, Minus, Plus, PlusCircle, Zap } from "lucide-react"; import { useParams, useRouter } from "next/navigation"; -import React, { Suspense, useCallback, useState } from "react"; +import React, { Suspense, useCallback, useState, useMemo } from "react"; +import { useAtom } from "jotai"; +import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; import { DocumentsDataTable } from "@/components/chat/DocumentsDataTable"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -25,7 +27,6 @@ import { } from "@/components/ui/select"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useDocumentTypes } from "@/hooks/use-document-types"; import type { Document } from "@/hooks/use-documents"; import { useGlobalLLMConfigs, useLLMConfigs, useLLMPreferences } from "@/hooks/use-llm-configs"; import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; @@ -118,11 +119,20 @@ const ConnectorSelector = React.memo( const router = useRouter(); const [isOpen, setIsOpen] = useState(false); - // Fetch immediately (not lazy) so the button can show the correct count - const { documentTypes, isLoading, isLoaded, fetchDocumentTypes } = useDocumentTypes( - Number(search_space_id), - false - ); + // Use the documentTypeCountsAtom for fetching document types + const [documentTypeCountsQuery] = useAtom(documentTypeCountsAtom); + const { data: documentTypeCountsData, isLoading, refetch: fetchDocumentTypes } = documentTypeCountsQuery; + + // Transform the response into the expected format + const documentTypes = useMemo(() => { + if (!documentTypeCountsData) return []; + return Object.entries(documentTypeCountsData).map(([type, count]) => ({ + type, + count, + })); + }, [documentTypeCountsData]); + + const isLoaded = !!documentTypeCountsData; // Fetch live search connectors immediately (non-indexable) const { diff --git a/surfsense_web/components/chat/DocumentsDataTable.tsx b/surfsense_web/components/chat/DocumentsDataTable.tsx index 331c1b404..53cee8299 100644 --- a/surfsense_web/components/chat/DocumentsDataTable.tsx +++ b/surfsense_web/components/chat/DocumentsDataTable.tsx @@ -10,6 +10,7 @@ import { import { ArrowUpDown, Calendar, FileText, Filter, Plus, Search } from "lucide-react"; import { useRouter } from "next/navigation"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; @@ -31,7 +32,11 @@ import { TableRow, } from "@/components/ui/table"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { type Document, type DocumentType, useDocuments } from "@/hooks/use-documents"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { Document, DocumentTypeEnum } from "@/contracts/types/document.types"; +import { useAtomValue } from "jotai"; +import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; interface DocumentsDataTableProps { searchSpaceId: number; @@ -104,8 +109,8 @@ const columns: ColumnDef[] = [ cell: ({ row }) => { const type = row.getValue("document_type") as DocumentType; return ( -
- {getConnectorIcon(type)} +
+ {getConnectorIcon(String(type))}
); }, @@ -182,52 +187,59 @@ export function DocumentsDataTable({ const [sorting, setSorting] = useState([]); const [search, setSearch] = useState(""); const debouncedSearch = useDebounced(search, 300); - const [documentTypeFilter, setDocumentTypeFilter] = useState([]); + const [documentTypeFilter, setDocumentTypeFilter] = useState([]); const [pageIndex, setPageIndex] = useState(0); const [pageSize, setPageSize] = useState(10); - const [typeCounts, setTypeCounts] = useState>({}); + const {data : typeCounts } = useAtomValue(documentTypeCountsAtom); - // Use server-side pagination, search, and filtering - const { documents, total, loading, fetchDocuments, searchDocuments, getDocumentTypeCounts } = - useDocuments(searchSpaceId, { - page: pageIndex, - pageSize: pageSize, - }); + const fetchQueryParams = useMemo( + () => ({ + search_space_id: searchSpaceId, + page: pageIndex , + page_size: pageSize, + ...(documentTypeFilter.length > 0 && { document_types: documentTypeFilter }), + }), + [searchSpaceId, pageIndex, pageSize, documentTypeFilter, debouncedSearch] + ); - // Fetch document type counts on mount - useEffect(() => { - if (searchSpaceId && getDocumentTypeCounts) { - getDocumentTypeCounts().then(setTypeCounts); + const searchQueryParams = useMemo(() => { + return { + search_space_id: searchSpaceId, + page: pageIndex , + page_size: pageSize, + ...(documentTypeFilter.length > 0 && { document_types: documentTypeFilter }), + title : debouncedSearch, } - }, [searchSpaceId, getDocumentTypeCounts]); + },[debouncedSearch, searchSpaceId, pageIndex, pageSize, documentTypeFilter, debouncedSearch]) - // Refetch when pagination changes or when search/filters change - useEffect(() => { - if (searchSpaceId) { - if (debouncedSearch.trim()) { - searchDocuments?.( - debouncedSearch, - pageIndex, - pageSize, - documentTypeFilter.length > 0 ? documentTypeFilter : undefined - ); - } else { - fetchDocuments?.( - pageIndex, - pageSize, - documentTypeFilter.length > 0 ? documentTypeFilter : undefined - ); - } - } - }, [ - pageIndex, - pageSize, - debouncedSearch, - documentTypeFilter, - searchSpaceId, - fetchDocuments, - searchDocuments, - ]); + // Use query for fetching documents + const { + data: documents, + isLoading: isDocumentsLoading, + } = useQuery({ + queryKey: cacheKeys.documents.withQueryParams(fetchQueryParams), + queryFn: () => documentsApiService.getDocuments({ queryParams : fetchQueryParams }), + staleTime: 3 * 60 * 1000, // 3 minutes + enabled: !!searchSpaceId && !debouncedSearch.trim(), + }); + + // Seaching + const { + data: searchedDocuments, + isLoading: isSearchedDocumentsLoading, + } = useQuery({ + queryKey: cacheKeys.documents.withQueryParams(searchQueryParams), + queryFn: () => documentsApiService.searchDocuments({ queryParams : searchQueryParams }), + staleTime: 3 * 60 * 1000, // 3 minutes + enabled: !!searchSpaceId && !!debouncedSearch.trim(), + }); + + + + // Use query data when not searching, otherwise use hook data + const actualDocuments = debouncedSearch.trim() ? searchedDocuments?.items|| [] : documents?.items|| []; + const actualTotal = debouncedSearch.trim() ? searchedDocuments?.total || 0 : documents?.total || 0; + const actualLoading = debouncedSearch.trim() ? isSearchedDocumentsLoading : isDocumentsLoading; // Memoize initial row selection to prevent infinite loops const initialRowSelection = useMemo(() => { @@ -272,14 +284,14 @@ export function DocumentsDataTable({ // Update the selected documents map when row selection changes useEffect(() => { - if (!documents || documents.length === 0) return; + if (!actualDocuments || actualDocuments.length === 0) return; setSelectedDocumentsMap((prev) => { const newMap = new Map(prev); let hasChanges = false; // Process only current page documents - for (const doc of documents) { + for (const doc of actualDocuments) { const docId = doc.id; const isSelected = rowSelection[docId.toString()]; const wasInMap = newMap.has(docId); @@ -319,14 +331,14 @@ export function DocumentsDataTable({ }, [selectedDocumentsArray, onSelectionChange]); const table = useReactTable({ - data: documents || [], + data: actualDocuments || [], columns, getRowId: (row) => row.id.toString(), onSortingChange: setSorting, getCoreRowModel: getCoreRowModel(), onRowSelectionChange: setRowSelection, manualPagination: true, - pageCount: Math.ceil(total / pageSize), + pageCount: Math.ceil(actualTotal / pageSize), state: { sorting, rowSelection, pagination: { pageIndex, pageSize } }, }); @@ -344,7 +356,7 @@ export function DocumentsDataTable({ setRowSelection(newSelection); }, [table, rowSelection]); - const handleToggleType = useCallback((type: string, checked: boolean) => { + const handleToggleType = useCallback((type: DocumentTypeEnum, checked: boolean) => { setDocumentTypeFilter((prev) => { if (checked) { return [...prev, type]; @@ -358,7 +370,7 @@ export function DocumentsDataTable({ // Get available document types from type counts (memoized) const availableTypes = useMemo(() => { - const types = Object.keys(typeCounts); + const types = typeCounts ? Object.keys(typeCounts) as DocumentTypeEnum[] : []; return types.length > 0 ? types.sort() : []; }, [typeCounts]); @@ -408,7 +420,7 @@ export function DocumentsDataTable({ className="flex grow justify-between gap-2 font-normal text-sm cursor-pointer" > {type.replace(/_/g, " ")} - {typeCounts[type]} + {typeCounts?.[type]}
))} @@ -435,7 +447,7 @@ export function DocumentsDataTable({
- {selectedCount} selected {loading && "· Loading..."} + {selectedCount} selected {actualLoading && "· Loading..."}
@@ -453,7 +465,7 @@ export function DocumentsDataTable({ size="sm" onClick={handleSelectPage} className="text-xs sm:text-sm" - disabled={loading} + disabled={actualLoading} > Select Page @@ -490,7 +502,7 @@ export function DocumentsDataTable({ {/* Table Container */}
- {loading ? ( + {actualLoading ? (
@@ -561,31 +573,31 @@ export function DocumentsDataTable({ {/* Footer Pagination */}
- Showing {pageIndex * pageSize + 1} to {Math.min((pageIndex + 1) * pageSize, total)} of{" "} - {total} documents + Showing {pageIndex * pageSize + 1} to {Math.min((pageIndex + 1) * pageSize, actualTotal)} of{" "} + {actualTotal} documents
Page - {pageIndex + 1} - of - {Math.ceil(total / pageSize)} -
+ {pageIndex + 1} + of + {Math.ceil(actualTotal / pageSize)} +
diff --git a/surfsense_web/components/chat/SourceDetailSheet.tsx b/surfsense_web/components/chat/SourceDetailSheet.tsx index e28e35070..70b73fcf2 100644 --- a/surfsense_web/components/chat/SourceDetailSheet.tsx +++ b/surfsense_web/components/chat/SourceDetailSheet.tsx @@ -1,8 +1,9 @@ "use client"; +import { useQuery } from "@tanstack/react-query"; import { ChevronDown, ChevronUp, ExternalLink, Loader2 } from "lucide-react"; import type React from "react"; -import { type ReactNode, useEffect, useRef, useState } from "react"; +import { type ReactNode, useEffect, useLayoutEffect, useRef, useState } from "react"; import { MarkdownViewer } from "@/components/markdown-viewer"; import { Button } from "@/components/ui/button"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; @@ -15,7 +16,8 @@ import { SheetTitle, } from "@/components/ui/sheet"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useDocumentByChunk } from "@/hooks/use-document-by-chunk"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; import { cn } from "@/lib/utils"; interface SourceDetailSheetProps { @@ -46,11 +48,22 @@ export function SourceDetailSheet({ url, children, }: SourceDetailSheetProps) { - const { document, loading, error, fetchDocumentByChunk, clearDocument } = useDocumentByChunk(); const chunksContainerRef = useRef(null); const highlightedChunkRef = useRef(null); const [summaryOpen, setSummaryOpen] = useState(false); + // Add useQuery to fetch document by chunk + const { + data: document, + isLoading: isDocumentByChunkFetching, + error: documentByChunkFetchingError, + } = useQuery({ + queryKey: cacheKeys.documents.byChunk(chunkId.toString()), + queryFn: () => documentsApiService.getDocumentByChunk({ chunk_id: chunkId }), + enabled: !!chunkId && open, + staleTime: 5 * 60 * 1000, // 5 minutes + }); + // Check if this is a source type that should render directly from node const isDirectRenderSource = sourceType === "TAVILY_API" || @@ -58,17 +71,9 @@ export function SourceDetailSheet({ sourceType === "SEARXNG_API" || sourceType === "BAIDU_SEARCH_API"; - useEffect(() => { - if (open && chunkId && !isDirectRenderSource) { - fetchDocumentByChunk(chunkId); - } else if (!open && !isDirectRenderSource) { - clearDocument(); - } - }, [open, chunkId, isDirectRenderSource, fetchDocumentByChunk, clearDocument]); - useEffect(() => { // Scroll to highlighted chunk when document loads - if (document && highlightedChunkRef.current && chunksContainerRef.current) { + if (document) { setTimeout(() => { highlightedChunkRef.current?.scrollIntoView({ behavior: "smooth", @@ -76,7 +81,7 @@ export function SourceDetailSheet({ }); }, 100); } - }, [document]); + }, [document, open]); const handleUrlClick = (e: React.MouseEvent, clickUrl: string) => { e.preventDefault(); @@ -100,15 +105,17 @@ export function SourceDetailSheet({ - {!isDirectRenderSource && loading && ( + {!isDirectRenderSource && isDocumentByChunkFetching && (
)} - {!isDirectRenderSource && error && ( + {!isDirectRenderSource && documentByChunkFetchingError && (
-

{error}

+

+ {documentByChunkFetchingError.message || "Failed to load document"} +

)} diff --git a/surfsense_web/components/sources/DocumentUploadTab.tsx b/surfsense_web/components/sources/DocumentUploadTab.tsx index c51522a74..7ad82d124 100644 --- a/surfsense_web/components/sources/DocumentUploadTab.tsx +++ b/surfsense_web/components/sources/DocumentUploadTab.tsx @@ -1,5 +1,6 @@ "use client"; +import { useAtom } from "jotai"; import { CheckCircle2, FileType, Info, Loader2, Tag, Upload, X } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; import { useRouter } from "next/navigation"; @@ -7,6 +8,7 @@ import { useTranslations } from "next-intl"; import { useCallback, useState } from "react"; import { useDropzone } from "react-dropzone"; import { toast } from "sonner"; +import { uploadDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Badge } from "@/components/ui/badge"; @@ -14,7 +16,6 @@ import { Button } from "@/components/ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Progress } from "@/components/ui/progress"; import { Separator } from "@/components/ui/separator"; -import { getAuthHeaders } from "@/lib/auth-utils"; import { GridPattern } from "./GridPattern"; interface DocumentUploadTabProps { @@ -25,9 +26,12 @@ export function DocumentUploadTab({ searchSpaceId }: DocumentUploadTabProps) { const t = useTranslations("upload_documents"); const router = useRouter(); const [files, setFiles] = useState([]); - const [isUploading, setIsUploading] = useState(false); const [uploadProgress, setUploadProgress] = useState(0); + // Use the uploadDocumentMutationAtom + const [uploadDocumentMutation] = useAtom(uploadDocumentMutationAtom); + const { mutate: uploadDocuments, isPending: isUploading } = uploadDocumentMutation; + const audioFileTypes = { "audio/mpeg": [".mp3", ".mpeg", ".mpga"], "audio/mp4": [".mp4", ".m4a"], @@ -148,53 +152,40 @@ export function DocumentUploadTab({ searchSpaceId }: DocumentUploadTabProps) { }; const handleUpload = async () => { - setIsUploading(true); setUploadProgress(0); - const formData = new FormData(); - files.forEach((file) => { - formData.append("files", file); - }); - formData.append("search_space_id", searchSpaceId); + // Create a progress interval to simulate progress + const progressInterval = setInterval(() => { + setUploadProgress((prev) => { + if (prev >= 90) return prev; + return prev + Math.random() * 10; + }); + }, 200); - try { - const progressInterval = setInterval(() => { - setUploadProgress((prev) => { - if (prev >= 90) return prev; - return prev + Math.random() * 10; - }); - }, 200); - - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/fileupload`, - { - method: "POST", - headers: getAuthHeaders(), - body: formData, - } - ); - - clearInterval(progressInterval); - setUploadProgress(100); - - if (!response.ok) { - throw new Error("Upload failed"); + // Use the mutation to upload documents + uploadDocuments( + { + files, + search_space_id: Number(searchSpaceId), + }, + { + onSuccess: () => { + clearInterval(progressInterval); + setUploadProgress(100); + toast(t("upload_initiated"), { + description: t("upload_initiated_desc"), + }); + router.push(`/dashboard/${searchSpaceId}/documents`); + }, + onError: (error: any) => { + clearInterval(progressInterval); + setUploadProgress(0); + toast(t("upload_error"), { + description: `${t("upload_error_desc")}: ${error.message || "Upload failed"}`, + }); + }, } - - await response.json(); - - toast(t("upload_initiated"), { - description: t("upload_initiated_desc"), - }); - - router.push(`/dashboard/${searchSpaceId}/documents`); - } catch (error: any) { - setIsUploading(false); - setUploadProgress(0); - toast(t("upload_error"), { - description: `${t("upload_error_desc")}: ${error.message}`, - }); - } + ); }; const getTotalFileSize = () => { diff --git a/surfsense_web/components/sources/YouTubeTab.tsx b/surfsense_web/components/sources/YouTubeTab.tsx index 8301f51cc..0a51d0dca 100644 --- a/surfsense_web/components/sources/YouTubeTab.tsx +++ b/surfsense_web/components/sources/YouTubeTab.tsx @@ -7,7 +7,9 @@ import { motion } from "motion/react"; import { useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useState } from "react"; +import { useAtom } from "jotai"; import { toast } from "sonner"; +import { createDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms"; import { Button } from "@/components/ui/button"; import { @@ -19,7 +21,6 @@ import { CardTitle, } from "@/components/ui/card"; import { Label } from "@/components/ui/label"; -import { authenticatedFetch } from "@/lib/auth-utils"; const youtubeRegex = /^(https:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})$/; @@ -33,9 +34,12 @@ export function YouTubeTab({ searchSpaceId }: YouTubeTabProps) { const router = useRouter(); const [videoTags, setVideoTags] = useState([]); const [activeTagIndex, setActiveTagIndex] = useState(null); - const [isSubmitting, setIsSubmitting] = useState(false); const [error, setError] = useState(null); + // Use the createDocumentMutationAtom + const [createDocumentMutation] = useAtom(createDocumentMutationAtom); + const { mutate: createYouTubeDocument, isPending: isSubmitting } = createDocumentMutation; + const isValidYoutubeUrl = (url: string): boolean => { return youtubeRegex.test(url); }; @@ -58,47 +62,35 @@ export function YouTubeTab({ searchSpaceId }: YouTubeTabProps) { } setError(null); - setIsSubmitting(true); - try { - toast(t("processing_toast"), { - description: t("processing_toast_desc"), - }); + toast(t("processing_toast"), { + description: t("processing_toast_desc"), + }); - const videoUrls = videoTags.map((tag) => tag.text); + const videoUrls = videoTags.map((tag) => tag.text); - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - document_type: "YOUTUBE_VIDEO", - content: videoUrls, - search_space_id: parseInt(searchSpaceId), - }), - } - ); - - if (!response.ok) { - throw new Error("Failed to process YouTube videos"); + // Use the mutation to create YouTube documents + createYouTubeDocument( + { + document_type: "YOUTUBE_VIDEO", + content: videoUrls, + search_space_id: parseInt(searchSpaceId), + }, + { + onSuccess: () => { + toast(t("success_toast"), { + description: t("success_toast_desc"), + }); + router.push(`/dashboard/${searchSpaceId}/documents`); + }, + onError: (error: any) => { + setError(error.message || t("error_generic")); + toast(t("error_toast"), { + description: `${t("error_toast_desc")}: ${error.message || "Failed to process YouTube videos"}`, + }); + }, } - - await response.json(); - - toast(t("success_toast"), { - description: t("success_toast_desc"), - }); - - router.push(`/dashboard/${searchSpaceId}/documents`); - } catch (error: any) { - setError(error.message || t("error_generic")); - toast(t("error_toast"), { - description: `${t("error_toast_desc")}: ${error.message}`, - }); - } finally { - setIsSubmitting(false); - } + ); }; const handleAddTag = (text: string) => { diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts new file mode 100644 index 000000000..ae755f5f3 --- /dev/null +++ b/surfsense_web/contracts/types/document.types.ts @@ -0,0 +1,183 @@ +import { z } from "zod"; +import { paginationQueryParams } from "."; + +export const documentTypeEnum = z.enum([ + "EXTENSION", + "CRAWLED_URL", + "FILE", + "SLACK_CONNECTOR", + "NOTION_CONNECTOR", + "YOUTUBE_VIDEO", + "GITHUB_CONNECTOR", + "DISCORD_CONNECTOR", + "JIRA_CONNECTOR", + "CONFLUENCE_CONNECTOR", + "CLICKUP_CONNECTOR", + "GOOGLE_CALENDAR_CONNECTOR", + "GOOGLE_GMAIL_CONNECTOR", + "AIRTABLE_CONNECTOR", + "LUMA_CONNECTOR", + "ELASTICSEARCH_CONNECTOR", + "LINEAR_CONNECTOR", +]); + +export const document = z.object({ + id: z.number(), + title: z.string(), + document_type: documentTypeEnum, + document_metadata: z.record(z.string(), z.any()), + content: z.string(), + created_at: z.string(), + search_space_id: z.number(), +}); + +export const extensionDocumentContent = z.object({ + metadata: z.object({ + BrowsingSessionId: z.string(), + VisitedWebPageURL: z.string(), + VisitedWebPageTitle: z.string(), + VisitedWebPageDateWithTimeInISOString: z.string(), + VisitedWebPageReffererURL: z.string(), + VisitedWebPageVisitDurationInMilliseconds: z.string(), + }), + pageContent: z.string(), +}); + +export const documentWithChunks = document.extend({ + chunks: z.array( + z.object({ + id: z.number(), + content: z.string(), + created_at: z.string(), + }) + ), +}); + +/** + * Get documents + */ +export const getDocumentsRequest = z.object({ + queryParams: paginationQueryParams + .extend({ + search_space_id: z.number().or(z.string()).optional(), + document_types: z.array(documentTypeEnum).optional(), + }) + .nullish(), +}); + +export const getDocumentsResponse = z.object({ + items: z.array(document), + total: z.number(), +}); + +/** + * Get document + */ +export const getDocumentRequest = document.pick({ id: true }); + +export const getDocumentResponse = document; + +/** + * Create documents + */ +export const createDocumentRequest = document + .pick({ document_type: true, search_space_id: true }) + .extend({ + content: z.string().or(z.array(z.string())).or(z.array(extensionDocumentContent)), + }); + +export const createDocumentResponse = z.object({ + message: z.literal("Documents created successfully"), +}); + +/** + * Upload documents + */ +export const uploadDocumentRequest = z.object({ + files: z.array(z.instanceof(File)), + search_space_id: z.number(), +}); + +export const uploadDocumentResponse = z.object({ + message: z.literal("Files uploaded for processing"), +}); + +/** + * Search documents + */ +export const searchDocumentsRequest = z.object({ + queryParams: paginationQueryParams + .extend({ + search_space_id: z.number().or(z.string()).optional(), + document_types: z.array(documentTypeEnum).optional(), + title: z.string().optional(), + }) + .nullish(), +}); + +export const searchDocumentsResponse = z.object({ + items: z.array(document), + total: z.number(), +}); + +/** + * Get document type counts + */ +export const getDocumentTypeCountsRequest = z.object({ + queryParams: z + .object({ + search_space_id: z.number().or(z.string()).optional(), + }) + .nullish(), +}); + +export const getDocumentTypeCountsResponse = z.record(z.string(), z.number()); + +/** + * Get document by chunk + */ +export const getDocumentByChunkRequest = z.object({ + chunk_id: z.number(), +}); + +export const getDocumentByChunkResponse = documentWithChunks; + +/** + * Update document + */ +export const updateDocumentRequest = z.object({ + id: z.number(), + data: document.pick({ search_space_id: true, document_type: true, content: true }), +}); + +export const updateDocumentResponse = document; + +/** + * Delete document + */ +export const deleteDocumentRequest = document.pick({ id: true }); + +export const deleteDocumentResponse = z.object({ + message: z.literal("Document deleted successfully"), +}); + +export type Document = z.infer +export type GetDocumentsRequest = z.infer; +export type GetDocumentsResponse = z.infer; +export type GetDocumentRequest = z.infer; +export type GetDocumentResponse = z.infer; +export type CreateDocumentRequest = z.infer; +export type CreateDocumentResponse = z.infer; +export type UploadDocumentRequest = z.infer; +export type UploadDocumentResponse = z.infer; +export type SearchDocumentsRequest = z.infer; +export type SearchDocumentsResponse = z.infer; +export type GetDocumentTypeCountsRequest = z.infer; +export type GetDocumentTypeCountsResponse = z.infer; +export type GetDocumentByChunkRequest = z.infer; +export type GetDocumentByChunkResponse = z.infer; +export type UpdateDocumentRequest = z.infer; +export type UpdateDocumentResponse = z.infer; +export type DeleteDocumentRequest = z.infer; +export type DeleteDocumentResponse = z.infer; +export type DocumentTypeEnum = z.infer diff --git a/surfsense_web/contracts/types/index.ts b/surfsense_web/contracts/types/index.ts index d00f7903f..e6f5da376 100644 --- a/surfsense_web/contracts/types/index.ts +++ b/surfsense_web/contracts/types/index.ts @@ -3,6 +3,10 @@ import { z } from "zod"; export const paginationQueryParams = z.object({ limit: z.number().optional(), skip: z.number().optional(), + // OR + page: z.number().optional(), + page_size: z.number().optional(), + // TODO: Normalize to page/page_size or limit/skip in backend }); export type PaginationQueryParams = z.infer; diff --git a/surfsense_web/hooks/index.ts b/surfsense_web/hooks/index.ts index a244609a2..2cea293e8 100644 --- a/surfsense_web/hooks/index.ts +++ b/surfsense_web/hooks/index.ts @@ -1,4 +1,3 @@ -export * from "./use-document-by-chunk"; export * from "./use-logs"; export * from "./use-rbac"; export * from "./use-search-source-connectors"; diff --git a/surfsense_web/hooks/use-document-by-chunk.ts b/surfsense_web/hooks/use-document-by-chunk.ts deleted file mode 100644 index 630e810a2..000000000 --- a/surfsense_web/hooks/use-document-by-chunk.ts +++ /dev/null @@ -1,106 +0,0 @@ -"use client"; -import { useCallback, useState } from "react"; -import { toast } from "sonner"; -import { authenticatedFetch } from "@/lib/auth-utils"; - -export interface Chunk { - id: number; - content: string; - document_id: number; - created_at: string; -} - -export interface DocumentWithChunks { - id: number; - title: string; - document_type: DocumentType; - document_metadata: any; - content: string; - created_at: string; - search_space_id: number; - chunks: Chunk[]; -} - -export type DocumentType = - | "EXTENSION" - | "CRAWLED_URL" - | "SLACK_CONNECTOR" - | "NOTION_CONNECTOR" - | "FILE" - | "YOUTUBE_VIDEO" - | "GITHUB_CONNECTOR" - | "LINEAR_CONNECTOR" - | "DISCORD_CONNECTOR" - | "JIRA_CONNECTOR" - | "CONFLUENCE_CONNECTOR" - | "CLICKUP_CONNECTOR" - | "GOOGLE_CALENDAR_CONNECTOR" - | "GOOGLE_GMAIL_CONNECTOR" - | "LUMA_CONNECTOR" - | "ELASTICSEARCH_CONNECTOR"; - -export function useDocumentByChunk() { - const [document, setDocument] = useState(null); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); - - const fetchDocumentByChunk = useCallback(async (chunkId: number) => { - try { - setLoading(true); - setError(null); - setDocument(null); - - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/by-chunk/${chunkId}`, - { - headers: { "Content-Type": "application/json" }, - method: "GET", - } - ); - - if (!response.ok) { - const errorText = await response.text(); - let errorMessage = "Failed to fetch document"; - - try { - const errorData = JSON.parse(errorText); - errorMessage = errorData.detail || errorMessage; - } catch { - // If parsing fails, use default message - } - - if (response.status === 404) { - errorMessage = "Chunk not found or you don't have access to it"; - } - - toast.error(errorMessage); - throw new Error(errorMessage); - } - - const data: DocumentWithChunks = await response.json(); - setDocument(data); - setError(null); - return data; - } catch (err: any) { - const errorMessage = err.message || "Failed to fetch document"; - setError(errorMessage); - console.error("Error fetching document by chunk:", err); - throw err; - } finally { - setLoading(false); - } - }, []); - - const clearDocument = useCallback(() => { - setDocument(null); - setError(null); - }, []); - - return { - document, - loading, - error, - fetchDocumentByChunk, - clearDocument, - }; -} diff --git a/surfsense_web/hooks/use-document-types.ts b/surfsense_web/hooks/use-document-types.ts deleted file mode 100644 index 21c9eb6fe..000000000 --- a/surfsense_web/hooks/use-document-types.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { useCallback, useEffect, useState } from "react"; -import { authenticatedFetch } from "@/lib/auth-utils"; - -export interface DocumentTypeCount { - type: string; - count: number; -} - -/** - * Hook to fetch document type counts from the API - * @param searchSpaceId - The search space ID to filter document types - * @param lazy - If true, types won't be fetched on mount - */ -export const useDocumentTypes = (searchSpaceId?: number, lazy: boolean = false) => { - const [documentTypes, setDocumentTypes] = useState([]); - const [isLoading, setIsLoading] = useState(!lazy); - const [isLoaded, setIsLoaded] = useState(false); - const [error, setError] = useState(null); - - const fetchDocumentTypes = useCallback( - async (spaceId?: number) => { - if (isLoaded && lazy) return; - - try { - setIsLoading(true); - setError(null); - - // Build URL with optional search_space_id query parameter - const url = new URL( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/type-counts` - ); - if (spaceId !== undefined) { - url.searchParams.append("search_space_id", spaceId.toString()); - } - - const response = await authenticatedFetch(url.toString(), { - method: "GET", - headers: { "Content-Type": "application/json" }, - }); - - if (!response.ok) { - throw new Error(`Failed to fetch document types: ${response.statusText}`); - } - - const data = await response.json(); - - // Convert the object to an array of DocumentTypeCount - const typeCounts: DocumentTypeCount[] = Object.entries(data).map(([type, count]) => ({ - type, - count: count as number, - })); - - setDocumentTypes(typeCounts); - setIsLoaded(true); - - return typeCounts; - } catch (err) { - setError(err instanceof Error ? err : new Error("An unknown error occurred")); - console.error("Error fetching document types:", err); - } finally { - setIsLoading(false); - } - }, - [isLoaded, lazy] - ); - - useEffect(() => { - if (!lazy) { - fetchDocumentTypes(searchSpaceId); - } - }, [lazy, fetchDocumentTypes, searchSpaceId]); - - // Function to refresh the document types - const refreshDocumentTypes = useCallback( - async (spaceId?: number) => { - setIsLoaded(false); - await fetchDocumentTypes(spaceId !== undefined ? spaceId : searchSpaceId); - }, - [fetchDocumentTypes, searchSpaceId] - ); - - return { - documentTypes, - isLoading, - isLoaded, - error, - fetchDocumentTypes, - refreshDocumentTypes, - }; -}; diff --git a/surfsense_web/hooks/use-documents.ts b/surfsense_web/hooks/use-documents.ts deleted file mode 100644 index b5c349091..000000000 --- a/surfsense_web/hooks/use-documents.ts +++ /dev/null @@ -1,246 +0,0 @@ -"use client"; -import { useCallback, useEffect, useState } from "react"; -import { toast } from "sonner"; -import { authenticatedFetch } from "@/lib/auth-utils"; -import { normalizeListResponse } from "@/lib/pagination"; - -export interface Document { - id: number; - title: string; - document_type: DocumentType; - document_metadata: any; - content: string; - created_at: string; - search_space_id: number; -} - -export type DocumentType = - | "EXTENSION" - | "CRAWLED_URL" - | "SLACK_CONNECTOR" - | "NOTION_CONNECTOR" - | "FILE" - | "YOUTUBE_VIDEO" - | "GITHUB_CONNECTOR" - | "LINEAR_CONNECTOR" - | "DISCORD_CONNECTOR" - | "JIRA_CONNECTOR" - | "CONFLUENCE_CONNECTOR" - | "CLICKUP_CONNECTOR" - | "GOOGLE_CALENDAR_CONNECTOR" - | "GOOGLE_GMAIL_CONNECTOR" - | "AIRTABLE_CONNECTOR" - | "LUMA_CONNECTOR" - | "ELASTICSEARCH_CONNECTOR"; - -export interface UseDocumentsOptions { - page?: number; - pageSize?: number; - lazy?: boolean; - documentTypes?: string[]; -} - -export function useDocuments(searchSpaceId: number, options?: UseDocumentsOptions | boolean) { - // Support both old boolean API and new options API for backward compatibility - const opts = typeof options === "boolean" ? { lazy: options } : options || {}; - const { page, pageSize = 300, lazy = false, documentTypes } = opts; - - const [documents, setDocuments] = useState([]); - const [total, setTotal] = useState(0); - const [loading, setLoading] = useState(!lazy); // Don't show loading initially for lazy mode - const [error, setError] = useState(null); - const [isLoaded, setIsLoaded] = useState(false); // Memoization flag - - const fetchDocuments = useCallback( - async (fetchPage?: number, fetchPageSize?: number, fetchDocumentTypes?: string[]) => { - if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode - - try { - setLoading(true); - - // Build query params - const params = new URLSearchParams({ - search_space_id: searchSpaceId.toString(), - }); - - // Use passed parameters or fall back to state/options - const effectivePage = fetchPage !== undefined ? fetchPage : page; - const effectivePageSize = fetchPageSize !== undefined ? fetchPageSize : pageSize; - const effectiveDocumentTypes = - fetchDocumentTypes !== undefined ? fetchDocumentTypes : documentTypes; - - if (effectivePage !== undefined) { - params.append("page", effectivePage.toString()); - } - if (effectivePageSize !== undefined) { - params.append("page_size", effectivePageSize.toString()); - } - if (effectiveDocumentTypes && effectiveDocumentTypes.length > 0) { - params.append("document_types", effectiveDocumentTypes.join(",")); - } - - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?${params.toString()}`, - { method: "GET" } - ); - - if (!response.ok) { - toast.error("Failed to fetch documents"); - throw new Error("Failed to fetch documents"); - } - - const data = await response.json(); - const normalized = normalizeListResponse(data); - setDocuments(normalized.items); - setTotal(normalized.total); - setError(null); - setIsLoaded(true); - } catch (err: any) { - setError(err.message || "Failed to fetch documents"); - console.error("Error fetching documents:", err); - } finally { - setLoading(false); - } - }, - [searchSpaceId, page, pageSize, documentTypes, isLoaded, lazy] - ); - - useEffect(() => { - if (!lazy && searchSpaceId) { - fetchDocuments(); - } - }, [searchSpaceId, lazy, fetchDocuments]); - - // Function to refresh the documents list - const refreshDocuments = useCallback(async () => { - setIsLoaded(false); // Reset memoization flag to allow refetch - await fetchDocuments(); - }, [fetchDocuments]); - - // Function to search documents by title - const searchDocuments = useCallback( - async ( - searchQuery: string, - fetchPage?: number, - fetchPageSize?: number, - fetchDocumentTypes?: string[] - ) => { - if (!searchQuery.trim()) { - // If search is empty, fetch all documents - return fetchDocuments(fetchPage, fetchPageSize, fetchDocumentTypes); - } - - try { - setLoading(true); - - // Build query params - const params = new URLSearchParams({ - search_space_id: searchSpaceId.toString(), - title: searchQuery, - }); - - // Use passed parameters or fall back to state/options - const effectivePage = fetchPage !== undefined ? fetchPage : page; - const effectivePageSize = fetchPageSize !== undefined ? fetchPageSize : pageSize; - const effectiveDocumentTypes = - fetchDocumentTypes !== undefined ? fetchDocumentTypes : documentTypes; - - if (effectivePage !== undefined) { - params.append("page", effectivePage.toString()); - } - if (effectivePageSize !== undefined) { - params.append("page_size", effectivePageSize.toString()); - } - if (effectiveDocumentTypes && effectiveDocumentTypes.length > 0) { - params.append("document_types", effectiveDocumentTypes.join(",")); - } - - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/search?${params.toString()}`, - { method: "GET" } - ); - - if (!response.ok) { - toast.error("Failed to search documents"); - throw new Error("Failed to search documents"); - } - - const data = await response.json(); - const normalized = normalizeListResponse(data); - setDocuments(normalized.items); - setTotal(normalized.total); - setError(null); - } catch (err: any) { - setError(err.message || "Failed to search documents"); - console.error("Error searching documents:", err); - } finally { - setLoading(false); - } - }, - [searchSpaceId, page, pageSize, documentTypes, fetchDocuments] - ); - - // Function to delete a document - const deleteDocument = useCallback( - async (documentId: number) => { - try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/${documentId}`, - { method: "DELETE" } - ); - - if (!response.ok) { - toast.error("Failed to delete document"); - throw new Error("Failed to delete document"); - } - - toast.success("Document deleted successfully"); - // Update the local state after successful deletion - setDocuments(documents.filter((doc) => doc.id !== documentId)); - return true; - } catch (err: any) { - toast.error(err.message || "Failed to delete document"); - console.error("Error deleting document:", err); - return false; - } - }, - [documents] - ); - - // Function to get document type counts - const getDocumentTypeCounts = useCallback(async () => { - try { - const params = new URLSearchParams({ - search_space_id: searchSpaceId.toString(), - }); - - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/type-counts?${params.toString()}`, - { method: "GET" } - ); - - if (!response.ok) { - throw new Error("Failed to fetch document type counts"); - } - - const counts = await response.json(); - return counts as Record; - } catch (err: any) { - console.error("Error fetching document type counts:", err); - return {}; - } - }, [searchSpaceId]); - - return { - documents, - total, - loading, - error, - isLoaded, - fetchDocuments, // Manual fetch function for lazy mode - searchDocuments, // Search function - refreshDocuments, - deleteDocument, - getDocumentTypeCounts, // Get type counts function - }; -} diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts index 3013be70a..e84e43be2 100644 --- a/surfsense_web/lib/apis/base-api.service.ts +++ b/surfsense_web/lib/apis/base-api.service.ts @@ -58,7 +58,6 @@ class BaseApiService { */ const defaultOptions: RequestOptions = { headers: { - "Content-Type": "application/json", Authorization: `Bearer ${this.bearerToken || ""}`, }, method: "GET", @@ -211,8 +210,11 @@ class BaseApiService { options?: Omit ) { return this.request(url, responseSchema, { - ...options, method: "GET", + headers: { + "Content-Type": "application/json", + }, + ...options, responseType: ResponseType.JSON, }); } @@ -224,6 +226,9 @@ class BaseApiService { ) { return this.request(url, responseSchema, { method: "POST", + headers: { + "Content-Type": "application/json", + }, ...options, responseType: ResponseType.JSON, }); @@ -236,6 +241,9 @@ class BaseApiService { ) { return this.request(url, responseSchema, { method: "PUT", + headers: { + "Content-Type": "application/json", + }, ...options, responseType: ResponseType.JSON, }); @@ -248,6 +256,9 @@ class BaseApiService { ) { return this.request(url, responseSchema, { method: "DELETE", + headers: { + "Content-Type": "application/json", + }, ...options, responseType: ResponseType.JSON, }); @@ -260,6 +271,26 @@ class BaseApiService { responseType: ResponseType.BLOB, }); } + + async postFormData( + url: string, + responseSchema?: z.ZodSchema, + options?: Omit & { body: FormData } + ) { + // Remove Content-Type from options headers if present + const { "Content-Type": _, ...headersWithoutContentType } = options?.headers ?? {}; + + return this.request(url, responseSchema, { + method: "POST", + ...options, + headers: { + // Don't set Content-Type - let browser set it with multipart boundary + Authorization: `Bearer ${this.bearerToken}`, + ...headersWithoutContentType, + }, + responseType: ResponseType.JSON, + }); + } } export const baseApiService = new BaseApiService( diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts new file mode 100644 index 000000000..5a82da439 --- /dev/null +++ b/surfsense_web/lib/apis/documents-api.service.ts @@ -0,0 +1,253 @@ +import { + type CreateDocumentRequest, + createDocumentRequest, + createDocumentResponse, + type DeleteDocumentRequest, + deleteDocumentRequest, + deleteDocumentResponse, + type GetDocumentByChunkRequest, + type GetDocumentRequest, + type GetDocumentsRequest, + type GetDocumentTypeCountsRequest, + getDocumentByChunkRequest, + getDocumentByChunkResponse, + getDocumentRequest, + getDocumentResponse, + getDocumentsRequest, + getDocumentsResponse, + getDocumentTypeCountsRequest, + getDocumentTypeCountsResponse, + type SearchDocumentsRequest, + searchDocumentsRequest, + searchDocumentsResponse, + type UpdateDocumentRequest, + type UploadDocumentRequest, + updateDocumentRequest, + updateDocumentResponse, + uploadDocumentRequest, + uploadDocumentResponse, +} from "@/contracts/types/document.types"; +import { ValidationError } from "../error"; +import { baseApiService } from "./base-api.service"; + +class DocumentsApiService { + /** + * Get a list of documents with optional filtering and pagination + */ + getDocuments = async (request: GetDocumentsRequest) => { + const parsedRequest = getDocumentsRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + // Transform query params to be string values + const transformedQueryParams = parsedRequest.data.queryParams + ? Object.fromEntries( + Object.entries(parsedRequest.data.queryParams).map(([k, v]) => { + // Handle array values (document_type) + if (Array.isArray(v)) { + return [k, v.join(",")]; + } + return [k, String(v)]; + }) + ) + : undefined; + + const queryParams = transformedQueryParams + ? new URLSearchParams(transformedQueryParams).toString() + : ""; + + return baseApiService.get(`/api/v1/documents?${queryParams}`, getDocumentsResponse); + }; + + /** + * Get a single document by ID + */ + getDocument = async (request: GetDocumentRequest) => { + const parsedRequest = getDocumentRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + return baseApiService.get(`/api/v1/documents/${request.id}`, getDocumentResponse); + }; + + /** + * Create documents (extension, crawled URL, or YouTube video) + */ + createDocument = async (request: CreateDocumentRequest) => { + const parsedRequest = createDocumentRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + return baseApiService.post(`/api/v1/documents`, createDocumentResponse, { + body: parsedRequest.data, + }); + }; + + /** + * Upload document files + */ + uploadDocument = async (request: UploadDocumentRequest) => { + const parsedRequest = uploadDocumentRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + // Create FormData for file upload + const formData = new FormData(); + parsedRequest.data.files.forEach((file) => { + formData.append("files", file); + }); + formData.append("search_space_id", String(parsedRequest.data.search_space_id)); + + return baseApiService.postFormData(`/api/v1/documents/fileupload`, uploadDocumentResponse, { + body: formData, + }); + }; + + /** + * Search documents by title + */ + searchDocuments = async (request: SearchDocumentsRequest) => { + const parsedRequest = searchDocumentsRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + // Transform query params to be string values + const transformedQueryParams = parsedRequest.data.queryParams + ? Object.fromEntries( + Object.entries(parsedRequest.data.queryParams).map(([k, v]) => { + // Handle array values (document_type) + if (Array.isArray(v)) { + return [k, v.join(",")]; + } + return [k, String(v)]; + }) + ) + : undefined; + + const queryParams = transformedQueryParams + ? new URLSearchParams(transformedQueryParams).toString() + : ""; + + return baseApiService.get(`/api/v1/documents/search?${queryParams}`, searchDocumentsResponse); + }; + + /** + * Get document type counts + */ + getDocumentTypeCounts = async (request: GetDocumentTypeCountsRequest) => { + // Validate the request + const parsedRequest = getDocumentTypeCountsRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + // Format a user friendly error message + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + // Transform query params to be string values + const transformedQueryParams = parsedRequest.data.queryParams + ? Object.fromEntries( + Object.entries(parsedRequest.data.queryParams).map(([k, v]) => [k, String(v)]) + ) + : undefined; + + const queryParams = transformedQueryParams + ? new URLSearchParams(transformedQueryParams).toString() + : ""; + + return baseApiService.get( + `/api/v1/documents/type-counts?${queryParams}`, + getDocumentTypeCountsResponse + ); + }; + + /** + * Get document by chunk ID (includes all chunks) + */ + getDocumentByChunk = async (request: GetDocumentByChunkRequest) => { + // Validate the request + const parsedRequest = getDocumentByChunkRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + // Format a user friendly error message + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + return baseApiService.get( + `/api/v1/documents/by-chunk/${request.chunk_id}`, + getDocumentByChunkResponse + ); + }; + + /** + * Update a document + */ + updateDocument = async (request: UpdateDocumentRequest) => { + // Validate the request + const parsedRequest = updateDocumentRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + // Format a user friendly error message + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + const { id, data } = parsedRequest.data; + + return baseApiService.put(`/api/v1/documents/${id}`, updateDocumentResponse, { + body: data, + }); + }; + + /** + * Delete a document + */ + deleteDocument = async (request: DeleteDocumentRequest) => { + // Validate the request + const parsedRequest = deleteDocumentRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + // Format a user friendly error message + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + return baseApiService.delete(`/api/v1/documents/${request.id}`, deleteDocumentResponse); + }; +} + +export const documentsApiService = new DocumentsApiService(); diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index 12b73bc35..72fac6dc5 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -1,4 +1,5 @@ import type { GetChatsRequest } from "@/contracts/types/chat.types"; +import type { GetDocumentsRequest } from "@/contracts/types/document.types"; import type { GetPodcastsRequest } from "@/contracts/types/podcast.types"; export const cacheKeys = { @@ -11,6 +12,14 @@ export const cacheKeys = { globalQueryParams: (queries: GetPodcastsRequest["queryParams"]) => ["podcasts", ...(queries ? Object.values(queries) : [])] as const, }, + documents: { + globalQueryParams: (queries: GetDocumentsRequest["queryParams"]) => + ["documents", ...(queries ? Object.values(queries) : [])] as const, + withQueryParams :(queries: GetDocumentsRequest["queryParams"]) => ["documents-with-queries", ...(queries ? Object.values(queries) : [])] as const, + document: (documentId: string) => ["document", documentId] as const, + typeCounts: (searchSpaceId?: string) => ["documents", "type-counts", searchSpaceId] as const, + byChunk: (chunkId: string) => ["documents", "by-chunk", chunkId] as const, + }, auth: { user: ["auth", "user"] as const, },