mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-25 19:15:18 +02:00
Merge pull request #529 from CREDO23/feat/add-jotai-tanstack
[Feat] Documents | Add Jotai & Tanstack
This commit is contained in:
Rohan Verma
GitHub
commit
194292a641
21 changed files with 948 additions and 688 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -2,4 +2,6 @@
|
|||
./surfsense_backend/podcasts/
|
||||
.env
|
||||
node_modules/
|
||||
.ruff_cache/
|
||||
.ruff_cache/
|
||||
.venv
|
||||
.pnpm-store
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import { Input } from "@/components/ui/input";
|
|||
import { Label } from "@/components/ui/label";
|
||||
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
|
||||
import type { ColumnVisibility } from "./types";
|
||||
import { DocumentTypeEnum } from "@/contracts/types/document.types";
|
||||
|
||||
const fadeInScale: Variants = {
|
||||
hidden: { opacity: 0, scale: 0.95 },
|
||||
|
|
@ -46,13 +47,13 @@ export function DocumentsFilters({
|
|||
columnVisibility,
|
||||
onToggleColumn,
|
||||
}: {
|
||||
typeCounts: Record<string, number>;
|
||||
typeCounts: Record<DocumentTypeEnum, number>;
|
||||
selectedIds: Set<number>;
|
||||
onSearch: (v: string) => void;
|
||||
searchValue: string;
|
||||
onBulkDelete: () => Promise<void>;
|
||||
onToggleType: (type: string, checked: boolean) => void;
|
||||
activeTypes: string[];
|
||||
onToggleType: (type: DocumentTypeEnum, checked: boolean) => void;
|
||||
activeTypes: DocumentTypeEnum[];
|
||||
columnVisibility: ColumnVisibility;
|
||||
onToggleColumn: (id: keyof ColumnVisibility, checked: boolean) => void;
|
||||
}) {
|
||||
|
|
@ -61,7 +62,7 @@ export function DocumentsFilters({
|
|||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const uniqueTypes = useMemo(() => {
|
||||
return Object.keys(typeCountsRecord).sort();
|
||||
return Object.keys(typeCountsRecord).sort() as DocumentTypeEnum[];
|
||||
}, [typeCountsRecord]);
|
||||
|
||||
const typeCounts = useMemo(() => {
|
||||
|
|
@ -156,7 +157,7 @@ export function DocumentsFilters({
|
|||
<div className="text-xs font-medium text-muted-foreground">Filters</div>
|
||||
<div className="space-y-3">
|
||||
<AnimatePresence>
|
||||
{uniqueTypes.map((value, i) => (
|
||||
{uniqueTypes.map((value : DocumentTypeEnum, i) => (
|
||||
<motion.div
|
||||
key={value}
|
||||
className="flex items-center gap-2"
|
||||
|
|
|
|||
|
|
@ -5,13 +5,18 @@ import { useParams } from "next/navigation";
|
|||
import { useTranslations } from "next-intl";
|
||||
import { useCallback, useEffect, useId, useMemo, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
|
||||
import { useDocuments } from "@/hooks/use-documents";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { useAtomValue } from "jotai";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
|
||||
import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
|
||||
|
||||
import { DocumentsFilters } from "./components/DocumentsFilters";
|
||||
import { DocumentsTableShell, type SortKey } from "./components/DocumentsTableShell";
|
||||
import { PaginationControls } from "./components/PaginationControls";
|
||||
import type { ColumnVisibility, Document } from "./components/types";
|
||||
import type { ColumnVisibility } from "./components/types";
|
||||
import { DocumentTypeEnum } from "@/contracts/types/document.types";
|
||||
|
||||
function useDebounced<T>(value: T, delay = 250) {
|
||||
const [debounced, setDebounced] = useState(value);
|
||||
|
|
@ -30,7 +35,7 @@ export default function DocumentsTable() {
|
|||
|
||||
const [search, setSearch] = useState("");
|
||||
const debouncedSearch = useDebounced(search, 250);
|
||||
const [activeTypes, setActiveTypes] = useState<string[]>([]);
|
||||
const [activeTypes, setActiveTypes] = useState<DocumentTypeEnum[]>([]);
|
||||
const [columnVisibility, setColumnVisibility] = useState<ColumnVisibility>({
|
||||
title: true,
|
||||
document_type: true,
|
||||
|
|
@ -42,55 +47,67 @@ export default function DocumentsTable() {
|
|||
const [sortKey, setSortKey] = useState<SortKey>("title");
|
||||
const [sortDesc, setSortDesc] = useState(false);
|
||||
const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
|
||||
const [typeCounts, setTypeCounts] = useState<Record<string, number>>({});
|
||||
const {data: typeCounts} = useAtomValue(documentTypeCountsAtom) ;
|
||||
const {mutateAsync : deleteDocumentMutation} = useAtomValue(deleteDocumentMutationAtom);
|
||||
|
||||
// Use server-side pagination, search, and filtering
|
||||
// Build query parameters for fetching documents
|
||||
const queryParams = useMemo(
|
||||
() => ({
|
||||
search_space_id: searchSpaceId,
|
||||
page: pageIndex,
|
||||
page_size: pageSize,
|
||||
...(activeTypes.length > 0 && { document_types: activeTypes }),
|
||||
}),
|
||||
[searchSpaceId, pageIndex, pageSize, activeTypes]
|
||||
);
|
||||
|
||||
// Build search query parameters
|
||||
const searchQueryParams = useMemo(
|
||||
() => ({
|
||||
search_space_id: searchSpaceId,
|
||||
page: pageIndex,
|
||||
page_size: pageSize,
|
||||
title: debouncedSearch.trim(),
|
||||
...(activeTypes.length > 0 && { document_types: activeTypes }),
|
||||
}),
|
||||
[searchSpaceId, pageIndex, pageSize, activeTypes, debouncedSearch]
|
||||
);
|
||||
|
||||
// Use query for fetching documents
|
||||
const {
|
||||
documents,
|
||||
total,
|
||||
loading,
|
||||
error,
|
||||
fetchDocuments,
|
||||
searchDocuments,
|
||||
deleteDocument,
|
||||
getDocumentTypeCounts,
|
||||
} = useDocuments(searchSpaceId, {
|
||||
page: pageIndex,
|
||||
pageSize: pageSize,
|
||||
data: documentsResponse,
|
||||
isLoading: isDocumentsLoading,
|
||||
refetch: refetchDocuments,
|
||||
error : documentsError
|
||||
} = useQuery({
|
||||
queryKey: cacheKeys.documents.globalQueryParams(queryParams),
|
||||
queryFn: () => documentsApiService.getDocuments({ queryParams }),
|
||||
staleTime: 3 * 60 * 1000, // 3 minutes
|
||||
enabled: !!searchSpaceId && !debouncedSearch.trim(),
|
||||
});
|
||||
|
||||
// Fetch document type counts on mount and when search space changes
|
||||
useEffect(() => {
|
||||
if (searchSpaceId && getDocumentTypeCounts) {
|
||||
getDocumentTypeCounts().then(setTypeCounts);
|
||||
}
|
||||
}, [searchSpaceId, getDocumentTypeCounts]);
|
||||
// Use query for searching documents
|
||||
const {
|
||||
data: searchResponse,
|
||||
isLoading: isSearchLoading,
|
||||
refetch: refetchSearch,
|
||||
error: searchError
|
||||
} = useQuery({
|
||||
queryKey: cacheKeys.documents.globalQueryParams(searchQueryParams),
|
||||
queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }),
|
||||
staleTime: 3 * 60 * 1000, // 3 minutes
|
||||
enabled: !!searchSpaceId && !!debouncedSearch.trim(),
|
||||
});
|
||||
|
||||
// Refetch when pagination changes or when search/filters change
|
||||
useEffect(() => {
|
||||
if (searchSpaceId) {
|
||||
if (debouncedSearch.trim()) {
|
||||
// Use search endpoint if there's a search query
|
||||
searchDocuments?.(
|
||||
debouncedSearch,
|
||||
pageIndex,
|
||||
pageSize,
|
||||
activeTypes.length > 0 ? activeTypes : undefined
|
||||
);
|
||||
} else {
|
||||
// Use regular fetch if no search
|
||||
fetchDocuments?.(pageIndex, pageSize, activeTypes.length > 0 ? activeTypes : undefined);
|
||||
}
|
||||
}
|
||||
}, [
|
||||
pageIndex,
|
||||
pageSize,
|
||||
debouncedSearch,
|
||||
activeTypes,
|
||||
searchSpaceId,
|
||||
fetchDocuments,
|
||||
searchDocuments,
|
||||
]);
|
||||
// Extract documents and total based on search state
|
||||
const documents = debouncedSearch.trim()
|
||||
? searchResponse?.items || []
|
||||
: documentsResponse?.items || [];
|
||||
const total = debouncedSearch.trim()
|
||||
? searchResponse?.total || 0
|
||||
: documentsResponse?.total || 0;
|
||||
const loading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading;
|
||||
const error = debouncedSearch.trim() ? searchError : documentsError
|
||||
|
||||
// Display server-filtered results directly
|
||||
const displayDocs = documents || [];
|
||||
|
|
@ -98,7 +115,7 @@ export default function DocumentsTable() {
|
|||
const pageStart = pageIndex * pageSize;
|
||||
const pageEnd = Math.min(pageStart + pageSize, displayTotal);
|
||||
|
||||
const onToggleType = (type: string, checked: boolean) => {
|
||||
const onToggleType = (type: DocumentTypeEnum, checked: boolean) => {
|
||||
setActiveTypes((prev) => (checked ? [...prev, type] : prev.filter((t) => t !== type)));
|
||||
setPageIndex(0);
|
||||
};
|
||||
|
|
@ -109,16 +126,25 @@ export default function DocumentsTable() {
|
|||
|
||||
const refreshCurrentView = useCallback(async () => {
|
||||
if (debouncedSearch.trim()) {
|
||||
await searchDocuments?.(
|
||||
debouncedSearch,
|
||||
pageIndex,
|
||||
pageSize,
|
||||
activeTypes.length > 0 ? activeTypes : undefined
|
||||
);
|
||||
await refetchSearch();
|
||||
} else {
|
||||
await fetchDocuments?.(pageIndex, pageSize, activeTypes.length > 0 ? activeTypes : undefined);
|
||||
await refetchDocuments();
|
||||
}
|
||||
}, [debouncedSearch, pageIndex, pageSize, activeTypes, searchDocuments, fetchDocuments]);
|
||||
}, [debouncedSearch, refetchSearch, refetchDocuments]);
|
||||
|
||||
// Create a delete function for single document deletion
|
||||
const deleteDocument = useCallback(
|
||||
async (id: number) => {
|
||||
try {
|
||||
await deleteDocumentMutation({ id });
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error("Failed to delete document:", error);
|
||||
return false;
|
||||
}
|
||||
},
|
||||
[deleteDocumentMutation]
|
||||
);
|
||||
|
||||
const onBulkDelete = async () => {
|
||||
if (selectedIds.size === 0) {
|
||||
|
|
@ -126,7 +152,17 @@ export default function DocumentsTable() {
|
|||
return;
|
||||
}
|
||||
try {
|
||||
const results = await Promise.all(Array.from(selectedIds).map((id) => deleteDocument?.(id)));
|
||||
// Delete documents one by one using the mutation
|
||||
const results = await Promise.all(
|
||||
Array.from(selectedIds).map(async (id) => {
|
||||
try {
|
||||
await deleteDocumentMutation({ id });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
);
|
||||
const okCount = results.filter((r) => r === true).length;
|
||||
if (okCount === selectedIds.size)
|
||||
toast.success(t("delete_success_count", { count: okCount }));
|
||||
|
|
@ -159,7 +195,7 @@ export default function DocumentsTable() {
|
|||
className="w-full px-6 py-4"
|
||||
>
|
||||
<DocumentsFilters
|
||||
typeCounts={typeCounts}
|
||||
typeCounts={typeCounts ?? {}}
|
||||
selectedIds={selectedIds}
|
||||
onSearch={setSearch}
|
||||
searchValue={search}
|
||||
|
|
@ -178,7 +214,7 @@ export default function DocumentsTable() {
|
|||
selectedIds={selectedIds}
|
||||
setSelectedIds={setSelectedIds}
|
||||
columnVisibility={columnVisibility}
|
||||
deleteDocument={(id) => deleteDocument?.(id) ?? Promise.resolve(false)}
|
||||
deleteDocument={deleteDocument}
|
||||
sortKey={sortKey}
|
||||
sortDesc={sortDesc}
|
||||
onSortChange={(key) => {
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
"use client";
|
||||
|
||||
import { type CreateMessage, type Message, useChat } from "@ai-sdk/react";
|
||||
import { useAtomValue } from "jotai";
|
||||
import { useAtom, useAtomValue } from "jotai";
|
||||
import { useParams, useRouter } from "next/navigation";
|
||||
import { useEffect, useMemo, useRef } from "react";
|
||||
import { createChatMutationAtom, updateChatMutationAtom } from "@/atoms/chats/chat-mutation.atoms";
|
||||
import { activeChatAtom } from "@/atoms/chats/chat-query.atoms";
|
||||
import { activeChatIdAtom } from "@/atoms/chats/ui.atoms";
|
||||
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
|
||||
import ChatInterface from "@/components/chat/ChatInterface";
|
||||
import { useChatState } from "@/hooks/use-chat";
|
||||
import { useDocumentTypes } from "@/hooks/use-document-types";
|
||||
import type { Document } from "@/hooks/use-documents";
|
||||
import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
|
||||
|
||||
|
|
@ -46,7 +46,19 @@ export default function ResearcherPage() {
|
|||
});
|
||||
|
||||
// Fetch all available sources (document types + live search connectors)
|
||||
const { documentTypes } = useDocumentTypes(Number(search_space_id));
|
||||
// Use the documentTypeCountsAtom for fetching document types
|
||||
const [documentTypeCountsQuery] = useAtom(documentTypeCountsAtom);
|
||||
const { data: documentTypeCountsData } = documentTypeCountsQuery;
|
||||
|
||||
// Transform the response into the expected format
|
||||
const documentTypes = useMemo(() => {
|
||||
if (!documentTypeCountsData) return [];
|
||||
return Object.entries(documentTypeCountsData).map(([type, count]) => ({
|
||||
type,
|
||||
count,
|
||||
}));
|
||||
}, [documentTypeCountsData]);
|
||||
|
||||
const { connectors: searchConnectors } = useSearchSourceConnectors(
|
||||
false,
|
||||
Number(search_space_id)
|
||||
|
|
|
|||
115
surfsense_web/atoms/documents/document-mutation.atoms.ts
Normal file
115
surfsense_web/atoms/documents/document-mutation.atoms.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
import { atomWithMutation } from "jotai-tanstack-query";
|
||||
import { toast } from "sonner";
|
||||
import { activeSearchSpaceIdAtom } from "@/atoms/seach-spaces/seach-space-queries.atom";
|
||||
import type {
|
||||
CreateDocumentRequest,
|
||||
DeleteDocumentRequest,
|
||||
GetDocumentsResponse,
|
||||
UpdateDocumentRequest,
|
||||
UploadDocumentRequest,
|
||||
} from "@/contracts/types/document.types";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
import { queryClient } from "@/lib/query-client/client";
|
||||
import { globalDocumentsQueryParamsAtom } from "./ui.atoms";
|
||||
|
||||
export const createDocumentMutationAtom = atomWithMutation((get) => {
|
||||
const searchSpaceId = get(activeSearchSpaceIdAtom);
|
||||
const documentsQueryParams = get(globalDocumentsQueryParamsAtom);
|
||||
|
||||
return {
|
||||
mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
|
||||
enabled: !!searchSpaceId,
|
||||
mutationFn: async (request: CreateDocumentRequest) => {
|
||||
return documentsApiService.createDocument(request);
|
||||
},
|
||||
|
||||
onSuccess: () => {
|
||||
toast.success("Document created successfully");
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
|
||||
});
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined),
|
||||
});
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
export const uploadDocumentMutationAtom = atomWithMutation((get) => {
|
||||
const searchSpaceId = get(activeSearchSpaceIdAtom);
|
||||
const documentsQueryParams = get(globalDocumentsQueryParamsAtom);
|
||||
|
||||
return {
|
||||
mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
|
||||
enabled: !!searchSpaceId,
|
||||
mutationFn: async (request: UploadDocumentRequest) => {
|
||||
return documentsApiService.uploadDocument(request);
|
||||
},
|
||||
|
||||
onSuccess: () => {
|
||||
toast.success("Files uploaded for processing");
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
export const updateDocumentMutationAtom = atomWithMutation((get) => {
|
||||
const searchSpaceId = get(activeSearchSpaceIdAtom);
|
||||
const documentsQueryParams = get(globalDocumentsQueryParamsAtom);
|
||||
|
||||
return {
|
||||
mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
|
||||
enabled: !!searchSpaceId,
|
||||
mutationFn: async (request: UpdateDocumentRequest) => {
|
||||
return documentsApiService.updateDocument(request);
|
||||
},
|
||||
|
||||
onSuccess: (_, request: UpdateDocumentRequest) => {
|
||||
toast.success("Document updated successfully");
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
|
||||
});
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.documents.document(String(request.id)),
|
||||
});
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined),
|
||||
});
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
export const deleteDocumentMutationAtom = atomWithMutation((get) => {
|
||||
const searchSpaceId = get(activeSearchSpaceIdAtom);
|
||||
const authToken = localStorage.getItem("surfsense_bearer_token");
|
||||
const documentsQueryParams = get(globalDocumentsQueryParamsAtom);
|
||||
|
||||
return {
|
||||
mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
|
||||
enabled: !!searchSpaceId && !!authToken,
|
||||
mutationFn: async (request: DeleteDocumentRequest) => {
|
||||
return documentsApiService.deleteDocument(request);
|
||||
},
|
||||
|
||||
onSuccess: (_, request: DeleteDocumentRequest) => {
|
||||
toast.success("Document deleted successfully");
|
||||
queryClient.setQueryData(
|
||||
cacheKeys.documents.globalQueryParams(documentsQueryParams),
|
||||
(oldData: GetDocumentsResponse | undefined) => {
|
||||
if (!oldData) return oldData;
|
||||
return {
|
||||
...oldData,
|
||||
items: oldData.items.filter((doc) => doc.id !== request.id),
|
||||
total: oldData.total - 1,
|
||||
};
|
||||
}
|
||||
);
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.documents.document(String(request.id)),
|
||||
});
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined),
|
||||
});
|
||||
},
|
||||
};
|
||||
});
|
||||
38
surfsense_web/atoms/documents/document-query.atoms.ts
Normal file
38
surfsense_web/atoms/documents/document-query.atoms.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import { atomWithQuery } from "jotai-tanstack-query";
|
||||
import { activeSearchSpaceIdAtom } from "@/atoms/seach-spaces/seach-space-queries.atom";
|
||||
import type { SearchDocumentsRequest } from "@/contracts/types/document.types";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
import { globalDocumentsQueryParamsAtom } from "./ui.atoms";
|
||||
|
||||
export const documentsAtom = atomWithQuery((get) => {
|
||||
const searchSpaceId = get(activeSearchSpaceIdAtom);
|
||||
const queryParams = get(globalDocumentsQueryParamsAtom);
|
||||
|
||||
return {
|
||||
queryKey: cacheKeys.documents.globalQueryParams(queryParams),
|
||||
enabled: !!searchSpaceId,
|
||||
queryFn: async () => {
|
||||
return documentsApiService.getDocuments({
|
||||
queryParams: queryParams,
|
||||
});
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
export const documentTypeCountsAtom = atomWithQuery((get) => {
|
||||
const searchSpaceId = get(activeSearchSpaceIdAtom);
|
||||
|
||||
return {
|
||||
queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined),
|
||||
enabled: !!searchSpaceId,
|
||||
staleTime: 10 * 60 * 1000, // 10 minutes
|
||||
queryFn: async () => {
|
||||
return documentsApiService.getDocumentTypeCounts({
|
||||
queryParams: {
|
||||
search_space_id: searchSpaceId ?? undefined,
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
});
|
||||
7
surfsense_web/atoms/documents/ui.atoms.ts
Normal file
7
surfsense_web/atoms/documents/ui.atoms.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import { atom } from "jotai";
|
||||
import type { GetDocumentsRequest } from "@/contracts/types/document.types";
|
||||
|
||||
export const globalDocumentsQueryParamsAtom = atom<GetDocumentsRequest["queryParams"]>({
|
||||
page_size: 10,
|
||||
page: 0,
|
||||
});
|
||||
|
|
@ -3,7 +3,9 @@
|
|||
import { ChatInput } from "@llamaindex/chat-ui";
|
||||
import { Brain, Check, FolderOpen, Minus, Plus, PlusCircle, Zap } from "lucide-react";
|
||||
import { useParams, useRouter } from "next/navigation";
|
||||
import React, { Suspense, useCallback, useState } from "react";
|
||||
import React, { Suspense, useCallback, useState, useMemo } from "react";
|
||||
import { useAtom } from "jotai";
|
||||
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
|
||||
import { DocumentsDataTable } from "@/components/chat/DocumentsDataTable";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
|
@ -25,7 +27,6 @@ import {
|
|||
} from "@/components/ui/select";
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||
import { useDocumentTypes } from "@/hooks/use-document-types";
|
||||
import type { Document } from "@/hooks/use-documents";
|
||||
import { useGlobalLLMConfigs, useLLMConfigs, useLLMPreferences } from "@/hooks/use-llm-configs";
|
||||
import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
|
||||
|
|
@ -118,11 +119,20 @@ const ConnectorSelector = React.memo(
|
|||
const router = useRouter();
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
|
||||
// Fetch immediately (not lazy) so the button can show the correct count
|
||||
const { documentTypes, isLoading, isLoaded, fetchDocumentTypes } = useDocumentTypes(
|
||||
Number(search_space_id),
|
||||
false
|
||||
);
|
||||
// Use the documentTypeCountsAtom for fetching document types
|
||||
const [documentTypeCountsQuery] = useAtom(documentTypeCountsAtom);
|
||||
const { data: documentTypeCountsData, isLoading, refetch: fetchDocumentTypes } = documentTypeCountsQuery;
|
||||
|
||||
// Transform the response into the expected format
|
||||
const documentTypes = useMemo(() => {
|
||||
if (!documentTypeCountsData) return [];
|
||||
return Object.entries(documentTypeCountsData).map(([type, count]) => ({
|
||||
type,
|
||||
count,
|
||||
}));
|
||||
}, [documentTypeCountsData]);
|
||||
|
||||
const isLoaded = !!documentTypeCountsData;
|
||||
|
||||
// Fetch live search connectors immediately (non-indexable)
|
||||
const {
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
import { ArrowUpDown, Calendar, FileText, Filter, Plus, Search } from "lucide-react";
|
||||
import { useRouter } from "next/navigation";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Input } from "@/components/ui/input";
|
||||
|
|
@ -31,7 +32,11 @@ import {
|
|||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||
import { type Document, type DocumentType, useDocuments } from "@/hooks/use-documents";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
import { Document, DocumentTypeEnum } from "@/contracts/types/document.types";
|
||||
import { useAtomValue } from "jotai";
|
||||
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
|
||||
|
||||
interface DocumentsDataTableProps {
|
||||
searchSpaceId: number;
|
||||
|
|
@ -104,8 +109,8 @@ const columns: ColumnDef<Document>[] = [
|
|||
cell: ({ row }) => {
|
||||
const type = row.getValue("document_type") as DocumentType;
|
||||
return (
|
||||
<div className="flex items-center gap-2" title={type}>
|
||||
<span className="text-primary">{getConnectorIcon(type)}</span>
|
||||
<div className="flex items-center gap-2" title={String(type)}>
|
||||
<span className="text-primary">{getConnectorIcon(String(type))}</span>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
|
|
@ -182,52 +187,59 @@ export function DocumentsDataTable({
|
|||
const [sorting, setSorting] = useState<SortingState>([]);
|
||||
const [search, setSearch] = useState("");
|
||||
const debouncedSearch = useDebounced(search, 300);
|
||||
const [documentTypeFilter, setDocumentTypeFilter] = useState<string[]>([]);
|
||||
const [documentTypeFilter, setDocumentTypeFilter] = useState<DocumentTypeEnum[]>([]);
|
||||
const [pageIndex, setPageIndex] = useState(0);
|
||||
const [pageSize, setPageSize] = useState(10);
|
||||
const [typeCounts, setTypeCounts] = useState<Record<string, number>>({});
|
||||
const {data : typeCounts } = useAtomValue(documentTypeCountsAtom);
|
||||
|
||||
// Use server-side pagination, search, and filtering
|
||||
const { documents, total, loading, fetchDocuments, searchDocuments, getDocumentTypeCounts } =
|
||||
useDocuments(searchSpaceId, {
|
||||
page: pageIndex,
|
||||
pageSize: pageSize,
|
||||
});
|
||||
const fetchQueryParams = useMemo(
|
||||
() => ({
|
||||
search_space_id: searchSpaceId,
|
||||
page: pageIndex ,
|
||||
page_size: pageSize,
|
||||
...(documentTypeFilter.length > 0 && { document_types: documentTypeFilter }),
|
||||
}),
|
||||
[searchSpaceId, pageIndex, pageSize, documentTypeFilter, debouncedSearch]
|
||||
);
|
||||
|
||||
// Fetch document type counts on mount
|
||||
useEffect(() => {
|
||||
if (searchSpaceId && getDocumentTypeCounts) {
|
||||
getDocumentTypeCounts().then(setTypeCounts);
|
||||
const searchQueryParams = useMemo(() => {
|
||||
return {
|
||||
search_space_id: searchSpaceId,
|
||||
page: pageIndex ,
|
||||
page_size: pageSize,
|
||||
...(documentTypeFilter.length > 0 && { document_types: documentTypeFilter }),
|
||||
title : debouncedSearch,
|
||||
}
|
||||
}, [searchSpaceId, getDocumentTypeCounts]);
|
||||
},[debouncedSearch, searchSpaceId, pageIndex, pageSize, documentTypeFilter, debouncedSearch])
|
||||
|
||||
// Refetch when pagination changes or when search/filters change
|
||||
useEffect(() => {
|
||||
if (searchSpaceId) {
|
||||
if (debouncedSearch.trim()) {
|
||||
searchDocuments?.(
|
||||
debouncedSearch,
|
||||
pageIndex,
|
||||
pageSize,
|
||||
documentTypeFilter.length > 0 ? documentTypeFilter : undefined
|
||||
);
|
||||
} else {
|
||||
fetchDocuments?.(
|
||||
pageIndex,
|
||||
pageSize,
|
||||
documentTypeFilter.length > 0 ? documentTypeFilter : undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
}, [
|
||||
pageIndex,
|
||||
pageSize,
|
||||
debouncedSearch,
|
||||
documentTypeFilter,
|
||||
searchSpaceId,
|
||||
fetchDocuments,
|
||||
searchDocuments,
|
||||
]);
|
||||
// Use query for fetching documents
|
||||
const {
|
||||
data: documents,
|
||||
isLoading: isDocumentsLoading,
|
||||
} = useQuery({
|
||||
queryKey: cacheKeys.documents.withQueryParams(fetchQueryParams),
|
||||
queryFn: () => documentsApiService.getDocuments({ queryParams : fetchQueryParams }),
|
||||
staleTime: 3 * 60 * 1000, // 3 minutes
|
||||
enabled: !!searchSpaceId && !debouncedSearch.trim(),
|
||||
});
|
||||
|
||||
// Seaching
|
||||
const {
|
||||
data: searchedDocuments,
|
||||
isLoading: isSearchedDocumentsLoading,
|
||||
} = useQuery({
|
||||
queryKey: cacheKeys.documents.withQueryParams(searchQueryParams),
|
||||
queryFn: () => documentsApiService.searchDocuments({ queryParams : searchQueryParams }),
|
||||
staleTime: 3 * 60 * 1000, // 3 minutes
|
||||
enabled: !!searchSpaceId && !!debouncedSearch.trim(),
|
||||
});
|
||||
|
||||
|
||||
|
||||
// Use query data when not searching, otherwise use hook data
|
||||
const actualDocuments = debouncedSearch.trim() ? searchedDocuments?.items|| [] : documents?.items|| [];
|
||||
const actualTotal = debouncedSearch.trim() ? searchedDocuments?.total || 0 : documents?.total || 0;
|
||||
const actualLoading = debouncedSearch.trim() ? isSearchedDocumentsLoading : isDocumentsLoading;
|
||||
|
||||
// Memoize initial row selection to prevent infinite loops
|
||||
const initialRowSelection = useMemo(() => {
|
||||
|
|
@ -272,14 +284,14 @@ export function DocumentsDataTable({
|
|||
|
||||
// Update the selected documents map when row selection changes
|
||||
useEffect(() => {
|
||||
if (!documents || documents.length === 0) return;
|
||||
if (!actualDocuments || actualDocuments.length === 0) return;
|
||||
|
||||
setSelectedDocumentsMap((prev) => {
|
||||
const newMap = new Map(prev);
|
||||
let hasChanges = false;
|
||||
|
||||
// Process only current page documents
|
||||
for (const doc of documents) {
|
||||
for (const doc of actualDocuments) {
|
||||
const docId = doc.id;
|
||||
const isSelected = rowSelection[docId.toString()];
|
||||
const wasInMap = newMap.has(docId);
|
||||
|
|
@ -319,14 +331,14 @@ export function DocumentsDataTable({
|
|||
}, [selectedDocumentsArray, onSelectionChange]);
|
||||
|
||||
const table = useReactTable({
|
||||
data: documents || [],
|
||||
data: actualDocuments || [],
|
||||
columns,
|
||||
getRowId: (row) => row.id.toString(),
|
||||
onSortingChange: setSorting,
|
||||
getCoreRowModel: getCoreRowModel(),
|
||||
onRowSelectionChange: setRowSelection,
|
||||
manualPagination: true,
|
||||
pageCount: Math.ceil(total / pageSize),
|
||||
pageCount: Math.ceil(actualTotal / pageSize),
|
||||
state: { sorting, rowSelection, pagination: { pageIndex, pageSize } },
|
||||
});
|
||||
|
||||
|
|
@ -344,7 +356,7 @@ export function DocumentsDataTable({
|
|||
setRowSelection(newSelection);
|
||||
}, [table, rowSelection]);
|
||||
|
||||
const handleToggleType = useCallback((type: string, checked: boolean) => {
|
||||
const handleToggleType = useCallback((type: DocumentTypeEnum, checked: boolean) => {
|
||||
setDocumentTypeFilter((prev) => {
|
||||
if (checked) {
|
||||
return [...prev, type];
|
||||
|
|
@ -358,7 +370,7 @@ export function DocumentsDataTable({
|
|||
|
||||
// Get available document types from type counts (memoized)
|
||||
const availableTypes = useMemo(() => {
|
||||
const types = Object.keys(typeCounts);
|
||||
const types = typeCounts ? Object.keys(typeCounts) as DocumentTypeEnum[] : [];
|
||||
return types.length > 0 ? types.sort() : [];
|
||||
}, [typeCounts]);
|
||||
|
||||
|
|
@ -408,7 +420,7 @@ export function DocumentsDataTable({
|
|||
className="flex grow justify-between gap-2 font-normal text-sm cursor-pointer"
|
||||
>
|
||||
<span>{type.replace(/_/g, " ")}</span>
|
||||
<span className="text-xs text-muted-foreground">{typeCounts[type]}</span>
|
||||
<span className="text-xs text-muted-foreground">{typeCounts?.[type]}</span>
|
||||
</Label>
|
||||
</div>
|
||||
))}
|
||||
|
|
@ -435,7 +447,7 @@ export function DocumentsDataTable({
|
|||
<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3">
|
||||
<div className="flex flex-col sm:flex-row sm:items-center gap-2">
|
||||
<span className="text-sm text-muted-foreground whitespace-nowrap">
|
||||
{selectedCount} selected {loading && "· Loading..."}
|
||||
{selectedCount} selected {actualLoading && "· Loading..."}
|
||||
</span>
|
||||
<div className="hidden sm:block h-4 w-px bg-border mx-2" />
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
|
|
@ -453,7 +465,7 @@ export function DocumentsDataTable({
|
|||
size="sm"
|
||||
onClick={handleSelectPage}
|
||||
className="text-xs sm:text-sm"
|
||||
disabled={loading}
|
||||
disabled={actualLoading}
|
||||
>
|
||||
Select Page
|
||||
</Button>
|
||||
|
|
@ -490,7 +502,7 @@ export function DocumentsDataTable({
|
|||
{/* Table Container */}
|
||||
<div className="border rounded-lg flex-1 min-h-0 overflow-hidden bg-background">
|
||||
<div className="overflow-auto h-full">
|
||||
{loading ? (
|
||||
{actualLoading ? (
|
||||
<div className="flex items-center justify-center h-full">
|
||||
<div className="text-center space-y-2">
|
||||
<div className="animate-spin h-8 w-8 border-2 border-primary border-t-transparent rounded-full mx-auto" />
|
||||
|
|
@ -561,31 +573,31 @@ export function DocumentsDataTable({
|
|||
{/* Footer Pagination */}
|
||||
<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3 text-xs sm:text-sm text-muted-foreground border-t pt-3 md:pt-4 flex-shrink-0">
|
||||
<div className="text-center sm:text-left">
|
||||
Showing {pageIndex * pageSize + 1} to {Math.min((pageIndex + 1) * pageSize, total)} of{" "}
|
||||
{total} documents
|
||||
Showing {pageIndex * pageSize + 1} to {Math.min((pageIndex + 1) * pageSize, actualTotal)} of{" "}
|
||||
{actualTotal} documents
|
||||
</div>
|
||||
<div className="flex items-center justify-center sm:justify-end space-x-2">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => setPageIndex((p) => Math.max(0, p - 1))}
|
||||
disabled={pageIndex === 0 || loading}
|
||||
className="text-xs sm:text-sm"
|
||||
size="sm"
|
||||
onClick={() => setPageIndex((p) => Math.max(0, p - 1))}
|
||||
disabled={pageIndex === 0 || actualLoading}
|
||||
className="text-xs sm:text-sm"
|
||||
>
|
||||
Previous
|
||||
</Button>
|
||||
<div className="flex items-center space-x-1 text-xs sm:text-sm">
|
||||
<span>Page</span>
|
||||
<strong>{pageIndex + 1}</strong>
|
||||
<span>of</span>
|
||||
<strong>{Math.ceil(total / pageSize)}</strong>
|
||||
</div>
|
||||
<strong>{pageIndex + 1}</strong>
|
||||
<span>of</span>
|
||||
<strong>{Math.ceil(actualTotal / pageSize)}</strong>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => setPageIndex((p) => p + 1)}
|
||||
disabled={pageIndex >= Math.ceil(total / pageSize) - 1 || loading}
|
||||
className="text-xs sm:text-sm"
|
||||
size="sm"
|
||||
onClick={() => setPageIndex((p) => p + 1)}
|
||||
disabled={pageIndex >= Math.ceil(actualTotal / pageSize) - 1 || actualLoading}
|
||||
className="text-xs sm:text-sm"
|
||||
>
|
||||
Next
|
||||
</Button>
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
"use client";
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { ChevronDown, ChevronUp, ExternalLink, Loader2 } from "lucide-react";
|
||||
import type React from "react";
|
||||
import { type ReactNode, useEffect, useRef, useState } from "react";
|
||||
import { type ReactNode, useEffect, useLayoutEffect, useRef, useState } from "react";
|
||||
import { MarkdownViewer } from "@/components/markdown-viewer";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
|
||||
|
|
@ -15,7 +16,8 @@ import {
|
|||
SheetTitle,
|
||||
} from "@/components/ui/sheet";
|
||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||
import { useDocumentByChunk } from "@/hooks/use-document-by-chunk";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
interface SourceDetailSheetProps {
|
||||
|
|
@ -46,11 +48,22 @@ export function SourceDetailSheet({
|
|||
url,
|
||||
children,
|
||||
}: SourceDetailSheetProps) {
|
||||
const { document, loading, error, fetchDocumentByChunk, clearDocument } = useDocumentByChunk();
|
||||
const chunksContainerRef = useRef<HTMLDivElement>(null);
|
||||
const highlightedChunkRef = useRef<HTMLDivElement>(null);
|
||||
const [summaryOpen, setSummaryOpen] = useState(false);
|
||||
|
||||
// Add useQuery to fetch document by chunk
|
||||
const {
|
||||
data: document,
|
||||
isLoading: isDocumentByChunkFetching,
|
||||
error: documentByChunkFetchingError,
|
||||
} = useQuery({
|
||||
queryKey: cacheKeys.documents.byChunk(chunkId.toString()),
|
||||
queryFn: () => documentsApiService.getDocumentByChunk({ chunk_id: chunkId }),
|
||||
enabled: !!chunkId && open,
|
||||
staleTime: 5 * 60 * 1000, // 5 minutes
|
||||
});
|
||||
|
||||
// Check if this is a source type that should render directly from node
|
||||
const isDirectRenderSource =
|
||||
sourceType === "TAVILY_API" ||
|
||||
|
|
@ -58,17 +71,9 @@ export function SourceDetailSheet({
|
|||
sourceType === "SEARXNG_API" ||
|
||||
sourceType === "BAIDU_SEARCH_API";
|
||||
|
||||
useEffect(() => {
|
||||
if (open && chunkId && !isDirectRenderSource) {
|
||||
fetchDocumentByChunk(chunkId);
|
||||
} else if (!open && !isDirectRenderSource) {
|
||||
clearDocument();
|
||||
}
|
||||
}, [open, chunkId, isDirectRenderSource, fetchDocumentByChunk, clearDocument]);
|
||||
|
||||
useEffect(() => {
|
||||
// Scroll to highlighted chunk when document loads
|
||||
if (document && highlightedChunkRef.current && chunksContainerRef.current) {
|
||||
if (document) {
|
||||
setTimeout(() => {
|
||||
highlightedChunkRef.current?.scrollIntoView({
|
||||
behavior: "smooth",
|
||||
|
|
@ -76,7 +81,7 @@ export function SourceDetailSheet({
|
|||
});
|
||||
}, 100);
|
||||
}
|
||||
}, [document]);
|
||||
}, [document, open]);
|
||||
|
||||
const handleUrlClick = (e: React.MouseEvent, clickUrl: string) => {
|
||||
e.preventDefault();
|
||||
|
|
@ -100,15 +105,17 @@ export function SourceDetailSheet({
|
|||
</SheetDescription>
|
||||
</SheetHeader>
|
||||
|
||||
{!isDirectRenderSource && loading && (
|
||||
{!isDirectRenderSource && isDocumentByChunkFetching && (
|
||||
<div className="flex items-center justify-center h-64 px-6">
|
||||
<Loader2 className="h-8 w-8 animate-spin text-muted-foreground" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!isDirectRenderSource && error && (
|
||||
{!isDirectRenderSource && documentByChunkFetchingError && (
|
||||
<div className="flex items-center justify-center h-64 px-6">
|
||||
<p className="text-sm text-destructive">{error}</p>
|
||||
<p className="text-sm text-destructive">
|
||||
{documentByChunkFetchingError.message || "Failed to load document"}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import { useAtom } from "jotai";
|
||||
import { CheckCircle2, FileType, Info, Loader2, Tag, Upload, X } from "lucide-react";
|
||||
import { AnimatePresence, motion } from "motion/react";
|
||||
import { useRouter } from "next/navigation";
|
||||
|
|
@ -7,6 +8,7 @@ import { useTranslations } from "next-intl";
|
|||
import { useCallback, useState } from "react";
|
||||
import { useDropzone } from "react-dropzone";
|
||||
import { toast } from "sonner";
|
||||
import { uploadDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
|
||||
|
||||
import { Alert, AlertDescription } from "@/components/ui/alert";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
|
|
@ -14,7 +16,6 @@ import { Button } from "@/components/ui/button";
|
|||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import { Separator } from "@/components/ui/separator";
|
||||
import { getAuthHeaders } from "@/lib/auth-utils";
|
||||
import { GridPattern } from "./GridPattern";
|
||||
|
||||
interface DocumentUploadTabProps {
|
||||
|
|
@ -25,9 +26,12 @@ export function DocumentUploadTab({ searchSpaceId }: DocumentUploadTabProps) {
|
|||
const t = useTranslations("upload_documents");
|
||||
const router = useRouter();
|
||||
const [files, setFiles] = useState<File[]>([]);
|
||||
const [isUploading, setIsUploading] = useState(false);
|
||||
const [uploadProgress, setUploadProgress] = useState(0);
|
||||
|
||||
// Use the uploadDocumentMutationAtom
|
||||
const [uploadDocumentMutation] = useAtom(uploadDocumentMutationAtom);
|
||||
const { mutate: uploadDocuments, isPending: isUploading } = uploadDocumentMutation;
|
||||
|
||||
const audioFileTypes = {
|
||||
"audio/mpeg": [".mp3", ".mpeg", ".mpga"],
|
||||
"audio/mp4": [".mp4", ".m4a"],
|
||||
|
|
@ -148,53 +152,40 @@ export function DocumentUploadTab({ searchSpaceId }: DocumentUploadTabProps) {
|
|||
};
|
||||
|
||||
const handleUpload = async () => {
|
||||
setIsUploading(true);
|
||||
setUploadProgress(0);
|
||||
|
||||
const formData = new FormData();
|
||||
files.forEach((file) => {
|
||||
formData.append("files", file);
|
||||
});
|
||||
formData.append("search_space_id", searchSpaceId);
|
||||
// Create a progress interval to simulate progress
|
||||
const progressInterval = setInterval(() => {
|
||||
setUploadProgress((prev) => {
|
||||
if (prev >= 90) return prev;
|
||||
return prev + Math.random() * 10;
|
||||
});
|
||||
}, 200);
|
||||
|
||||
try {
|
||||
const progressInterval = setInterval(() => {
|
||||
setUploadProgress((prev) => {
|
||||
if (prev >= 90) return prev;
|
||||
return prev + Math.random() * 10;
|
||||
});
|
||||
}, 200);
|
||||
|
||||
const response = await fetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/fileupload`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: getAuthHeaders(),
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
|
||||
clearInterval(progressInterval);
|
||||
setUploadProgress(100);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error("Upload failed");
|
||||
// Use the mutation to upload documents
|
||||
uploadDocuments(
|
||||
{
|
||||
files,
|
||||
search_space_id: Number(searchSpaceId),
|
||||
},
|
||||
{
|
||||
onSuccess: () => {
|
||||
clearInterval(progressInterval);
|
||||
setUploadProgress(100);
|
||||
toast(t("upload_initiated"), {
|
||||
description: t("upload_initiated_desc"),
|
||||
});
|
||||
router.push(`/dashboard/${searchSpaceId}/documents`);
|
||||
},
|
||||
onError: (error: any) => {
|
||||
clearInterval(progressInterval);
|
||||
setUploadProgress(0);
|
||||
toast(t("upload_error"), {
|
||||
description: `${t("upload_error_desc")}: ${error.message || "Upload failed"}`,
|
||||
});
|
||||
},
|
||||
}
|
||||
|
||||
await response.json();
|
||||
|
||||
toast(t("upload_initiated"), {
|
||||
description: t("upload_initiated_desc"),
|
||||
});
|
||||
|
||||
router.push(`/dashboard/${searchSpaceId}/documents`);
|
||||
} catch (error: any) {
|
||||
setIsUploading(false);
|
||||
setUploadProgress(0);
|
||||
toast(t("upload_error"), {
|
||||
description: `${t("upload_error_desc")}: ${error.message}`,
|
||||
});
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
const getTotalFileSize = () => {
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@ import { motion } from "motion/react";
|
|||
import { useRouter } from "next/navigation";
|
||||
import { useTranslations } from "next-intl";
|
||||
import { useState } from "react";
|
||||
import { useAtom } from "jotai";
|
||||
import { toast } from "sonner";
|
||||
import { createDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
|
|
@ -19,7 +21,6 @@ import {
|
|||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { authenticatedFetch } from "@/lib/auth-utils";
|
||||
|
||||
const youtubeRegex =
|
||||
/^(https:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})$/;
|
||||
|
|
@ -33,9 +34,12 @@ export function YouTubeTab({ searchSpaceId }: YouTubeTabProps) {
|
|||
const router = useRouter();
|
||||
const [videoTags, setVideoTags] = useState<TagType[]>([]);
|
||||
const [activeTagIndex, setActiveTagIndex] = useState<number | null>(null);
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
// Use the createDocumentMutationAtom
|
||||
const [createDocumentMutation] = useAtom(createDocumentMutationAtom);
|
||||
const { mutate: createYouTubeDocument, isPending: isSubmitting } = createDocumentMutation;
|
||||
|
||||
const isValidYoutubeUrl = (url: string): boolean => {
|
||||
return youtubeRegex.test(url);
|
||||
};
|
||||
|
|
@ -58,47 +62,35 @@ export function YouTubeTab({ searchSpaceId }: YouTubeTabProps) {
|
|||
}
|
||||
|
||||
setError(null);
|
||||
setIsSubmitting(true);
|
||||
|
||||
try {
|
||||
toast(t("processing_toast"), {
|
||||
description: t("processing_toast_desc"),
|
||||
});
|
||||
toast(t("processing_toast"), {
|
||||
description: t("processing_toast_desc"),
|
||||
});
|
||||
|
||||
const videoUrls = videoTags.map((tag) => tag.text);
|
||||
const videoUrls = videoTags.map((tag) => tag.text);
|
||||
|
||||
const response = await authenticatedFetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
document_type: "YOUTUBE_VIDEO",
|
||||
content: videoUrls,
|
||||
search_space_id: parseInt(searchSpaceId),
|
||||
}),
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error("Failed to process YouTube videos");
|
||||
// Use the mutation to create YouTube documents
|
||||
createYouTubeDocument(
|
||||
{
|
||||
document_type: "YOUTUBE_VIDEO",
|
||||
content: videoUrls,
|
||||
search_space_id: parseInt(searchSpaceId),
|
||||
},
|
||||
{
|
||||
onSuccess: () => {
|
||||
toast(t("success_toast"), {
|
||||
description: t("success_toast_desc"),
|
||||
});
|
||||
router.push(`/dashboard/${searchSpaceId}/documents`);
|
||||
},
|
||||
onError: (error: any) => {
|
||||
setError(error.message || t("error_generic"));
|
||||
toast(t("error_toast"), {
|
||||
description: `${t("error_toast_desc")}: ${error.message || "Failed to process YouTube videos"}`,
|
||||
});
|
||||
},
|
||||
}
|
||||
|
||||
await response.json();
|
||||
|
||||
toast(t("success_toast"), {
|
||||
description: t("success_toast_desc"),
|
||||
});
|
||||
|
||||
router.push(`/dashboard/${searchSpaceId}/documents`);
|
||||
} catch (error: any) {
|
||||
setError(error.message || t("error_generic"));
|
||||
toast(t("error_toast"), {
|
||||
description: `${t("error_toast_desc")}: ${error.message}`,
|
||||
});
|
||||
} finally {
|
||||
setIsSubmitting(false);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
const handleAddTag = (text: string) => {
|
||||
|
|
|
|||
183
surfsense_web/contracts/types/document.types.ts
Normal file
183
surfsense_web/contracts/types/document.types.ts
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
import { z } from "zod";
|
||||
import { paginationQueryParams } from ".";
|
||||
|
||||
export const documentTypeEnum = z.enum([
|
||||
"EXTENSION",
|
||||
"CRAWLED_URL",
|
||||
"FILE",
|
||||
"SLACK_CONNECTOR",
|
||||
"NOTION_CONNECTOR",
|
||||
"YOUTUBE_VIDEO",
|
||||
"GITHUB_CONNECTOR",
|
||||
"DISCORD_CONNECTOR",
|
||||
"JIRA_CONNECTOR",
|
||||
"CONFLUENCE_CONNECTOR",
|
||||
"CLICKUP_CONNECTOR",
|
||||
"GOOGLE_CALENDAR_CONNECTOR",
|
||||
"GOOGLE_GMAIL_CONNECTOR",
|
||||
"AIRTABLE_CONNECTOR",
|
||||
"LUMA_CONNECTOR",
|
||||
"ELASTICSEARCH_CONNECTOR",
|
||||
"LINEAR_CONNECTOR",
|
||||
]);
|
||||
|
||||
export const document = z.object({
|
||||
id: z.number(),
|
||||
title: z.string(),
|
||||
document_type: documentTypeEnum,
|
||||
document_metadata: z.record(z.string(), z.any()),
|
||||
content: z.string(),
|
||||
created_at: z.string(),
|
||||
search_space_id: z.number(),
|
||||
});
|
||||
|
||||
export const extensionDocumentContent = z.object({
|
||||
metadata: z.object({
|
||||
BrowsingSessionId: z.string(),
|
||||
VisitedWebPageURL: z.string(),
|
||||
VisitedWebPageTitle: z.string(),
|
||||
VisitedWebPageDateWithTimeInISOString: z.string(),
|
||||
VisitedWebPageReffererURL: z.string(),
|
||||
VisitedWebPageVisitDurationInMilliseconds: z.string(),
|
||||
}),
|
||||
pageContent: z.string(),
|
||||
});
|
||||
|
||||
export const documentWithChunks = document.extend({
|
||||
chunks: z.array(
|
||||
z.object({
|
||||
id: z.number(),
|
||||
content: z.string(),
|
||||
created_at: z.string(),
|
||||
})
|
||||
),
|
||||
});
|
||||
|
||||
/**
|
||||
* Get documents
|
||||
*/
|
||||
export const getDocumentsRequest = z.object({
|
||||
queryParams: paginationQueryParams
|
||||
.extend({
|
||||
search_space_id: z.number().or(z.string()).optional(),
|
||||
document_types: z.array(documentTypeEnum).optional(),
|
||||
})
|
||||
.nullish(),
|
||||
});
|
||||
|
||||
export const getDocumentsResponse = z.object({
|
||||
items: z.array(document),
|
||||
total: z.number(),
|
||||
});
|
||||
|
||||
/**
|
||||
* Get document
|
||||
*/
|
||||
export const getDocumentRequest = document.pick({ id: true });
|
||||
|
||||
export const getDocumentResponse = document;
|
||||
|
||||
/**
|
||||
* Create documents
|
||||
*/
|
||||
export const createDocumentRequest = document
|
||||
.pick({ document_type: true, search_space_id: true })
|
||||
.extend({
|
||||
content: z.string().or(z.array(z.string())).or(z.array(extensionDocumentContent)),
|
||||
});
|
||||
|
||||
export const createDocumentResponse = z.object({
|
||||
message: z.literal("Documents created successfully"),
|
||||
});
|
||||
|
||||
/**
|
||||
* Upload documents
|
||||
*/
|
||||
export const uploadDocumentRequest = z.object({
|
||||
files: z.array(z.instanceof(File)),
|
||||
search_space_id: z.number(),
|
||||
});
|
||||
|
||||
export const uploadDocumentResponse = z.object({
|
||||
message: z.literal("Files uploaded for processing"),
|
||||
});
|
||||
|
||||
/**
|
||||
* Search documents
|
||||
*/
|
||||
export const searchDocumentsRequest = z.object({
|
||||
queryParams: paginationQueryParams
|
||||
.extend({
|
||||
search_space_id: z.number().or(z.string()).optional(),
|
||||
document_types: z.array(documentTypeEnum).optional(),
|
||||
title: z.string().optional(),
|
||||
})
|
||||
.nullish(),
|
||||
});
|
||||
|
||||
export const searchDocumentsResponse = z.object({
|
||||
items: z.array(document),
|
||||
total: z.number(),
|
||||
});
|
||||
|
||||
/**
|
||||
* Get document type counts
|
||||
*/
|
||||
export const getDocumentTypeCountsRequest = z.object({
|
||||
queryParams: z
|
||||
.object({
|
||||
search_space_id: z.number().or(z.string()).optional(),
|
||||
})
|
||||
.nullish(),
|
||||
});
|
||||
|
||||
export const getDocumentTypeCountsResponse = z.record(z.string(), z.number());
|
||||
|
||||
/**
|
||||
* Get document by chunk
|
||||
*/
|
||||
export const getDocumentByChunkRequest = z.object({
|
||||
chunk_id: z.number(),
|
||||
});
|
||||
|
||||
export const getDocumentByChunkResponse = documentWithChunks;
|
||||
|
||||
/**
|
||||
* Update document
|
||||
*/
|
||||
export const updateDocumentRequest = z.object({
|
||||
id: z.number(),
|
||||
data: document.pick({ search_space_id: true, document_type: true, content: true }),
|
||||
});
|
||||
|
||||
export const updateDocumentResponse = document;
|
||||
|
||||
/**
|
||||
* Delete document
|
||||
*/
|
||||
export const deleteDocumentRequest = document.pick({ id: true });
|
||||
|
||||
export const deleteDocumentResponse = z.object({
|
||||
message: z.literal("Document deleted successfully"),
|
||||
});
|
||||
|
||||
export type Document = z.infer<typeof document>
|
||||
export type GetDocumentsRequest = z.infer<typeof getDocumentsRequest>;
|
||||
export type GetDocumentsResponse = z.infer<typeof getDocumentsResponse>;
|
||||
export type GetDocumentRequest = z.infer<typeof getDocumentRequest>;
|
||||
export type GetDocumentResponse = z.infer<typeof getDocumentResponse>;
|
||||
export type CreateDocumentRequest = z.infer<typeof createDocumentRequest>;
|
||||
export type CreateDocumentResponse = z.infer<typeof createDocumentResponse>;
|
||||
export type UploadDocumentRequest = z.infer<typeof uploadDocumentRequest>;
|
||||
export type UploadDocumentResponse = z.infer<typeof uploadDocumentResponse>;
|
||||
export type SearchDocumentsRequest = z.infer<typeof searchDocumentsRequest>;
|
||||
export type SearchDocumentsResponse = z.infer<typeof searchDocumentsResponse>;
|
||||
export type GetDocumentTypeCountsRequest = z.infer<typeof getDocumentTypeCountsRequest>;
|
||||
export type GetDocumentTypeCountsResponse = z.infer<typeof getDocumentTypeCountsResponse>;
|
||||
export type GetDocumentByChunkRequest = z.infer<typeof getDocumentByChunkRequest>;
|
||||
export type GetDocumentByChunkResponse = z.infer<typeof getDocumentByChunkResponse>;
|
||||
export type UpdateDocumentRequest = z.infer<typeof updateDocumentRequest>;
|
||||
export type UpdateDocumentResponse = z.infer<typeof updateDocumentResponse>;
|
||||
export type DeleteDocumentRequest = z.infer<typeof deleteDocumentRequest>;
|
||||
export type DeleteDocumentResponse = z.infer<typeof deleteDocumentResponse>;
|
||||
export type DocumentTypeEnum = z.infer<typeof documentTypeEnum>
|
||||
|
|
@ -3,6 +3,10 @@ import { z } from "zod";
|
|||
export const paginationQueryParams = z.object({
|
||||
limit: z.number().optional(),
|
||||
skip: z.number().optional(),
|
||||
// OR
|
||||
page: z.number().optional(),
|
||||
page_size: z.number().optional(),
|
||||
// TODO: Normalize to page/page_size or limit/skip in backend
|
||||
});
|
||||
|
||||
export type PaginationQueryParams = z.infer<typeof paginationQueryParams>;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
export * from "./use-document-by-chunk";
|
||||
export * from "./use-logs";
|
||||
export * from "./use-rbac";
|
||||
export * from "./use-search-source-connectors";
|
||||
|
|
|
|||
|
|
@ -1,106 +0,0 @@
|
|||
"use client";
|
||||
import { useCallback, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { authenticatedFetch } from "@/lib/auth-utils";
|
||||
|
||||
export interface Chunk {
|
||||
id: number;
|
||||
content: string;
|
||||
document_id: number;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface DocumentWithChunks {
|
||||
id: number;
|
||||
title: string;
|
||||
document_type: DocumentType;
|
||||
document_metadata: any;
|
||||
content: string;
|
||||
created_at: string;
|
||||
search_space_id: number;
|
||||
chunks: Chunk[];
|
||||
}
|
||||
|
||||
export type DocumentType =
|
||||
| "EXTENSION"
|
||||
| "CRAWLED_URL"
|
||||
| "SLACK_CONNECTOR"
|
||||
| "NOTION_CONNECTOR"
|
||||
| "FILE"
|
||||
| "YOUTUBE_VIDEO"
|
||||
| "GITHUB_CONNECTOR"
|
||||
| "LINEAR_CONNECTOR"
|
||||
| "DISCORD_CONNECTOR"
|
||||
| "JIRA_CONNECTOR"
|
||||
| "CONFLUENCE_CONNECTOR"
|
||||
| "CLICKUP_CONNECTOR"
|
||||
| "GOOGLE_CALENDAR_CONNECTOR"
|
||||
| "GOOGLE_GMAIL_CONNECTOR"
|
||||
| "LUMA_CONNECTOR"
|
||||
| "ELASTICSEARCH_CONNECTOR";
|
||||
|
||||
export function useDocumentByChunk() {
|
||||
const [document, setDocument] = useState<DocumentWithChunks | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const fetchDocumentByChunk = useCallback(async (chunkId: number) => {
|
||||
try {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
setDocument(null);
|
||||
|
||||
const response = await authenticatedFetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/by-chunk/${chunkId}`,
|
||||
{
|
||||
headers: { "Content-Type": "application/json" },
|
||||
method: "GET",
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
let errorMessage = "Failed to fetch document";
|
||||
|
||||
try {
|
||||
const errorData = JSON.parse(errorText);
|
||||
errorMessage = errorData.detail || errorMessage;
|
||||
} catch {
|
||||
// If parsing fails, use default message
|
||||
}
|
||||
|
||||
if (response.status === 404) {
|
||||
errorMessage = "Chunk not found or you don't have access to it";
|
||||
}
|
||||
|
||||
toast.error(errorMessage);
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
const data: DocumentWithChunks = await response.json();
|
||||
setDocument(data);
|
||||
setError(null);
|
||||
return data;
|
||||
} catch (err: any) {
|
||||
const errorMessage = err.message || "Failed to fetch document";
|
||||
setError(errorMessage);
|
||||
console.error("Error fetching document by chunk:", err);
|
||||
throw err;
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const clearDocument = useCallback(() => {
|
||||
setDocument(null);
|
||||
setError(null);
|
||||
}, []);
|
||||
|
||||
return {
|
||||
document,
|
||||
loading,
|
||||
error,
|
||||
fetchDocumentByChunk,
|
||||
clearDocument,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
import { useCallback, useEffect, useState } from "react";
|
||||
import { authenticatedFetch } from "@/lib/auth-utils";
|
||||
|
||||
export interface DocumentTypeCount {
|
||||
type: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook to fetch document type counts from the API
|
||||
* @param searchSpaceId - The search space ID to filter document types
|
||||
* @param lazy - If true, types won't be fetched on mount
|
||||
*/
|
||||
export const useDocumentTypes = (searchSpaceId?: number, lazy: boolean = false) => {
|
||||
const [documentTypes, setDocumentTypes] = useState<DocumentTypeCount[]>([]);
|
||||
const [isLoading, setIsLoading] = useState(!lazy);
|
||||
const [isLoaded, setIsLoaded] = useState(false);
|
||||
const [error, setError] = useState<Error | null>(null);
|
||||
|
||||
const fetchDocumentTypes = useCallback(
|
||||
async (spaceId?: number) => {
|
||||
if (isLoaded && lazy) return;
|
||||
|
||||
try {
|
||||
setIsLoading(true);
|
||||
setError(null);
|
||||
|
||||
// Build URL with optional search_space_id query parameter
|
||||
const url = new URL(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/type-counts`
|
||||
);
|
||||
if (spaceId !== undefined) {
|
||||
url.searchParams.append("search_space_id", spaceId.toString());
|
||||
}
|
||||
|
||||
const response = await authenticatedFetch(url.toString(), {
|
||||
method: "GET",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch document types: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Convert the object to an array of DocumentTypeCount
|
||||
const typeCounts: DocumentTypeCount[] = Object.entries(data).map(([type, count]) => ({
|
||||
type,
|
||||
count: count as number,
|
||||
}));
|
||||
|
||||
setDocumentTypes(typeCounts);
|
||||
setIsLoaded(true);
|
||||
|
||||
return typeCounts;
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err : new Error("An unknown error occurred"));
|
||||
console.error("Error fetching document types:", err);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
},
|
||||
[isLoaded, lazy]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!lazy) {
|
||||
fetchDocumentTypes(searchSpaceId);
|
||||
}
|
||||
}, [lazy, fetchDocumentTypes, searchSpaceId]);
|
||||
|
||||
// Function to refresh the document types
|
||||
const refreshDocumentTypes = useCallback(
|
||||
async (spaceId?: number) => {
|
||||
setIsLoaded(false);
|
||||
await fetchDocumentTypes(spaceId !== undefined ? spaceId : searchSpaceId);
|
||||
},
|
||||
[fetchDocumentTypes, searchSpaceId]
|
||||
);
|
||||
|
||||
return {
|
||||
documentTypes,
|
||||
isLoading,
|
||||
isLoaded,
|
||||
error,
|
||||
fetchDocumentTypes,
|
||||
refreshDocumentTypes,
|
||||
};
|
||||
};
|
||||
|
|
@ -1,246 +0,0 @@
|
|||
"use client";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { authenticatedFetch } from "@/lib/auth-utils";
|
||||
import { normalizeListResponse } from "@/lib/pagination";
|
||||
|
||||
export interface Document {
|
||||
id: number;
|
||||
title: string;
|
||||
document_type: DocumentType;
|
||||
document_metadata: any;
|
||||
content: string;
|
||||
created_at: string;
|
||||
search_space_id: number;
|
||||
}
|
||||
|
||||
export type DocumentType =
|
||||
| "EXTENSION"
|
||||
| "CRAWLED_URL"
|
||||
| "SLACK_CONNECTOR"
|
||||
| "NOTION_CONNECTOR"
|
||||
| "FILE"
|
||||
| "YOUTUBE_VIDEO"
|
||||
| "GITHUB_CONNECTOR"
|
||||
| "LINEAR_CONNECTOR"
|
||||
| "DISCORD_CONNECTOR"
|
||||
| "JIRA_CONNECTOR"
|
||||
| "CONFLUENCE_CONNECTOR"
|
||||
| "CLICKUP_CONNECTOR"
|
||||
| "GOOGLE_CALENDAR_CONNECTOR"
|
||||
| "GOOGLE_GMAIL_CONNECTOR"
|
||||
| "AIRTABLE_CONNECTOR"
|
||||
| "LUMA_CONNECTOR"
|
||||
| "ELASTICSEARCH_CONNECTOR";
|
||||
|
||||
export interface UseDocumentsOptions {
|
||||
page?: number;
|
||||
pageSize?: number;
|
||||
lazy?: boolean;
|
||||
documentTypes?: string[];
|
||||
}
|
||||
|
||||
export function useDocuments(searchSpaceId: number, options?: UseDocumentsOptions | boolean) {
|
||||
// Support both old boolean API and new options API for backward compatibility
|
||||
const opts = typeof options === "boolean" ? { lazy: options } : options || {};
|
||||
const { page, pageSize = 300, lazy = false, documentTypes } = opts;
|
||||
|
||||
const [documents, setDocuments] = useState<Document[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [loading, setLoading] = useState(!lazy); // Don't show loading initially for lazy mode
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [isLoaded, setIsLoaded] = useState(false); // Memoization flag
|
||||
|
||||
const fetchDocuments = useCallback(
|
||||
async (fetchPage?: number, fetchPageSize?: number, fetchDocumentTypes?: string[]) => {
|
||||
if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode
|
||||
|
||||
try {
|
||||
setLoading(true);
|
||||
|
||||
// Build query params
|
||||
const params = new URLSearchParams({
|
||||
search_space_id: searchSpaceId.toString(),
|
||||
});
|
||||
|
||||
// Use passed parameters or fall back to state/options
|
||||
const effectivePage = fetchPage !== undefined ? fetchPage : page;
|
||||
const effectivePageSize = fetchPageSize !== undefined ? fetchPageSize : pageSize;
|
||||
const effectiveDocumentTypes =
|
||||
fetchDocumentTypes !== undefined ? fetchDocumentTypes : documentTypes;
|
||||
|
||||
if (effectivePage !== undefined) {
|
||||
params.append("page", effectivePage.toString());
|
||||
}
|
||||
if (effectivePageSize !== undefined) {
|
||||
params.append("page_size", effectivePageSize.toString());
|
||||
}
|
||||
if (effectiveDocumentTypes && effectiveDocumentTypes.length > 0) {
|
||||
params.append("document_types", effectiveDocumentTypes.join(","));
|
||||
}
|
||||
|
||||
const response = await authenticatedFetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?${params.toString()}`,
|
||||
{ method: "GET" }
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
toast.error("Failed to fetch documents");
|
||||
throw new Error("Failed to fetch documents");
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const normalized = normalizeListResponse<Document>(data);
|
||||
setDocuments(normalized.items);
|
||||
setTotal(normalized.total);
|
||||
setError(null);
|
||||
setIsLoaded(true);
|
||||
} catch (err: any) {
|
||||
setError(err.message || "Failed to fetch documents");
|
||||
console.error("Error fetching documents:", err);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
},
|
||||
[searchSpaceId, page, pageSize, documentTypes, isLoaded, lazy]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!lazy && searchSpaceId) {
|
||||
fetchDocuments();
|
||||
}
|
||||
}, [searchSpaceId, lazy, fetchDocuments]);
|
||||
|
||||
// Function to refresh the documents list
|
||||
const refreshDocuments = useCallback(async () => {
|
||||
setIsLoaded(false); // Reset memoization flag to allow refetch
|
||||
await fetchDocuments();
|
||||
}, [fetchDocuments]);
|
||||
|
||||
// Function to search documents by title
|
||||
const searchDocuments = useCallback(
|
||||
async (
|
||||
searchQuery: string,
|
||||
fetchPage?: number,
|
||||
fetchPageSize?: number,
|
||||
fetchDocumentTypes?: string[]
|
||||
) => {
|
||||
if (!searchQuery.trim()) {
|
||||
// If search is empty, fetch all documents
|
||||
return fetchDocuments(fetchPage, fetchPageSize, fetchDocumentTypes);
|
||||
}
|
||||
|
||||
try {
|
||||
setLoading(true);
|
||||
|
||||
// Build query params
|
||||
const params = new URLSearchParams({
|
||||
search_space_id: searchSpaceId.toString(),
|
||||
title: searchQuery,
|
||||
});
|
||||
|
||||
// Use passed parameters or fall back to state/options
|
||||
const effectivePage = fetchPage !== undefined ? fetchPage : page;
|
||||
const effectivePageSize = fetchPageSize !== undefined ? fetchPageSize : pageSize;
|
||||
const effectiveDocumentTypes =
|
||||
fetchDocumentTypes !== undefined ? fetchDocumentTypes : documentTypes;
|
||||
|
||||
if (effectivePage !== undefined) {
|
||||
params.append("page", effectivePage.toString());
|
||||
}
|
||||
if (effectivePageSize !== undefined) {
|
||||
params.append("page_size", effectivePageSize.toString());
|
||||
}
|
||||
if (effectiveDocumentTypes && effectiveDocumentTypes.length > 0) {
|
||||
params.append("document_types", effectiveDocumentTypes.join(","));
|
||||
}
|
||||
|
||||
const response = await authenticatedFetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/search?${params.toString()}`,
|
||||
{ method: "GET" }
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
toast.error("Failed to search documents");
|
||||
throw new Error("Failed to search documents");
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const normalized = normalizeListResponse<Document>(data);
|
||||
setDocuments(normalized.items);
|
||||
setTotal(normalized.total);
|
||||
setError(null);
|
||||
} catch (err: any) {
|
||||
setError(err.message || "Failed to search documents");
|
||||
console.error("Error searching documents:", err);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
},
|
||||
[searchSpaceId, page, pageSize, documentTypes, fetchDocuments]
|
||||
);
|
||||
|
||||
// Function to delete a document
|
||||
const deleteDocument = useCallback(
|
||||
async (documentId: number) => {
|
||||
try {
|
||||
const response = await authenticatedFetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/${documentId}`,
|
||||
{ method: "DELETE" }
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
toast.error("Failed to delete document");
|
||||
throw new Error("Failed to delete document");
|
||||
}
|
||||
|
||||
toast.success("Document deleted successfully");
|
||||
// Update the local state after successful deletion
|
||||
setDocuments(documents.filter((doc) => doc.id !== documentId));
|
||||
return true;
|
||||
} catch (err: any) {
|
||||
toast.error(err.message || "Failed to delete document");
|
||||
console.error("Error deleting document:", err);
|
||||
return false;
|
||||
}
|
||||
},
|
||||
[documents]
|
||||
);
|
||||
|
||||
// Function to get document type counts
|
||||
const getDocumentTypeCounts = useCallback(async () => {
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
search_space_id: searchSpaceId.toString(),
|
||||
});
|
||||
|
||||
const response = await authenticatedFetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/type-counts?${params.toString()}`,
|
||||
{ method: "GET" }
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error("Failed to fetch document type counts");
|
||||
}
|
||||
|
||||
const counts = await response.json();
|
||||
return counts as Record<string, number>;
|
||||
} catch (err: any) {
|
||||
console.error("Error fetching document type counts:", err);
|
||||
return {};
|
||||
}
|
||||
}, [searchSpaceId]);
|
||||
|
||||
return {
|
||||
documents,
|
||||
total,
|
||||
loading,
|
||||
error,
|
||||
isLoaded,
|
||||
fetchDocuments, // Manual fetch function for lazy mode
|
||||
searchDocuments, // Search function
|
||||
refreshDocuments,
|
||||
deleteDocument,
|
||||
getDocumentTypeCounts, // Get type counts function
|
||||
};
|
||||
}
|
||||
|
|
@ -58,7 +58,6 @@ class BaseApiService {
|
|||
*/
|
||||
const defaultOptions: RequestOptions = {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${this.bearerToken || ""}`,
|
||||
},
|
||||
method: "GET",
|
||||
|
|
@ -211,8 +210,11 @@ class BaseApiService {
|
|||
options?: Omit<RequestOptions, "method" | "responseType">
|
||||
) {
|
||||
return this.request(url, responseSchema, {
|
||||
...options,
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
...options,
|
||||
responseType: ResponseType.JSON,
|
||||
});
|
||||
}
|
||||
|
|
@ -224,6 +226,9 @@ class BaseApiService {
|
|||
) {
|
||||
return this.request(url, responseSchema, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
...options,
|
||||
responseType: ResponseType.JSON,
|
||||
});
|
||||
|
|
@ -236,6 +241,9 @@ class BaseApiService {
|
|||
) {
|
||||
return this.request(url, responseSchema, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
...options,
|
||||
responseType: ResponseType.JSON,
|
||||
});
|
||||
|
|
@ -248,6 +256,9 @@ class BaseApiService {
|
|||
) {
|
||||
return this.request(url, responseSchema, {
|
||||
method: "DELETE",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
...options,
|
||||
responseType: ResponseType.JSON,
|
||||
});
|
||||
|
|
@ -260,6 +271,26 @@ class BaseApiService {
|
|||
responseType: ResponseType.BLOB,
|
||||
});
|
||||
}
|
||||
|
||||
async postFormData<T>(
|
||||
url: string,
|
||||
responseSchema?: z.ZodSchema<T>,
|
||||
options?: Omit<RequestOptions, "method" | "responseType" | "body"> & { body: FormData }
|
||||
) {
|
||||
// Remove Content-Type from options headers if present
|
||||
const { "Content-Type": _, ...headersWithoutContentType } = options?.headers ?? {};
|
||||
|
||||
return this.request(url, responseSchema, {
|
||||
method: "POST",
|
||||
...options,
|
||||
headers: {
|
||||
// Don't set Content-Type - let browser set it with multipart boundary
|
||||
Authorization: `Bearer ${this.bearerToken}`,
|
||||
...headersWithoutContentType,
|
||||
},
|
||||
responseType: ResponseType.JSON,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const baseApiService = new BaseApiService(
|
||||
|
|
|
|||
253
surfsense_web/lib/apis/documents-api.service.ts
Normal file
253
surfsense_web/lib/apis/documents-api.service.ts
Normal file
|
|
@ -0,0 +1,253 @@
|
|||
import {
|
||||
type CreateDocumentRequest,
|
||||
createDocumentRequest,
|
||||
createDocumentResponse,
|
||||
type DeleteDocumentRequest,
|
||||
deleteDocumentRequest,
|
||||
deleteDocumentResponse,
|
||||
type GetDocumentByChunkRequest,
|
||||
type GetDocumentRequest,
|
||||
type GetDocumentsRequest,
|
||||
type GetDocumentTypeCountsRequest,
|
||||
getDocumentByChunkRequest,
|
||||
getDocumentByChunkResponse,
|
||||
getDocumentRequest,
|
||||
getDocumentResponse,
|
||||
getDocumentsRequest,
|
||||
getDocumentsResponse,
|
||||
getDocumentTypeCountsRequest,
|
||||
getDocumentTypeCountsResponse,
|
||||
type SearchDocumentsRequest,
|
||||
searchDocumentsRequest,
|
||||
searchDocumentsResponse,
|
||||
type UpdateDocumentRequest,
|
||||
type UploadDocumentRequest,
|
||||
updateDocumentRequest,
|
||||
updateDocumentResponse,
|
||||
uploadDocumentRequest,
|
||||
uploadDocumentResponse,
|
||||
} from "@/contracts/types/document.types";
|
||||
import { ValidationError } from "../error";
|
||||
import { baseApiService } from "./base-api.service";
|
||||
|
||||
class DocumentsApiService {
|
||||
/**
|
||||
* Get a list of documents with optional filtering and pagination
|
||||
*/
|
||||
getDocuments = async (request: GetDocumentsRequest) => {
|
||||
const parsedRequest = getDocumentsRequest.safeParse(request);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
console.error("Invalid request:", parsedRequest.error);
|
||||
|
||||
const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
|
||||
throw new ValidationError(`Invalid request: ${errorMessage}`);
|
||||
}
|
||||
|
||||
// Transform query params to be string values
|
||||
const transformedQueryParams = parsedRequest.data.queryParams
|
||||
? Object.fromEntries(
|
||||
Object.entries(parsedRequest.data.queryParams).map(([k, v]) => {
|
||||
// Handle array values (document_type)
|
||||
if (Array.isArray(v)) {
|
||||
return [k, v.join(",")];
|
||||
}
|
||||
return [k, String(v)];
|
||||
})
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const queryParams = transformedQueryParams
|
||||
? new URLSearchParams(transformedQueryParams).toString()
|
||||
: "";
|
||||
|
||||
return baseApiService.get(`/api/v1/documents?${queryParams}`, getDocumentsResponse);
|
||||
};
|
||||
|
||||
/**
|
||||
* Get a single document by ID
|
||||
*/
|
||||
getDocument = async (request: GetDocumentRequest) => {
|
||||
const parsedRequest = getDocumentRequest.safeParse(request);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
console.error("Invalid request:", parsedRequest.error);
|
||||
|
||||
const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
|
||||
throw new ValidationError(`Invalid request: ${errorMessage}`);
|
||||
}
|
||||
|
||||
return baseApiService.get(`/api/v1/documents/${request.id}`, getDocumentResponse);
|
||||
};
|
||||
|
||||
/**
|
||||
* Create documents (extension, crawled URL, or YouTube video)
|
||||
*/
|
||||
createDocument = async (request: CreateDocumentRequest) => {
|
||||
const parsedRequest = createDocumentRequest.safeParse(request);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
console.error("Invalid request:", parsedRequest.error);
|
||||
|
||||
const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
|
||||
throw new ValidationError(`Invalid request: ${errorMessage}`);
|
||||
}
|
||||
|
||||
return baseApiService.post(`/api/v1/documents`, createDocumentResponse, {
|
||||
body: parsedRequest.data,
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Upload document files
|
||||
*/
|
||||
uploadDocument = async (request: UploadDocumentRequest) => {
|
||||
const parsedRequest = uploadDocumentRequest.safeParse(request);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
console.error("Invalid request:", parsedRequest.error);
|
||||
|
||||
const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
|
||||
throw new ValidationError(`Invalid request: ${errorMessage}`);
|
||||
}
|
||||
|
||||
// Create FormData for file upload
|
||||
const formData = new FormData();
|
||||
parsedRequest.data.files.forEach((file) => {
|
||||
formData.append("files", file);
|
||||
});
|
||||
formData.append("search_space_id", String(parsedRequest.data.search_space_id));
|
||||
|
||||
return baseApiService.postFormData(`/api/v1/documents/fileupload`, uploadDocumentResponse, {
|
||||
body: formData,
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Search documents by title
|
||||
*/
|
||||
searchDocuments = async (request: SearchDocumentsRequest) => {
|
||||
const parsedRequest = searchDocumentsRequest.safeParse(request);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
console.error("Invalid request:", parsedRequest.error);
|
||||
|
||||
const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
|
||||
throw new ValidationError(`Invalid request: ${errorMessage}`);
|
||||
}
|
||||
|
||||
// Transform query params to be string values
|
||||
const transformedQueryParams = parsedRequest.data.queryParams
|
||||
? Object.fromEntries(
|
||||
Object.entries(parsedRequest.data.queryParams).map(([k, v]) => {
|
||||
// Handle array values (document_type)
|
||||
if (Array.isArray(v)) {
|
||||
return [k, v.join(",")];
|
||||
}
|
||||
return [k, String(v)];
|
||||
})
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const queryParams = transformedQueryParams
|
||||
? new URLSearchParams(transformedQueryParams).toString()
|
||||
: "";
|
||||
|
||||
return baseApiService.get(`/api/v1/documents/search?${queryParams}`, searchDocumentsResponse);
|
||||
};
|
||||
|
||||
/**
|
||||
* Get document type counts
|
||||
*/
|
||||
getDocumentTypeCounts = async (request: GetDocumentTypeCountsRequest) => {
|
||||
// Validate the request
|
||||
const parsedRequest = getDocumentTypeCountsRequest.safeParse(request);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
console.error("Invalid request:", parsedRequest.error);
|
||||
|
||||
// Format a user friendly error message
|
||||
const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
|
||||
throw new ValidationError(`Invalid request: ${errorMessage}`);
|
||||
}
|
||||
|
||||
// Transform query params to be string values
|
||||
const transformedQueryParams = parsedRequest.data.queryParams
|
||||
? Object.fromEntries(
|
||||
Object.entries(parsedRequest.data.queryParams).map(([k, v]) => [k, String(v)])
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const queryParams = transformedQueryParams
|
||||
? new URLSearchParams(transformedQueryParams).toString()
|
||||
: "";
|
||||
|
||||
return baseApiService.get(
|
||||
`/api/v1/documents/type-counts?${queryParams}`,
|
||||
getDocumentTypeCountsResponse
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Get document by chunk ID (includes all chunks)
|
||||
*/
|
||||
getDocumentByChunk = async (request: GetDocumentByChunkRequest) => {
|
||||
// Validate the request
|
||||
const parsedRequest = getDocumentByChunkRequest.safeParse(request);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
console.error("Invalid request:", parsedRequest.error);
|
||||
|
||||
// Format a user friendly error message
|
||||
const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
|
||||
throw new ValidationError(`Invalid request: ${errorMessage}`);
|
||||
}
|
||||
|
||||
return baseApiService.get(
|
||||
`/api/v1/documents/by-chunk/${request.chunk_id}`,
|
||||
getDocumentByChunkResponse
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Update a document
|
||||
*/
|
||||
updateDocument = async (request: UpdateDocumentRequest) => {
|
||||
// Validate the request
|
||||
const parsedRequest = updateDocumentRequest.safeParse(request);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
console.error("Invalid request:", parsedRequest.error);
|
||||
|
||||
// Format a user friendly error message
|
||||
const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
|
||||
throw new ValidationError(`Invalid request: ${errorMessage}`);
|
||||
}
|
||||
|
||||
const { id, data } = parsedRequest.data;
|
||||
|
||||
return baseApiService.put(`/api/v1/documents/${id}`, updateDocumentResponse, {
|
||||
body: data,
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Delete a document
|
||||
*/
|
||||
deleteDocument = async (request: DeleteDocumentRequest) => {
|
||||
// Validate the request
|
||||
const parsedRequest = deleteDocumentRequest.safeParse(request);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
console.error("Invalid request:", parsedRequest.error);
|
||||
|
||||
// Format a user friendly error message
|
||||
const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
|
||||
throw new ValidationError(`Invalid request: ${errorMessage}`);
|
||||
}
|
||||
|
||||
return baseApiService.delete(`/api/v1/documents/${request.id}`, deleteDocumentResponse);
|
||||
};
|
||||
}
|
||||
|
||||
export const documentsApiService = new DocumentsApiService();
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import type { GetChatsRequest } from "@/contracts/types/chat.types";
|
||||
import type { GetDocumentsRequest } from "@/contracts/types/document.types";
|
||||
import type { GetPodcastsRequest } from "@/contracts/types/podcast.types";
|
||||
|
||||
export const cacheKeys = {
|
||||
|
|
@ -11,6 +12,14 @@ export const cacheKeys = {
|
|||
globalQueryParams: (queries: GetPodcastsRequest["queryParams"]) =>
|
||||
["podcasts", ...(queries ? Object.values(queries) : [])] as const,
|
||||
},
|
||||
documents: {
|
||||
globalQueryParams: (queries: GetDocumentsRequest["queryParams"]) =>
|
||||
["documents", ...(queries ? Object.values(queries) : [])] as const,
|
||||
withQueryParams :(queries: GetDocumentsRequest["queryParams"]) => ["documents-with-queries", ...(queries ? Object.values(queries) : [])] as const,
|
||||
document: (documentId: string) => ["document", documentId] as const,
|
||||
typeCounts: (searchSpaceId?: string) => ["documents", "type-counts", searchSpaceId] as const,
|
||||
byChunk: (chunkId: string) => ["documents", "by-chunk", chunkId] as const,
|
||||
},
|
||||
auth: {
|
||||
user: ["auth", "user"] as const,
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue