mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-29 19:35:20 +02:00
refactor: streamline document upload limits and enhance handling of mentioned documents
- Updated maximum file size limit to 500 MB per file. - Removed restrictions on the number of files per upload and total upload size. - Enhanced handling of user-mentioning documents in the knowledge base search middleware. - Improved document reading and processing logic to accommodate new features and optimizations.
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
6727266107
commit
62e698d8aa
33 changed files with 2889 additions and 2443 deletions
|
|
@ -1,12 +1,13 @@
|
|||
"use client";
|
||||
|
||||
import { useAtomValue, useSetAtom } from "jotai";
|
||||
import { AlertCircle, XIcon } from "lucide-react";
|
||||
import { AlertCircle, Download, FileText, Loader2, XIcon } from "lucide-react";
|
||||
import dynamic from "next/dynamic";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { closeEditorPanelAtom, editorPanelAtom } from "@/atoms/editor/editor-panel.atom";
|
||||
import { MarkdownViewer } from "@/components/markdown-viewer";
|
||||
import { Alert, AlertDescription } from "@/components/ui/alert";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Drawer, DrawerContent, DrawerHandle, DrawerTitle } from "@/components/ui/drawer";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
|
|
@ -18,11 +19,16 @@ const PlateEditor = dynamic(
|
|||
{ ssr: false, loading: () => <Skeleton className="h-64 w-full" /> }
|
||||
);
|
||||
|
||||
const LARGE_DOCUMENT_THRESHOLD = 2 * 1024 * 1024; // 2MB
|
||||
|
||||
interface EditorContent {
|
||||
document_id: number;
|
||||
title: string;
|
||||
document_type?: string;
|
||||
source_markdown: string;
|
||||
content_size_bytes?: number;
|
||||
chunk_count?: number;
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]);
|
||||
|
|
@ -62,6 +68,7 @@ export function EditorPanelContent({
|
|||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [downloading, setDownloading] = useState(false);
|
||||
|
||||
const [editedMarkdown, setEditedMarkdown] = useState<string | null>(null);
|
||||
const markdownRef = useRef<string>("");
|
||||
|
|
@ -69,6 +76,8 @@ export function EditorPanelContent({
|
|||
const changeCountRef = useRef(0);
|
||||
const [displayTitle, setDisplayTitle] = useState(title || "Untitled");
|
||||
|
||||
const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD;
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
setIsLoading(true);
|
||||
|
|
@ -86,10 +95,12 @@ export function EditorPanelContent({
|
|||
}
|
||||
|
||||
try {
|
||||
const response = await authenticatedFetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`,
|
||||
{ method: "GET" }
|
||||
const url = new URL(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`
|
||||
);
|
||||
url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD));
|
||||
|
||||
const response = await authenticatedFetch(url.toString(), { method: "GET" });
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
|
|
@ -175,7 +186,7 @@ export function EditorPanelContent({
|
|||
}, [documentId, searchSpaceId]);
|
||||
|
||||
const isEditableType = editorDoc
|
||||
? EDITABLE_DOCUMENT_TYPES.has(editorDoc.document_type ?? "")
|
||||
? EDITABLE_DOCUMENT_TYPES.has(editorDoc.document_type ?? "") && !isLargeDocument
|
||||
: false;
|
||||
|
||||
return (
|
||||
|
|
@ -206,6 +217,57 @@ export function EditorPanelContent({
|
|||
<p className="text-sm text-red-500 mt-1">{error || "An unknown error occurred"}</p>
|
||||
</div>
|
||||
</div>
|
||||
) : isLargeDocument ? (
|
||||
<div className="h-full overflow-y-auto px-5 py-4">
|
||||
<Alert className="mb-4">
|
||||
<FileText className="size-4" />
|
||||
<AlertDescription className="flex items-center justify-between gap-4">
|
||||
<span>
|
||||
This document is too large for the editor ({Math.round((editorDoc.content_size_bytes ?? 0) / 1024 / 1024)}MB, {editorDoc.chunk_count ?? 0} chunks). Showing a preview below.
|
||||
</span>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="shrink-0 gap-1.5"
|
||||
disabled={downloading}
|
||||
onClick={async () => {
|
||||
setDownloading(true);
|
||||
try {
|
||||
const response = await authenticatedFetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/download-markdown`,
|
||||
{ method: "GET" }
|
||||
);
|
||||
if (!response.ok) throw new Error("Download failed");
|
||||
const blob = await response.blob();
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
const disposition = response.headers.get("content-disposition");
|
||||
const match = disposition?.match(/filename="(.+)"/);
|
||||
a.download = match?.[1] ?? `${editorDoc.title || "document"}.md`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
URL.revokeObjectURL(url);
|
||||
toast.success("Download started");
|
||||
} catch {
|
||||
toast.error("Failed to download document");
|
||||
} finally {
|
||||
setDownloading(false);
|
||||
}
|
||||
}}
|
||||
>
|
||||
{downloading ? (
|
||||
<Loader2 className="size-3.5 animate-spin" />
|
||||
) : (
|
||||
<Download className="size-3.5" />
|
||||
)}
|
||||
{downloading ? "Preparing..." : "Download .md"}
|
||||
</Button>
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
<MarkdownViewer content={editorDoc.source_markdown} />
|
||||
</div>
|
||||
) : isEditableType ? (
|
||||
<PlateEditor
|
||||
key={documentId}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,24 @@
|
|||
"use client";
|
||||
|
||||
import { AlertCircle, Pencil } from "lucide-react";
|
||||
import { AlertCircle, Download, FileText, Loader2, Pencil } from "lucide-react";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { PlateEditor } from "@/components/editor/plate-editor";
|
||||
import { MarkdownViewer } from "@/components/markdown-viewer";
|
||||
import { Alert, AlertDescription } from "@/components/ui/alert";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { authenticatedFetch, getBearerToken, redirectToLogin } from "@/lib/auth-utils";
|
||||
|
||||
const LARGE_DOCUMENT_THRESHOLD = 2 * 1024 * 1024; // 2MB
|
||||
|
||||
interface DocumentContent {
|
||||
document_id: number;
|
||||
title: string;
|
||||
document_type?: string;
|
||||
source_markdown: string;
|
||||
content_size_bytes?: number;
|
||||
chunk_count?: number;
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
function DocumentSkeleton() {
|
||||
|
|
@ -49,11 +55,14 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
|
|||
const [error, setError] = useState<string | null>(null);
|
||||
const [isEditing, setIsEditing] = useState(false);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [downloading, setDownloading] = useState(false);
|
||||
const [editedMarkdown, setEditedMarkdown] = useState<string | null>(null);
|
||||
const markdownRef = useRef<string>("");
|
||||
const initialLoadDone = useRef(false);
|
||||
const changeCountRef = useRef(0);
|
||||
|
||||
const isLargeDocument = (doc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD;
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
setIsLoading(true);
|
||||
|
|
@ -72,10 +81,12 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
|
|||
}
|
||||
|
||||
try {
|
||||
const response = await authenticatedFetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`,
|
||||
{ method: "GET" }
|
||||
const url = new URL(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`
|
||||
);
|
||||
url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD));
|
||||
|
||||
const response = await authenticatedFetch(url.toString(), { method: "GET" });
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
|
|
@ -173,9 +184,9 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
|
|||
);
|
||||
}
|
||||
|
||||
const isEditable = EDITABLE_DOCUMENT_TYPES.has(doc.document_type ?? "");
|
||||
const isEditable = EDITABLE_DOCUMENT_TYPES.has(doc.document_type ?? "") && !isLargeDocument;
|
||||
|
||||
if (isEditing) {
|
||||
if (isEditing && !isLargeDocument) {
|
||||
return (
|
||||
<div className="flex flex-col h-full overflow-hidden">
|
||||
<div className="flex items-center justify-between px-6 py-3 border-b shrink-0">
|
||||
|
|
@ -236,7 +247,60 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen
|
|||
</div>
|
||||
<div className="flex-1 overflow-auto">
|
||||
<div className="max-w-4xl mx-auto px-6 py-6">
|
||||
<MarkdownViewer content={doc.source_markdown} />
|
||||
{isLargeDocument ? (
|
||||
<>
|
||||
<Alert className="mb-4">
|
||||
<FileText className="size-4" />
|
||||
<AlertDescription className="flex items-center justify-between gap-4">
|
||||
<span>
|
||||
This document is too large for the editor ({Math.round((doc.content_size_bytes ?? 0) / 1024 / 1024)}MB, {doc.chunk_count ?? 0} chunks). Showing a preview below.
|
||||
</span>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="shrink-0 gap-1.5"
|
||||
disabled={downloading}
|
||||
onClick={async () => {
|
||||
setDownloading(true);
|
||||
try {
|
||||
const response = await authenticatedFetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/download-markdown`,
|
||||
{ method: "GET" }
|
||||
);
|
||||
if (!response.ok) throw new Error("Download failed");
|
||||
const blob = await response.blob();
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
const disposition = response.headers.get("content-disposition");
|
||||
const match = disposition?.match(/filename="(.+)"/);
|
||||
a.download = match?.[1] ?? `${doc.title || "document"}.md`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
URL.revokeObjectURL(url);
|
||||
toast.success("Download started");
|
||||
} catch {
|
||||
toast.error("Failed to download document");
|
||||
} finally {
|
||||
setDownloading(false);
|
||||
}
|
||||
}}
|
||||
>
|
||||
{downloading ? (
|
||||
<Loader2 className="size-3.5 animate-spin" />
|
||||
) : (
|
||||
<Download className="size-3.5" />
|
||||
)}
|
||||
{downloading ? "Preparing..." : "Download .md"}
|
||||
</Button>
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
<MarkdownViewer content={doc.source_markdown} />
|
||||
</>
|
||||
) : (
|
||||
<MarkdownViewer content={doc.source_markdown} />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ const math = createMathPlugin({
|
|||
interface MarkdownViewerProps {
|
||||
content: string;
|
||||
className?: string;
|
||||
maxLength?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -79,8 +80,10 @@ function convertLatexDelimiters(content: string): string {
|
|||
return content;
|
||||
}
|
||||
|
||||
export function MarkdownViewer({ content, className }: MarkdownViewerProps) {
|
||||
const processedContent = convertLatexDelimiters(stripOuterMarkdownFence(content));
|
||||
export function MarkdownViewer({ content, className, maxLength }: MarkdownViewerProps) {
|
||||
const isTruncated = maxLength != null && content.length > maxLength;
|
||||
const displayContent = isTruncated ? content.slice(0, maxLength) : content;
|
||||
const processedContent = convertLatexDelimiters(stripOuterMarkdownFence(displayContent));
|
||||
const components: StreamdownProps["components"] = {
|
||||
p: ({ children, ...props }) => (
|
||||
<p className="my-2" {...props}>
|
||||
|
|
@ -171,6 +174,11 @@ export function MarkdownViewer({ content, className }: MarkdownViewerProps) {
|
|||
>
|
||||
{processedContent}
|
||||
</Streamdown>
|
||||
{isTruncated && (
|
||||
<p className="mt-4 text-sm text-muted-foreground italic">
|
||||
Content truncated ({Math.round(content.length / 1024)}KB total). Showing first {Math.round(maxLength / 1024)}KB.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"use client";
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { BookOpen, ChevronDown, ExternalLink, FileText, Hash, Sparkles, X } from "lucide-react";
|
||||
import { BookOpen, ChevronDown, ChevronUp, ExternalLink, FileText, Hash, Loader2, Sparkles, X } from "lucide-react";
|
||||
import { AnimatePresence, motion, useReducedMotion } from "motion/react";
|
||||
import { useTranslations } from "next-intl";
|
||||
import type React from "react";
|
||||
|
|
@ -10,7 +10,6 @@ import { createPortal } from "react-dom";
|
|||
import { MarkdownViewer } from "@/components/markdown-viewer";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
|
||||
import { ScrollArea } from "@/components/ui/scroll-area";
|
||||
import { Spinner } from "@/components/ui/spinner";
|
||||
import type {
|
||||
|
|
@ -48,7 +47,8 @@ const formatDocumentType = (type: string) => {
|
|||
// which break auto-scroll functionality
|
||||
interface ChunkCardProps {
|
||||
chunk: { id: number; content: string };
|
||||
index: number;
|
||||
localIndex: number;
|
||||
chunkNumber: number;
|
||||
totalChunks: number;
|
||||
isCited: boolean;
|
||||
isActive: boolean;
|
||||
|
|
@ -56,11 +56,11 @@ interface ChunkCardProps {
|
|||
}
|
||||
|
||||
const ChunkCard = memo(
|
||||
forwardRef<HTMLDivElement, ChunkCardProps>(({ chunk, index, totalChunks, isCited }, ref) => {
|
||||
forwardRef<HTMLDivElement, ChunkCardProps>(({ chunk, localIndex, chunkNumber, totalChunks, isCited }, ref) => {
|
||||
return (
|
||||
<div
|
||||
ref={ref}
|
||||
data-chunk-index={index}
|
||||
data-chunk-index={localIndex}
|
||||
className={cn(
|
||||
"group relative rounded-2xl border-2 transition-all duration-300",
|
||||
isCited
|
||||
|
|
@ -68,10 +68,8 @@ const ChunkCard = memo(
|
|||
: "bg-card border-border/50 hover:border-border hover:shadow-md"
|
||||
)}
|
||||
>
|
||||
{/* Cited indicator glow effect */}
|
||||
{isCited && <div className="absolute inset-0 rounded-2xl bg-primary/5 blur-xl -z-10" />}
|
||||
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between px-5 py-4 border-b border-border/50">
|
||||
<div className="flex items-center gap-3">
|
||||
<div
|
||||
|
|
@ -82,9 +80,9 @@ const ChunkCard = memo(
|
|||
: "bg-muted text-muted-foreground group-hover:bg-muted/80"
|
||||
)}
|
||||
>
|
||||
{index + 1}
|
||||
{chunkNumber}
|
||||
</div>
|
||||
<span className="text-sm text-muted-foreground">of {totalChunks} chunks</span>
|
||||
<span className="text-sm text-muted-foreground">Chunk {chunkNumber} of {totalChunks}</span>
|
||||
</div>
|
||||
{isCited && (
|
||||
<Badge variant="default" className="gap-1.5 px-3 py-1">
|
||||
|
|
@ -94,9 +92,8 @@ const ChunkCard = memo(
|
|||
)}
|
||||
</div>
|
||||
|
||||
{/* Content */}
|
||||
<div className="p-5 overflow-hidden">
|
||||
<MarkdownViewer content={chunk.content} />
|
||||
<MarkdownViewer content={chunk.content} maxLength={100_000} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
|
@ -118,7 +115,6 @@ export function SourceDetailPanel({
|
|||
const t = useTranslations("dashboard");
|
||||
const scrollAreaRef = useRef<HTMLDivElement>(null);
|
||||
const hasScrolledRef = useRef(false); // Use ref to avoid stale closures
|
||||
const [summaryOpen, setSummaryOpen] = useState(false);
|
||||
const [activeChunkIndex, setActiveChunkIndex] = useState<number | null>(null);
|
||||
const [mounted, setMounted] = useState(false);
|
||||
const [_hasScrolledToCited, setHasScrolledToCited] = useState(false);
|
||||
|
|
@ -140,20 +136,88 @@ export function SourceDetailPanel({
|
|||
if (isDocsChunk) {
|
||||
return documentsApiService.getSurfsenseDocByChunk(chunkId);
|
||||
}
|
||||
return documentsApiService.getDocumentByChunk({ chunk_id: chunkId });
|
||||
return documentsApiService.getDocumentByChunk({ chunk_id: chunkId, chunk_window: 5 });
|
||||
},
|
||||
enabled: !!chunkId && open,
|
||||
staleTime: 5 * 60 * 1000,
|
||||
});
|
||||
|
||||
const totalChunks = (documentData && "total_chunks" in documentData)
|
||||
? (documentData.total_chunks ?? documentData.chunks.length)
|
||||
: (documentData?.chunks?.length ?? 0);
|
||||
const [beforeChunks, setBeforeChunks] = useState<Array<{ id: number; content: string; created_at: string }>>([]);
|
||||
const [afterChunks, setAfterChunks] = useState<Array<{ id: number; content: string; created_at: string }>>([]);
|
||||
const [loadingBefore, setLoadingBefore] = useState(false);
|
||||
const [loadingAfter, setLoadingAfter] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
setBeforeChunks([]);
|
||||
setAfterChunks([]);
|
||||
}, [chunkId, open]);
|
||||
|
||||
const chunkStartIndex = (documentData && "chunk_start_index" in documentData)
|
||||
? (documentData.chunk_start_index ?? 0) : 0;
|
||||
const initialChunks = documentData?.chunks ?? [];
|
||||
const allChunks = [...beforeChunks, ...initialChunks, ...afterChunks];
|
||||
const absoluteStart = chunkStartIndex - beforeChunks.length;
|
||||
const absoluteEnd = chunkStartIndex + initialChunks.length + afterChunks.length;
|
||||
const canLoadBefore = absoluteStart > 0;
|
||||
const canLoadAfter = absoluteEnd < totalChunks;
|
||||
|
||||
const EXPAND_SIZE = 10;
|
||||
|
||||
const loadBefore = useCallback(async () => {
|
||||
if (!documentData || !("search_space_id" in documentData) || !canLoadBefore) return;
|
||||
setLoadingBefore(true);
|
||||
try {
|
||||
const count = Math.min(EXPAND_SIZE, absoluteStart);
|
||||
const result = await documentsApiService.getDocumentChunks({
|
||||
document_id: documentData.id,
|
||||
page: 0,
|
||||
page_size: count,
|
||||
start_offset: absoluteStart - count,
|
||||
});
|
||||
const existingIds = new Set(allChunks.map(c => c.id));
|
||||
const newChunks = result.items
|
||||
.filter(c => !existingIds.has(c.id))
|
||||
.map(c => ({ id: c.id, content: c.content, created_at: c.created_at }));
|
||||
setBeforeChunks(prev => [...newChunks, ...prev]);
|
||||
} catch (err) {
|
||||
console.error("Failed to load earlier chunks:", err);
|
||||
} finally {
|
||||
setLoadingBefore(false);
|
||||
}
|
||||
}, [documentData, absoluteStart, canLoadBefore, allChunks]);
|
||||
|
||||
const loadAfter = useCallback(async () => {
|
||||
if (!documentData || !("search_space_id" in documentData) || !canLoadAfter) return;
|
||||
setLoadingAfter(true);
|
||||
try {
|
||||
const result = await documentsApiService.getDocumentChunks({
|
||||
document_id: documentData.id,
|
||||
page: 0,
|
||||
page_size: EXPAND_SIZE,
|
||||
start_offset: absoluteEnd,
|
||||
});
|
||||
const existingIds = new Set(allChunks.map(c => c.id));
|
||||
const newChunks = result.items
|
||||
.filter(c => !existingIds.has(c.id))
|
||||
.map(c => ({ id: c.id, content: c.content, created_at: c.created_at }));
|
||||
setAfterChunks(prev => [...prev, ...newChunks]);
|
||||
} catch (err) {
|
||||
console.error("Failed to load later chunks:", err);
|
||||
} finally {
|
||||
setLoadingAfter(false);
|
||||
}
|
||||
}, [documentData, absoluteEnd, canLoadAfter, allChunks]);
|
||||
|
||||
const isDirectRenderSource =
|
||||
sourceType === "TAVILY_API" ||
|
||||
sourceType === "LINKUP_API" ||
|
||||
sourceType === "SEARXNG_API" ||
|
||||
sourceType === "BAIDU_SEARCH_API";
|
||||
|
||||
// Find cited chunk index
|
||||
const citedChunkIndex = documentData?.chunks?.findIndex((chunk) => chunk.id === chunkId) ?? -1;
|
||||
const citedChunkIndex = allChunks.findIndex((chunk) => chunk.id === chunkId);
|
||||
|
||||
// Simple scroll function that scrolls to a chunk by index
|
||||
const scrollToChunkByIndex = useCallback(
|
||||
|
|
@ -336,12 +400,12 @@ export function SourceDetailPanel({
|
|||
{documentData && "document_type" in documentData
|
||||
? formatDocumentType(documentData.document_type)
|
||||
: sourceType && formatDocumentType(sourceType)}
|
||||
{documentData?.chunks && (
|
||||
<span className="ml-2">
|
||||
• {documentData.chunks.length} chunk
|
||||
{documentData.chunks.length !== 1 ? "s" : ""}
|
||||
</span>
|
||||
)}
|
||||
{totalChunks > 0 && (
|
||||
<span className="ml-2">
|
||||
• {totalChunks} chunk{totalChunks !== 1 ? "s" : ""}
|
||||
{allChunks.length < totalChunks && ` (showing ${allChunks.length})`}
|
||||
</span>
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-3 shrink-0">
|
||||
|
|
@ -450,7 +514,7 @@ export function SourceDetailPanel({
|
|||
{!isDirectRenderSource && documentData && (
|
||||
<div className="flex-1 flex overflow-hidden">
|
||||
{/* Chunk Navigation Sidebar */}
|
||||
{documentData.chunks.length > 1 && (
|
||||
{allChunks.length > 1 && (
|
||||
<motion.div
|
||||
initial={{ opacity: 0, x: -20 }}
|
||||
animate={{ opacity: 1, x: 0 }}
|
||||
|
|
@ -459,7 +523,8 @@ export function SourceDetailPanel({
|
|||
>
|
||||
<ScrollArea className="flex-1 h-full">
|
||||
<div className="p-2 pt-3 flex flex-col gap-1.5">
|
||||
{documentData.chunks.map((chunk, idx) => {
|
||||
{allChunks.map((chunk, idx) => {
|
||||
const absNum = absoluteStart + idx + 1;
|
||||
const isCited = chunk.id === chunkId;
|
||||
const isActive = activeChunkIndex === idx;
|
||||
return (
|
||||
|
|
@ -478,9 +543,9 @@ export function SourceDetailPanel({
|
|||
? "bg-muted text-foreground"
|
||||
: "bg-muted/50 text-muted-foreground hover:bg-muted hover:text-foreground"
|
||||
)}
|
||||
title={isCited ? `Chunk ${idx + 1} (Cited)` : `Chunk ${idx + 1}`}
|
||||
title={isCited ? `Chunk ${absNum} (Cited)` : `Chunk ${absNum}`}
|
||||
>
|
||||
{idx + 1}
|
||||
{absNum}
|
||||
{isCited && (
|
||||
<span className="absolute -top-1.5 -right-1.5 flex items-center justify-center w-4 h-4 bg-primary rounded-full border-2 border-background shadow-sm">
|
||||
<Sparkles className="h-2.5 w-2.5 text-primary-foreground" />
|
||||
|
|
@ -524,44 +589,11 @@ export function SourceDetailPanel({
|
|||
</motion.div>
|
||||
)}
|
||||
|
||||
{/* Summary Collapsible */}
|
||||
{documentData.content && (
|
||||
<motion.div
|
||||
initial={{ opacity: 0, y: 10 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
transition={{ delay: 0.15 }}
|
||||
>
|
||||
<Collapsible open={summaryOpen} onOpenChange={setSummaryOpen}>
|
||||
<CollapsibleTrigger className="w-full flex items-center justify-between p-5 rounded-2xl bg-linear-to-r from-muted/50 to-muted/30 border hover:from-muted/70 hover:to-muted/50 transition-all duration-200">
|
||||
<span className="font-semibold flex items-center gap-2">
|
||||
<BookOpen className="h-4 w-4" />
|
||||
Document Summary
|
||||
</span>
|
||||
<motion.div
|
||||
animate={{ rotate: summaryOpen ? 180 : 0 }}
|
||||
transition={{ duration: 0.2 }}
|
||||
>
|
||||
<ChevronDown className="h-5 w-5 text-muted-foreground" />
|
||||
</motion.div>
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent>
|
||||
<motion.div
|
||||
initial={{ opacity: 0 }}
|
||||
animate={{ opacity: 1 }}
|
||||
className="mt-3 p-5 bg-muted/20 rounded-2xl border"
|
||||
>
|
||||
<MarkdownViewer content={documentData.content} />
|
||||
</motion.div>
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
</motion.div>
|
||||
)}
|
||||
|
||||
{/* Chunks Header */}
|
||||
<div className="flex items-center justify-between pt-4">
|
||||
<div className="flex items-center justify-between pt-2">
|
||||
<h3 className="text-sm font-semibold text-muted-foreground uppercase tracking-wider flex items-center gap-2">
|
||||
<Hash className="h-4 w-4" />
|
||||
Content Chunks
|
||||
Chunks {absoluteStart + 1}–{absoluteEnd} of {totalChunks}
|
||||
</h3>
|
||||
{citedChunkIndex !== -1 && (
|
||||
<Button
|
||||
|
|
@ -576,24 +608,70 @@ export function SourceDetailPanel({
|
|||
)}
|
||||
</div>
|
||||
|
||||
{/* Load Earlier */}
|
||||
{canLoadBefore && (
|
||||
<div className="flex items-center justify-center">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={loadBefore}
|
||||
disabled={loadingBefore}
|
||||
className="gap-2"
|
||||
>
|
||||
{loadingBefore ? (
|
||||
<Loader2 className="h-3.5 w-3.5 animate-spin" />
|
||||
) : (
|
||||
<ChevronUp className="h-3.5 w-3.5" />
|
||||
)}
|
||||
{loadingBefore
|
||||
? "Loading..."
|
||||
: `Load ${Math.min(EXPAND_SIZE, absoluteStart)} earlier chunks`}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Chunks */}
|
||||
<div className="space-y-4">
|
||||
{documentData.chunks.map((chunk, idx) => {
|
||||
{allChunks.map((chunk, idx) => {
|
||||
const isCited = chunk.id === chunkId;
|
||||
const chunkNumber = absoluteStart + idx + 1;
|
||||
return (
|
||||
<ChunkCard
|
||||
key={chunk.id}
|
||||
ref={isCited ? citedChunkRefCallback : undefined}
|
||||
chunk={chunk}
|
||||
index={idx}
|
||||
totalChunks={documentData.chunks.length}
|
||||
localIndex={idx}
|
||||
chunkNumber={chunkNumber}
|
||||
totalChunks={totalChunks}
|
||||
isCited={isCited}
|
||||
isActive={activeChunkIndex === idx}
|
||||
disableLayoutAnimation={documentData.chunks.length > 30}
|
||||
disableLayoutAnimation={allChunks.length > 30}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{/* Load Later */}
|
||||
{canLoadAfter && (
|
||||
<div className="flex items-center justify-center py-3">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={loadAfter}
|
||||
disabled={loadingAfter}
|
||||
className="gap-2"
|
||||
>
|
||||
{loadingAfter ? (
|
||||
<Loader2 className="h-3.5 w-3.5 animate-spin" />
|
||||
) : (
|
||||
<ChevronDown className="h-3.5 w-3.5" />
|
||||
)}
|
||||
{loadingAfter
|
||||
? "Loading..."
|
||||
: `Load ${Math.min(EXPAND_SIZE, totalChunks - absoluteEnd)} later chunks`}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</ScrollArea>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
"use client";
|
||||
|
||||
import { useAtom } from "jotai";
|
||||
import { CheckCircle2, FileType, Info, Upload, X } from "lucide-react";
|
||||
import { CheckCircle2, FileType, FolderOpen, Info, Upload, X } from "lucide-react";
|
||||
|
||||
import { useTranslations } from "next-intl";
|
||||
import { useCallback, useMemo, useRef, useState } from "react";
|
||||
import { type ChangeEvent, useCallback, useMemo, useRef, useState } from "react";
|
||||
import { useDropzone } from "react-dropzone";
|
||||
import { toast } from "sonner";
|
||||
import { uploadDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
|
||||
|
|
@ -51,6 +51,7 @@ const commonTypes = {
|
|||
"application/vnd.openxmlformats-officedocument.presentationml.presentation": [".pptx"],
|
||||
"text/html": [".html", ".htm"],
|
||||
"text/csv": [".csv"],
|
||||
"text/tab-separated-values": [".tsv"],
|
||||
"image/jpeg": [".jpg", ".jpeg"],
|
||||
"image/png": [".png"],
|
||||
"image/bmp": [".bmp"],
|
||||
|
|
@ -76,7 +77,6 @@ const FILE_TYPE_CONFIG: Record<string, Record<string, string[]>> = {
|
|||
"application/rtf": [".rtf"],
|
||||
"application/xml": [".xml"],
|
||||
"application/epub+zip": [".epub"],
|
||||
"text/tab-separated-values": [".tsv"],
|
||||
"text/html": [".html", ".htm", ".web"],
|
||||
"image/gif": [".gif"],
|
||||
"image/svg+xml": [".svg"],
|
||||
|
|
@ -102,7 +102,6 @@ const FILE_TYPE_CONFIG: Record<string, Record<string, string[]>> = {
|
|||
"application/vnd.ms-powerpoint": [".ppt"],
|
||||
"text/x-rst": [".rst"],
|
||||
"application/rtf": [".rtf"],
|
||||
"text/tab-separated-values": [".tsv"],
|
||||
"application/vnd.ms-excel": [".xls"],
|
||||
"application/xml": [".xml"],
|
||||
...audioFileTypes,
|
||||
|
|
@ -116,10 +115,8 @@ interface FileWithId {
|
|||
|
||||
const cardClass = "border border-border bg-slate-400/5 dark:bg-white/5";
|
||||
|
||||
// Upload limits — files are sent in batches of 5 to avoid proxy timeouts
|
||||
const MAX_FILES = 50;
|
||||
const MAX_TOTAL_SIZE_MB = 200;
|
||||
const MAX_TOTAL_SIZE_BYTES = MAX_TOTAL_SIZE_MB * 1024 * 1024;
|
||||
const MAX_FILE_SIZE_MB = 500;
|
||||
const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024;
|
||||
|
||||
export function DocumentUploadTab({
|
||||
searchSpaceId,
|
||||
|
|
@ -134,6 +131,7 @@ export function DocumentUploadTab({
|
|||
const [uploadDocumentMutation] = useAtom(uploadDocumentMutationAtom);
|
||||
const { mutate: uploadDocuments, isPending: isUploading } = uploadDocumentMutation;
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
const folderInputRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const acceptedFileTypes = useMemo(() => {
|
||||
const etlService = process.env.NEXT_PUBLIC_ETL_SERVICE;
|
||||
|
|
@ -145,49 +143,76 @@ export function DocumentUploadTab({
|
|||
[acceptedFileTypes]
|
||||
);
|
||||
|
||||
const onDrop = useCallback(
|
||||
(acceptedFiles: File[]) => {
|
||||
const supportedExtensionsSet = useMemo(
|
||||
() => new Set(supportedExtensions.map((ext) => ext.toLowerCase())),
|
||||
[supportedExtensions]
|
||||
);
|
||||
|
||||
const addFiles = useCallback(
|
||||
(incoming: File[]) => {
|
||||
const oversized = incoming.filter((f) => f.size > MAX_FILE_SIZE_BYTES);
|
||||
if (oversized.length > 0) {
|
||||
toast.error(t("file_too_large"), {
|
||||
description: t("file_too_large_desc", {
|
||||
name: oversized[0].name,
|
||||
maxMB: MAX_FILE_SIZE_MB,
|
||||
}),
|
||||
});
|
||||
}
|
||||
const valid = incoming.filter((f) => f.size <= MAX_FILE_SIZE_BYTES);
|
||||
if (valid.length === 0) return;
|
||||
|
||||
setFiles((prev) => {
|
||||
const newEntries = acceptedFiles.map((f) => ({
|
||||
const newEntries = valid.map((f) => ({
|
||||
id: crypto.randomUUID?.() ?? `file-${Date.now()}-${Math.random().toString(36)}`,
|
||||
file: f,
|
||||
}));
|
||||
const newFiles = [...prev, ...newEntries];
|
||||
|
||||
if (newFiles.length > MAX_FILES) {
|
||||
toast.error(t("max_files_exceeded"), {
|
||||
description: t("max_files_exceeded_desc", { max: MAX_FILES }),
|
||||
});
|
||||
return prev;
|
||||
}
|
||||
|
||||
const newTotalSize = newFiles.reduce((sum, entry) => sum + entry.file.size, 0);
|
||||
if (newTotalSize > MAX_TOTAL_SIZE_BYTES) {
|
||||
toast.error(t("max_size_exceeded"), {
|
||||
description: t("max_size_exceeded_desc", { max: MAX_TOTAL_SIZE_MB }),
|
||||
});
|
||||
return prev;
|
||||
}
|
||||
|
||||
return newFiles;
|
||||
return [...prev, ...newEntries];
|
||||
});
|
||||
},
|
||||
[t]
|
||||
);
|
||||
|
||||
const onDrop = useCallback(
|
||||
(acceptedFiles: File[]) => {
|
||||
addFiles(acceptedFiles);
|
||||
},
|
||||
[addFiles]
|
||||
);
|
||||
|
||||
const { getRootProps, getInputProps, isDragActive } = useDropzone({
|
||||
onDrop,
|
||||
accept: acceptedFileTypes,
|
||||
maxSize: 50 * 1024 * 1024, // 50MB per file
|
||||
maxSize: MAX_FILE_SIZE_BYTES,
|
||||
noClick: false,
|
||||
disabled: files.length >= MAX_FILES,
|
||||
});
|
||||
|
||||
// Handle file input click to prevent event bubbling that might reopen dialog
|
||||
const handleFileInputClick = useCallback((e: React.MouseEvent<HTMLInputElement>) => {
|
||||
e.stopPropagation();
|
||||
}, []);
|
||||
|
||||
const handleFolderChange = useCallback(
|
||||
(e: ChangeEvent<HTMLInputElement>) => {
|
||||
const fileList = e.target.files;
|
||||
if (!fileList || fileList.length === 0) return;
|
||||
|
||||
const folderFiles = Array.from(fileList).filter((f) => {
|
||||
const ext = f.name.includes(".") ? `.${f.name.split(".").pop()?.toLowerCase()}` : "";
|
||||
return ext !== "" && supportedExtensionsSet.has(ext);
|
||||
});
|
||||
|
||||
if (folderFiles.length === 0) {
|
||||
toast.error(t("no_supported_files_in_folder"));
|
||||
e.target.value = "";
|
||||
return;
|
||||
}
|
||||
|
||||
addFiles(folderFiles);
|
||||
e.target.value = "";
|
||||
},
|
||||
[addFiles, supportedExtensionsSet, t]
|
||||
);
|
||||
|
||||
const formatFileSize = (bytes: number) => {
|
||||
if (bytes === 0) return "0 Bytes";
|
||||
const k = 1024;
|
||||
|
|
@ -198,15 +223,6 @@ export function DocumentUploadTab({
|
|||
|
||||
const totalFileSize = files.reduce((total, entry) => total + entry.file.size, 0);
|
||||
|
||||
// Check if limits are reached
|
||||
const isFileCountLimitReached = files.length >= MAX_FILES;
|
||||
const isSizeLimitReached = totalFileSize >= MAX_TOTAL_SIZE_BYTES;
|
||||
const remainingFiles = MAX_FILES - files.length;
|
||||
const remainingSizeMB = Math.max(
|
||||
0,
|
||||
(MAX_TOTAL_SIZE_BYTES - totalFileSize) / (1024 * 1024)
|
||||
).toFixed(1);
|
||||
|
||||
// Track accordion state changes
|
||||
const handleAccordionChange = useCallback(
|
||||
(value: string) => {
|
||||
|
|
@ -257,11 +273,21 @@ export function DocumentUploadTab({
|
|||
<Alert className="border border-border bg-slate-400/5 dark:bg-white/5">
|
||||
<Info className="h-4 w-4 shrink-0 mt-0.5" />
|
||||
<AlertDescription className="text-xs sm:text-sm leading-relaxed pt-0.5">
|
||||
{t("file_size_limit")}{" "}
|
||||
{t("upload_limits", { maxFiles: MAX_FILES, maxSizeMB: MAX_TOTAL_SIZE_MB })}
|
||||
{t("file_size_limit", { maxMB: MAX_FILE_SIZE_MB })}{" "}
|
||||
{t("upload_limits")}
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
|
||||
{/* Hidden folder input */}
|
||||
<input
|
||||
ref={folderInputRef}
|
||||
type="file"
|
||||
className="hidden"
|
||||
onChange={handleFolderChange}
|
||||
multiple
|
||||
{...({ webkitdirectory: "", directory: "" } as React.InputHTMLAttributes<HTMLInputElement>)}
|
||||
/>
|
||||
|
||||
<Card className={`relative overflow-hidden ${cardClass}`}>
|
||||
<div className="absolute inset-0 [mask-image:radial-gradient(ellipse_at_center,white,transparent)] opacity-30">
|
||||
<GridPattern />
|
||||
|
|
@ -269,11 +295,7 @@ export function DocumentUploadTab({
|
|||
<CardContent className="p-4 sm:p-10 relative z-10">
|
||||
<div
|
||||
{...getRootProps()}
|
||||
className={`flex flex-col items-center justify-center min-h-[200px] sm:min-h-[300px] border-2 border-dashed rounded-lg transition-colors ${
|
||||
isFileCountLimitReached || isSizeLimitReached
|
||||
? "border-destructive/50 bg-destructive/5 cursor-not-allowed"
|
||||
: "border-border hover:border-primary/50 cursor-pointer"
|
||||
}`}
|
||||
className="flex flex-col items-center justify-center min-h-[200px] sm:min-h-[300px] border-2 border-dashed rounded-lg transition-colors border-border hover:border-primary/50 cursor-pointer"
|
||||
>
|
||||
<input
|
||||
{...getInputProps()}
|
||||
|
|
@ -281,19 +303,7 @@ export function DocumentUploadTab({
|
|||
className="hidden"
|
||||
onClick={handleFileInputClick}
|
||||
/>
|
||||
{isFileCountLimitReached ? (
|
||||
<div className="flex flex-col items-center gap-2 sm:gap-4 text-center px-4">
|
||||
<Upload className="h-8 w-8 sm:h-12 sm:w-12 text-destructive/70" />
|
||||
<div>
|
||||
<p className="text-sm sm:text-lg font-medium text-destructive">
|
||||
{t("file_limit_reached")}
|
||||
</p>
|
||||
<p className="text-xs sm:text-sm text-muted-foreground mt-1">
|
||||
{t("file_limit_reached_desc", { max: MAX_FILES })}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
) : isDragActive ? (
|
||||
{isDragActive ? (
|
||||
<div className="flex flex-col items-center gap-2 sm:gap-4">
|
||||
<Upload className="h-8 w-8 sm:h-12 sm:w-12 text-primary" />
|
||||
<p className="text-sm sm:text-lg font-medium text-primary">{t("drop_files")}</p>
|
||||
|
|
@ -305,29 +315,35 @@ export function DocumentUploadTab({
|
|||
<p className="text-sm sm:text-lg font-medium">{t("drag_drop")}</p>
|
||||
<p className="text-xs sm:text-sm text-muted-foreground mt-1">{t("or_browse")}</p>
|
||||
</div>
|
||||
{files.length > 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{t("remaining_capacity", { files: remainingFiles, sizeMB: remainingSizeMB })}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{!isFileCountLimitReached && (
|
||||
<div className="mt-2 sm:mt-4">
|
||||
<Button
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
className="text-xs sm:text-sm"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
e.preventDefault();
|
||||
fileInputRef.current?.click();
|
||||
}}
|
||||
>
|
||||
{t("browse_files")}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
<div className="mt-2 sm:mt-4 flex gap-2">
|
||||
<Button
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
className="text-xs sm:text-sm"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
e.preventDefault();
|
||||
fileInputRef.current?.click();
|
||||
}}
|
||||
>
|
||||
{t("browse_files")}
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="text-xs sm:text-sm"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
e.preventDefault();
|
||||
folderInputRef.current?.click();
|
||||
}}
|
||||
>
|
||||
<FolderOpen className="h-4 w-4 mr-1.5" />
|
||||
{t("browse_folder")}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue