diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx index b92a9539b..b4be1df71 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx @@ -1,414 +1,16 @@ "use client"; -import { - IconBrandWindows, - IconBrandZoom, - IconChevronDown, - IconChevronRight, -} from "@tabler/icons-react"; -import { AnimatePresence, motion, type Variants } from "motion/react"; -import Link from "next/link"; -import { useParams } from "next/navigation"; -import { useTranslations } from "next-intl"; -import { useState } from "react"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { Card, CardContent, CardFooter, CardHeader } from "@/components/ui/card"; -import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; -import { EnumConnectorName } from "@/contracts/enums/connector"; -import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { useParams, useRouter } from "next/navigation"; +import { useEffect } from "react"; -// Define the Connector type -interface Connector { - id: string; - title: string; - description: string; - icon: React.ReactNode; - status: "available" | "coming-soon" | "connected"; -} - -interface ConnectorCategory { - id: string; - title: string; - connectors: Connector[]; -} - -// Define connector categories and their connectors -const connectorCategories: ConnectorCategory[] = [ - { - id: "search-engines", - title: "search_engines", - connectors: [ - { - id: "tavily-api", - title: "Tavily API", - description: "tavily_desc", - icon: getConnectorIcon(EnumConnectorName.TAVILY_API, "h-6 w-6"), - status: "available", - }, - { - id: "searxng", - title: "SearxNG", - description: "searxng_desc", - icon: getConnectorIcon(EnumConnectorName.SEARXNG_API, "h-6 w-6"), - status: "available", - }, - { - id: "linkup-api", - title: "Linkup API", - description: "linkup_desc", - icon: getConnectorIcon(EnumConnectorName.LINKUP_API, "h-6 w-6"), - status: "available", - }, - { - id: "elasticsearch-connector", - title: "Elasticsearch", - description: "elasticsearch_desc", - icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "baidu-search-api", - title: "Baidu Search", - description: "baidu_desc", - icon: getConnectorIcon(EnumConnectorName.BAIDU_SEARCH_API, "h-6 w-6"), - status: "available", - }, - ], - }, - { - id: "team-chats", - title: "team_chats", - connectors: [ - { - id: "slack-connector", - title: "Slack", - description: "slack_desc", - icon: getConnectorIcon(EnumConnectorName.SLACK_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "ms-teams", - title: "Microsoft Teams", - description: "teams_desc", - icon: , - status: "coming-soon", - }, - { - id: "discord-connector", - title: "Discord", - description: "discord_desc", - icon: getConnectorIcon(EnumConnectorName.DISCORD_CONNECTOR, "h-6 w-6"), - status: "available", - }, - ], - }, - { - id: "project-management", - title: "project_management", - connectors: [ - { - id: "linear-connector", - title: "Linear", - description: "linear_desc", - icon: getConnectorIcon(EnumConnectorName.LINEAR_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "jira-connector", - title: "Jira", - description: "jira_desc", - icon: getConnectorIcon(EnumConnectorName.JIRA_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "clickup-connector", - title: "ClickUp", - description: "clickup_desc", - icon: getConnectorIcon(EnumConnectorName.CLICKUP_CONNECTOR, "h-6 w-6"), - status: "available", - }, - ], - }, - { - id: "knowledge-bases", - title: "knowledge_bases", - connectors: [ - { - id: "notion-connector", - title: "Notion", - description: "notion_desc", - icon: getConnectorIcon(EnumConnectorName.NOTION_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "github-connector", - title: "GitHub", - description: "github_desc", - icon: getConnectorIcon(EnumConnectorName.GITHUB_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "confluence-connector", - title: "Confluence", - description: "confluence_desc", - icon: getConnectorIcon(EnumConnectorName.CONFLUENCE_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "airtable-connector", - title: "Airtable", - description: "airtable_desc", - icon: getConnectorIcon(EnumConnectorName.AIRTABLE_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "luma-connector", - title: "Luma", - description: "luma_desc", - icon: getConnectorIcon(EnumConnectorName.LUMA_CONNECTOR, "h-6 w-6"), - status: "available", - }, - ], - }, - { - id: "communication", - title: "communication", - connectors: [ - { - id: "google-calendar-connector", - title: "Google Calendar", - description: "calendar_desc", - icon: getConnectorIcon(EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "google-gmail-connector", - title: "Gmail", - description: "gmail_desc", - icon: getConnectorIcon(EnumConnectorName.GOOGLE_GMAIL_CONNECTOR, "h-6 w-6"), - status: "available", - }, - { - id: "zoom", - title: "Zoom", - description: "zoom_desc", - icon: , - status: "coming-soon", - }, - ], - }, -]; - -// Animation variants -const fadeIn = { - hidden: { opacity: 0 }, - visible: { opacity: 1, transition: { duration: 0.4 } }, -}; - -const staggerContainer = { - hidden: { opacity: 0 }, - visible: { - opacity: 1, - transition: { - staggerChildren: 0.1, - }, - }, -}; - -const cardVariants: Variants = { - hidden: { opacity: 0, y: 20 }, - visible: { - opacity: 1, - y: 0, - transition: { - type: "spring", - stiffness: 260, - damping: 20, - }, - }, - hover: { - scale: 1.02, - boxShadow: "0 10px 15px -3px rgba(0, 0, 0, 0.1), 0 4px 6px -2px rgba(0, 0, 0, 0.05)", - transition: { - type: "spring", - stiffness: 400, - damping: 10, - }, - }, -}; - -export default function ConnectorsPage() { - const t = useTranslations("add_connector"); +export default function AddConnectorRedirect() { const params = useParams(); - const searchSpaceId = params.search_space_id as string; - const [expandedCategories, setExpandedCategories] = useState([ - "search-engines", - "knowledge-bases", - "project-management", - "team-chats", - "communication", - ]); + const router = useRouter(); + const search_space_id = params.search_space_id as string; - const toggleCategory = (categoryId: string) => { - setExpandedCategories((prev) => - prev.includes(categoryId) ? prev.filter((id) => id !== categoryId) : [...prev, categoryId] - ); - }; + useEffect(() => { + router.replace(`/dashboard/${search_space_id}/sources/add?tab=connectors`); + }, [search_space_id, router]); - return ( -
- -

- {t("title")} -

-

{t("subtitle")}

-
- - - {connectorCategories.map((category) => ( - - toggleCategory(category.id)} - className="w-full" - > -
-

{t(category.title)}

- - - -
- - - - - {category.connectors.map((connector) => ( - - - -
- - {connector.icon} - -
-
-
-

{connector.title}

- {connector.status === "coming-soon" && ( - - {t("coming_soon")} - - )} - {connector.status === "connected" && ( - - {t("connected")} - - )} -
-
-
- - -

- {t(connector.description)} -

-
- - - {connector.status === "available" && ( - - - - )} - {connector.status === "coming-soon" && ( - - )} - {connector.status === "connected" && ( - - )} - -
-
- ))} -
-
-
-
-
- ))} -
-
- ); + return null; } diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/upload/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/upload/page.tsx index 6f6dfcc24..2c3220524 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/upload/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/upload/page.tsx @@ -1,580 +1,16 @@ "use client"; -import { CheckCircle2, FileType, Info, Tag, Upload, X } from "lucide-react"; -import { AnimatePresence, motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; -import { useTranslations } from "next-intl"; -import { useCallback, useState } from "react"; -import { useDropzone } from "react-dropzone"; -import { toast } from "sonner"; -import { Alert, AlertDescription } from "@/components/ui/alert"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { Progress } from "@/components/ui/progress"; -import { Separator } from "@/components/ui/separator"; +import { useEffect } from "react"; -// Grid pattern component inspired by Aceternity UI -function GridPattern() { - const columns = 41; - const rows = 11; - return ( -
- {Array.from({ length: rows }).map((_, row) => - Array.from({ length: columns }).map((_, col) => { - const index = row * columns + col; - return ( -
- ); - }) - )} -
- ); -} - -export default function FileUploader() { - const t = useTranslations("upload_documents"); +export default function UploadDocumentsRedirect() { const params = useParams(); + const router = useRouter(); const search_space_id = params.search_space_id as string; - const [files, setFiles] = useState([]); - const [isUploading, setIsUploading] = useState(false); - const [uploadProgress, setUploadProgress] = useState(0); - const router = useRouter(); + useEffect(() => { + router.replace(`/dashboard/${search_space_id}/sources/add?tab=documents`); + }, [search_space_id, router]); - // Audio files are always supported (using whisper) - const audioFileTypes = { - "audio/mpeg": [".mp3", ".mpeg", ".mpga"], - "audio/mp4": [".mp4", ".m4a"], - "audio/wav": [".wav"], - "audio/webm": [".webm"], - "text/markdown": [".md", ".markdown"], - "text/plain": [".txt"], - }; - - // Conditionally set accepted file types based on ETL service - const getAcceptedFileTypes = () => { - const etlService = process.env.NEXT_PUBLIC_ETL_SERVICE; - - if (etlService === "LLAMACLOUD") { - return { - // LlamaCloud supported file types - "application/pdf": [".pdf"], - "application/msword": [".doc"], - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [".docx"], - "application/vnd.ms-word.document.macroEnabled.12": [".docm"], - "application/msword-template": [".dot"], - "application/vnd.ms-word.template.macroEnabled.12": [".dotm"], - "application/vnd.ms-powerpoint": [".ppt"], - "application/vnd.ms-powerpoint.template.macroEnabled.12": [".pptm"], - "application/vnd.openxmlformats-officedocument.presentationml.presentation": [".pptx"], - "application/vnd.ms-powerpoint.template": [".pot"], - "application/vnd.openxmlformats-officedocument.presentationml.template": [".potx"], - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [".xlsx"], - "application/vnd.ms-excel": [".xls"], - "application/vnd.ms-excel.sheet.macroEnabled.12": [".xlsm"], - "application/vnd.ms-excel.sheet.binary.macroEnabled.12": [".xlsb"], - "application/vnd.ms-excel.workspace": [".xlw"], - "application/rtf": [".rtf"], - "application/xml": [".xml"], - "application/epub+zip": [".epub"], - "application/vnd.apple.keynote": [".key"], - "application/vnd.apple.pages": [".pages"], - "application/vnd.apple.numbers": [".numbers"], - "application/vnd.wordperfect": [".wpd"], - "application/vnd.oasis.opendocument.text": [".odt"], - "application/vnd.oasis.opendocument.presentation": [".odp"], - "application/vnd.oasis.opendocument.graphics": [".odg"], - "application/vnd.oasis.opendocument.spreadsheet": [".ods"], - "application/vnd.oasis.opendocument.formula": [".fods"], - "text/csv": [".csv"], - "text/tab-separated-values": [".tsv"], - "text/html": [".html", ".htm", ".web"], - "image/jpeg": [".jpg", ".jpeg"], - "image/png": [".png"], - "image/gif": [".gif"], - "image/bmp": [".bmp"], - "image/svg+xml": [".svg"], - "image/tiff": [".tiff"], - "image/webp": [".webp"], - "application/dbase": [".dbf"], - "application/vnd.lotus-1-2-3": [".123"], - "text/x-web-markdown": [ - ".602", - ".abw", - ".cgm", - ".cwk", - ".hwp", - ".lwp", - ".mw", - ".mcw", - ".pbd", - ".sda", - ".sdd", - ".sdp", - ".sdw", - ".sgl", - ".sti", - ".sxi", - ".sxw", - ".stw", - ".sxg", - ".uof", - ".uop", - ".uot", - ".vor", - ".wps", - ".zabw", - ], - "text/x-spreadsheet": [ - ".dif", - ".sylk", - ".slk", - ".prn", - ".et", - ".uos1", - ".uos2", - ".wk1", - ".wk2", - ".wk3", - ".wk4", - ".wks", - ".wq1", - ".wq2", - ".wb1", - ".wb2", - ".wb3", - ".qpw", - ".xlr", - ".eth", - ], - // Audio files (always supported) - ...audioFileTypes, - }; - } else if (etlService === "DOCLING") { - return { - // Docling supported file types - "application/pdf": [".pdf"], - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [".docx"], - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [".xlsx"], - "application/vnd.openxmlformats-officedocument.presentationml.presentation": [".pptx"], - "text/asciidoc": [".adoc", ".asciidoc"], - "text/html": [".html", ".htm", ".xhtml"], - "text/csv": [".csv"], - "image/png": [".png"], - "image/jpeg": [".jpg", ".jpeg"], - "image/tiff": [".tiff", ".tif"], - "image/bmp": [".bmp"], - "image/webp": [".webp"], - // Audio files (always supported) - ...audioFileTypes, - }; - } else { - return { - // Unstructured supported file types - "image/bmp": [".bmp"], - "text/csv": [".csv"], - "application/msword": [".doc"], - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [".docx"], - "message/rfc822": [".eml"], - "application/epub+zip": [".epub"], - "image/heic": [".heic"], - "text/html": [".html"], - "image/jpeg": [".jpeg", ".jpg"], - "image/png": [".png"], - "application/vnd.ms-outlook": [".msg"], - "application/vnd.oasis.opendocument.text": [".odt"], - "text/x-org": [".org"], - "application/pkcs7-signature": [".p7s"], - "application/pdf": [".pdf"], - "application/vnd.ms-powerpoint": [".ppt"], - "application/vnd.openxmlformats-officedocument.presentationml.presentation": [".pptx"], - "text/x-rst": [".rst"], - "application/rtf": [".rtf"], - "image/tiff": [".tiff"], - "text/tab-separated-values": [".tsv"], - "application/vnd.ms-excel": [".xls"], - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [".xlsx"], - "application/xml": [".xml"], - // Audio files (always supported) - ...audioFileTypes, - }; - } - }; - - const acceptedFileTypes = getAcceptedFileTypes(); - const supportedExtensions = Array.from(new Set(Object.values(acceptedFileTypes).flat())).sort(); - - const onDrop = useCallback((acceptedFiles: File[]) => { - setFiles((prevFiles) => [...prevFiles, ...acceptedFiles]); - }, []); - - const { getRootProps, getInputProps, isDragActive } = useDropzone({ - onDrop, - accept: acceptedFileTypes, - maxSize: 50 * 1024 * 1024, // 50MB - noClick: false, // Ensure clicking is enabled - noKeyboard: false, // Ensure keyboard navigation is enabled - }); - - const removeFile = (index: number) => { - setFiles((prevFiles) => prevFiles.filter((_, i) => i !== index)); - }; - - const formatFileSize = (bytes: number) => { - if (bytes === 0) return "0 Bytes"; - const k = 1024; - const sizes = ["Bytes", "KB", "MB", "GB", "TB"]; - const i = Math.floor(Math.log(bytes) / Math.log(k)); - return `${parseFloat((bytes / k ** i).toFixed(2))} ${sizes[i]}`; - }; - - const handleUpload = async () => { - setIsUploading(true); - setUploadProgress(0); - - const formData = new FormData(); - files.forEach((file) => { - formData.append("files", file); - }); - - formData.append("search_space_id", search_space_id); - - try { - // Simulate progress for better UX - const progressInterval = setInterval(() => { - setUploadProgress((prev) => { - if (prev >= 90) return prev; - return prev + Math.random() * 10; - }); - }, 200); - - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/fileupload`, - { - method: "POST", - headers: { - Authorization: `Bearer ${window.localStorage.getItem("surfsense_bearer_token")}`, - }, - body: formData, - } - ); - - clearInterval(progressInterval); - setUploadProgress(100); - - if (!response.ok) { - throw new Error("Upload failed"); - } - - await response.json(); - - toast(t("upload_initiated"), { - description: t("upload_initiated_desc"), - }); - - router.push(`/dashboard/${search_space_id}/documents`); - } catch (error: any) { - setIsUploading(false); - setUploadProgress(0); - toast(t("upload_error"), { - description: `${t("upload_error_desc")}: ${error.message}`, - }); - } - }; - - const getTotalFileSize = () => { - return files.reduce((total, file) => total + file.size, 0); - }; - - const containerVariants = { - hidden: { opacity: 0, y: 20 }, - visible: { - opacity: 1, - y: 0, - transition: { - duration: 0.5, - when: "beforeChildren", - staggerChildren: 0.1, - }, - }, - }; - - const itemVariants = { - hidden: { opacity: 0, y: 10 }, - visible: { opacity: 1, y: 0, transition: { duration: 0.3 } }, - }; - - const fileItemVariants = { - hidden: { opacity: 0, x: -20 }, - visible: { opacity: 1, x: 0, transition: { duration: 0.3 } }, - exit: { opacity: 0, x: 20, transition: { duration: 0.2 } }, - }; - - return ( -
- - {/* Header Card */} - - - - - - {t("title")} - - {t("subtitle")} - - - - - {t("file_size_limit")} - - - - - - {/* Upload Area Card */} - - - {/* Grid background pattern */} -
- -
- - -
- - - {isDragActive ? ( - - -

{t("drop_files")}

-
- ) : ( - - -
-

{t("drag_drop")}

-

{t("or_browse")}

-
-
- )} - - {/* Fallback button for better accessibility */} -
- -
-
-
-
-
- - {/* File List Card */} - - {files.length > 0 && ( - - - -
-
- {t("selected_files", { count: files.length })} - - {t("total_size")}: {formatFileSize(getTotalFileSize())} - -
- -
-
- -
- - {files.map((file, index) => ( - -
-
- -
-
-

{file.name}

-
- - {formatFileSize(file.size)} - - - {file.type || "Unknown type"} - -
-
-
-
- -
-
- ))} -
-
- - {isUploading && ( - - -
-
- {t("uploading_files")} - {Math.round(uploadProgress)}% -
- -
-
- )} - - - - -
-
-
- )} -
- - {/* Supported File Types Card */} - - - - - - {t("supported_file_types")} - - {t("file_types_desc")} - - -
- {supportedExtensions.map((ext) => ( - - {ext} - - ))} -
-
-
-
-
- - -
- ); + return null; } diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx index d2469a0a2..4e39ab39c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx @@ -1,302 +1,16 @@ "use client"; -import { IconBrandYoutube } from "@tabler/icons-react"; -import { type Tag, TagInput } from "emblor"; -import { Loader2 } from "lucide-react"; -import { motion, type Variants } from "motion/react"; import { useParams, useRouter } from "next/navigation"; -import { useTranslations } from "next-intl"; -import { useState } from "react"; -import { toast } from "sonner"; -import { Button } from "@/components/ui/button"; -import { - Card, - CardContent, - CardDescription, - CardFooter, - CardHeader, - CardTitle, -} from "@/components/ui/card"; -import { Label } from "@/components/ui/label"; +import { useEffect } from "react"; -// YouTube video ID validation regex -const youtubeRegex = - /^(https:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})$/; - -export default function YouTubeVideoAdder() { - const t = useTranslations("add_youtube"); +export default function YouTubeRedirect() { const params = useParams(); const router = useRouter(); const search_space_id = params.search_space_id as string; - const [videoTags, setVideoTags] = useState([]); - const [activeTagIndex, setActiveTagIndex] = useState(null); - const [isSubmitting, setIsSubmitting] = useState(false); - const [error, setError] = useState(null); + useEffect(() => { + router.replace(`/dashboard/${search_space_id}/sources/add?tab=youtube`); + }, [search_space_id, router]); - // Function to validate a YouTube URL - const isValidYoutubeUrl = (url: string): boolean => { - return youtubeRegex.test(url); - }; - - // Function to extract video ID from URL - const extractVideoId = (url: string): string | null => { - const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})/); - return match ? match[1] : null; - }; - - // Function to handle video URL submission - const handleSubmit = async () => { - // Validate that we have at least one video URL - if (videoTags.length === 0) { - setError(t("error_no_video")); - return; - } - - // Validate all URLs - const invalidUrls = videoTags.filter((tag) => !isValidYoutubeUrl(tag.text)); - if (invalidUrls.length > 0) { - setError(t("error_invalid_urls", { urls: invalidUrls.map((tag) => tag.text).join(", ") })); - return; - } - - setError(null); - setIsSubmitting(true); - - try { - toast(t("processing_toast"), { - description: t("processing_toast_desc"), - }); - - // Extract URLs from tags - const videoUrls = videoTags.map((tag) => tag.text); - - // Make API call to backend - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`, - }, - body: JSON.stringify({ - document_type: "YOUTUBE_VIDEO", - content: videoUrls, - search_space_id: parseInt(search_space_id), - }), - } - ); - - if (!response.ok) { - throw new Error("Failed to process YouTube videos"); - } - - await response.json(); - - toast(t("success_toast"), { - description: t("success_toast_desc"), - }); - - // Redirect to documents page - router.push(`/dashboard/${search_space_id}/documents`); - } catch (error: any) { - setError(error.message || t("error_generic")); - toast(t("error_toast"), { - description: `${t("error_toast_desc")}: ${error.message}`, - }); - } finally { - setIsSubmitting(false); - } - }; - - // Function to add a new video URL tag - const handleAddTag = (text: string) => { - // Basic URL validation - if (!isValidYoutubeUrl(text)) { - toast(t("invalid_url_toast"), { - description: t("invalid_url_toast_desc"), - }); - return; - } - - // Check for duplicates - if (videoTags.some((tag) => tag.text === text)) { - toast(t("duplicate_url_toast"), { - description: t("duplicate_url_toast_desc"), - }); - return; - } - - // Add the new tag - const newTag: Tag = { - id: Date.now().toString(), - text: text, - }; - - setVideoTags([...videoTags, newTag]); - }; - - // Animation variants - const containerVariants: Variants = { - hidden: { opacity: 0 }, - visible: { - opacity: 1, - transition: { - staggerChildren: 0.1, - }, - }, - }; - - const itemVariants: Variants = { - hidden: { y: 20, opacity: 0 }, - visible: { - y: 0, - opacity: 1, - transition: { - type: "spring", - stiffness: 300, - damping: 24, - }, - }, - }; - - return ( -
- - - - - - - {t("title")} - - {t("subtitle")} - - - - - -
-
- - -

{t("hint")}

-
- - {error && ( - - {error} - - )} - - -

{t("tips_title")}

-
    -
  • {t("tip_1")}
  • -
  • {t("tip_2")}
  • -
  • {t("tip_3")}
  • -
  • {t("tip_4")}
  • -
-
- - {videoTags.length > 0 && ( - -

{t("preview")}:

-
- {videoTags.map((tag, index) => { - const videoId = extractVideoId(tag.text); - return videoId ? ( - - - - ) : null; - })} -
-
- )} -
-
-
- - - - - - - -
-
-
- ); + return null; } diff --git a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx index 5cd9d5ad2..558220305 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx @@ -41,37 +41,18 @@ export default function DashboardLayout({ }, { - title: "Documents", + title: "Sources", url: "#", - icon: "FileStack", + icon: "Database", items: [ { - title: "Upload Documents", - url: `/dashboard/${search_space_id}/documents/upload`, - }, - // { - // title: "Add Webpages", - // url: `/dashboard/${search_space_id}/documents/webpage`, - // }, - { - title: "Add Youtube Videos", - url: `/dashboard/${search_space_id}/documents/youtube`, + title: "Add Sources", + url: `/dashboard/${search_space_id}/sources/add`, }, { title: "Manage Documents", url: `/dashboard/${search_space_id}/documents`, }, - ], - }, - { - title: "Connectors", - url: `#`, - icon: "Cable", - items: [ - { - title: "Add Connector", - url: `/dashboard/${search_space_id}/connectors/add`, - }, { title: "Manage Connectors", url: `/dashboard/${search_space_id}/connectors`, diff --git a/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx new file mode 100644 index 000000000..5172a0fa8 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx @@ -0,0 +1,78 @@ +"use client"; + +import { IconBrandYoutube } from "@tabler/icons-react"; +import { Cable, Database, Upload } from "lucide-react"; +import { motion } from "motion/react"; +import { useParams, useSearchParams } from "next/navigation"; +import { useEffect, useState } from "react"; +import { ConnectorsTab } from "@/components/sources/ConnectorsTab"; +import { DocumentUploadTab } from "@/components/sources/DocumentUploadTab"; +import { YouTubeTab } from "@/components/sources/YouTubeTab"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; + +export default function AddSourcesPage() { + const params = useParams(); + const searchParams = useSearchParams(); + const search_space_id = params.search_space_id as string; + const [activeTab, setActiveTab] = useState("documents"); + + // Handle tab from query parameter + useEffect(() => { + const tabParam = searchParams.get("tab"); + if (tabParam && ["documents", "youtube", "connectors"].includes(tabParam)) { + setActiveTab(tabParam); + } + }, [searchParams]); + + return ( +
+ + {/* Header */} +
+

+ + Add Sources +

+

Add your sources to your search space

+
+ + {/* Tabs */} + + + + + Documents + + + + YouTube + + + + Connectors + + + +
+ + + + + + + + + + + +
+
+
+
+ ); +} diff --git a/surfsense_web/components/dashboard-breadcrumb.tsx b/surfsense_web/components/dashboard-breadcrumb.tsx index db8d5981b..dd23a4f28 100644 --- a/surfsense_web/components/dashboard-breadcrumb.tsx +++ b/surfsense_web/components/dashboard-breadcrumb.tsx @@ -46,6 +46,7 @@ export function DashboardBreadcrumb() { researcher: t("researcher"), documents: t("documents"), connectors: t("connectors"), + sources: "Sources", podcasts: t("podcasts"), logs: t("logs"), chats: t("chats"), @@ -59,6 +60,21 @@ export function DashboardBreadcrumb() { const subSection = segments[3]; let subSectionLabel = subSection.charAt(0).toUpperCase() + subSection.slice(1); + // Handle sources sub-sections + if (section === "sources") { + const sourceLabels: Record = { + add: "Add Sources", + }; + + const sourceLabel = sourceLabels[subSection] || subSectionLabel; + breadcrumbs.push({ + label: "Sources", + href: `/dashboard/${segments[1]}/sources`, + }); + breadcrumbs.push({ label: sourceLabel }); + return breadcrumbs; + } + // Handle documents sub-sections if (section === "documents") { const documentLabels: Record = { diff --git a/surfsense_web/components/sidebar/app-sidebar.tsx b/surfsense_web/components/sidebar/app-sidebar.tsx index 7883ab1e2..2fc99d262 100644 --- a/surfsense_web/components/sidebar/app-sidebar.tsx +++ b/surfsense_web/components/sidebar/app-sidebar.tsx @@ -4,6 +4,7 @@ import { AlertCircle, BookOpen, Cable, + Database, ExternalLink, FileStack, FileText, @@ -40,6 +41,7 @@ import { export const iconMap: Record = { BookOpen, Cable, + Database, FileStack, Undo2, MessageCircleMore, @@ -69,54 +71,24 @@ const defaultData = { items: [], }, { - title: "Documents", + title: "Sources", url: "#", - icon: "FileStack", + icon: "Database", items: [ { - title: "Upload Documents", + title: "Add Sources", url: "#", }, - // { - // title: "Add Webpages", - // url: "#", - // }, { title: "Manage Documents", url: "#", }, - ], - }, - { - title: "Connectors", - url: "#", - icon: "Cable", - items: [ - { - title: "Add Connector", - url: "#", - }, { title: "Manage Connectors", url: "#", }, ], }, - { - title: "Research Synthesizer's", - url: "#", - icon: "SquareLibrary", - items: [ - { - title: "Podcast Creator", - url: "#", - }, - { - title: "Presentation Creator", - url: "#", - }, - ], - }, ], navSecondary: [ { diff --git a/surfsense_web/components/sidebar/nav-main.tsx b/surfsense_web/components/sidebar/nav-main.tsx index caea766b3..27d6d9fb7 100644 --- a/surfsense_web/components/sidebar/nav-main.tsx +++ b/surfsense_web/components/sidebar/nav-main.tsx @@ -36,13 +36,9 @@ export function NavMain({ items }: { items: NavItem[] }) { const titleMap: Record = { Researcher: "researcher", "Manage LLMs": "manage_llms", - Documents: "documents", - "Upload Documents": "upload_documents", - "Add Webpages": "add_webpages", - "Add Youtube Videos": "add_youtube", + Sources: "sources", + "Add Sources": "add_sources", "Manage Documents": "manage_documents", - Connectors: "connectors", - "Add Connector": "add_connector", "Manage Connectors": "manage_connectors", Podcasts: "podcasts", Logs: "logs", diff --git a/surfsense_web/components/sources/ConnectorsTab.tsx b/surfsense_web/components/sources/ConnectorsTab.tsx new file mode 100644 index 000000000..b8d3486f6 --- /dev/null +++ b/surfsense_web/components/sources/ConnectorsTab.tsx @@ -0,0 +1,200 @@ +"use client"; + +import { IconChevronDown, IconChevronRight } from "@tabler/icons-react"; +import { AnimatePresence, motion, type Variants } from "motion/react"; +import Link from "next/link"; +import { useTranslations } from "next-intl"; +import { useState } from "react"; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardFooter, CardHeader } from "@/components/ui/card"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { connectorCategories } from "./connector-data"; + +interface ConnectorsTabProps { + searchSpaceId: string; +} + +export function ConnectorsTab({ searchSpaceId }: ConnectorsTabProps) { + const t = useTranslations("add_connector"); + const [expandedCategories, setExpandedCategories] = useState([ + "search-engines", + "knowledge-bases", + "project-management", + "team-chats", + "communication", + ]); + + const toggleCategory = (categoryId: string) => { + setExpandedCategories((prev) => + prev.includes(categoryId) ? prev.filter((id) => id !== categoryId) : [...prev, categoryId] + ); + }; + + const cardVariants: Variants = { + hidden: { opacity: 0, y: 20 }, + visible: { + opacity: 1, + y: 0, + transition: { + type: "spring", + stiffness: 260, + damping: 20, + }, + }, + hover: { + scale: 1.02, + transition: { + type: "spring", + stiffness: 400, + damping: 10, + }, + }, + }; + + const staggerContainer = { + hidden: { opacity: 0 }, + visible: { + opacity: 1, + transition: { + staggerChildren: 0.1, + }, + }, + }; + + return ( + + {connectorCategories.map((category) => ( +
+ toggleCategory(category.id)} + className="w-full" + > +
+

{t(category.title)}

+ + + +
+ + + + + {category.connectors.map((connector) => ( + + + +
+ + {connector.icon} + +
+
+
+

{connector.title}

+ {connector.status === "coming-soon" && ( + + {t("coming_soon")} + + )} + {connector.status === "connected" && ( + + {t("connected")} + + )} +
+
+
+ + +

+ {t(connector.description)} +

+
+ + + {connector.status === "available" && ( + + + + )} + {connector.status === "coming-soon" && ( + + )} + {connector.status === "connected" && ( + + )} + +
+
+ ))} +
+
+
+
+
+ ))} +
+ ); +} diff --git a/surfsense_web/components/sources/DocumentUploadTab.tsx b/surfsense_web/components/sources/DocumentUploadTab.tsx new file mode 100644 index 000000000..c9976bb64 --- /dev/null +++ b/surfsense_web/components/sources/DocumentUploadTab.tsx @@ -0,0 +1,401 @@ +"use client"; + +import { CheckCircle2, FileType, Info, Loader2, Tag, Upload, X } from "lucide-react"; +import { AnimatePresence, motion } from "motion/react"; +import { useRouter } from "next/navigation"; +import { useTranslations } from "next-intl"; +import { useCallback, useState } from "react"; +import { useDropzone } from "react-dropzone"; +import { toast } from "sonner"; + +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Progress } from "@/components/ui/progress"; +import { Separator } from "@/components/ui/separator"; +import { GridPattern } from "./GridPattern"; + +interface DocumentUploadTabProps { + searchSpaceId: string; +} + +export function DocumentUploadTab({ searchSpaceId }: DocumentUploadTabProps) { + const t = useTranslations("upload_documents"); + const router = useRouter(); + const [files, setFiles] = useState([]); + const [isUploading, setIsUploading] = useState(false); + const [uploadProgress, setUploadProgress] = useState(0); + + const audioFileTypes = { + "audio/mpeg": [".mp3", ".mpeg", ".mpga"], + "audio/mp4": [".mp4", ".m4a"], + "audio/wav": [".wav"], + "audio/webm": [".webm"], + "text/markdown": [".md", ".markdown"], + "text/plain": [".txt"], + }; + + const getAcceptedFileTypes = () => { + const etlService = process.env.NEXT_PUBLIC_ETL_SERVICE; + + if (etlService === "LLAMACLOUD") { + return { + "application/pdf": [".pdf"], + "application/msword": [".doc"], + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [".docx"], + "application/vnd.ms-word.document.macroEnabled.12": [".docm"], + "application/msword-template": [".dot"], + "application/vnd.ms-word.template.macroEnabled.12": [".dotm"], + "application/vnd.ms-powerpoint": [".ppt"], + "application/vnd.ms-powerpoint.template.macroEnabled.12": [".pptm"], + "application/vnd.openxmlformats-officedocument.presentationml.presentation": [".pptx"], + "application/vnd.ms-powerpoint.template": [".pot"], + "application/vnd.openxmlformats-officedocument.presentationml.template": [".potx"], + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [".xlsx"], + "application/vnd.ms-excel": [".xls"], + "application/vnd.ms-excel.sheet.macroEnabled.12": [".xlsm"], + "application/vnd.ms-excel.sheet.binary.macroEnabled.12": [".xlsb"], + "application/vnd.ms-excel.workspace": [".xlw"], + "application/rtf": [".rtf"], + "application/xml": [".xml"], + "application/epub+zip": [".epub"], + "text/csv": [".csv"], + "text/tab-separated-values": [".tsv"], + "text/html": [".html", ".htm", ".web"], + "image/jpeg": [".jpg", ".jpeg"], + "image/png": [".png"], + "image/gif": [".gif"], + "image/bmp": [".bmp"], + "image/svg+xml": [".svg"], + "image/tiff": [".tiff"], + "image/webp": [".webp"], + ...audioFileTypes, + }; + } else if (etlService === "DOCLING") { + return { + "application/pdf": [".pdf"], + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [".docx"], + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [".xlsx"], + "application/vnd.openxmlformats-officedocument.presentationml.presentation": [".pptx"], + "text/asciidoc": [".adoc", ".asciidoc"], + "text/html": [".html", ".htm", ".xhtml"], + "text/csv": [".csv"], + "image/png": [".png"], + "image/jpeg": [".jpg", ".jpeg"], + "image/tiff": [".tiff", ".tif"], + "image/bmp": [".bmp"], + "image/webp": [".webp"], + ...audioFileTypes, + }; + } else { + return { + "image/bmp": [".bmp"], + "text/csv": [".csv"], + "application/msword": [".doc"], + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [".docx"], + "message/rfc822": [".eml"], + "application/epub+zip": [".epub"], + "image/heic": [".heic"], + "text/html": [".html"], + "image/jpeg": [".jpeg", ".jpg"], + "image/png": [".png"], + "application/vnd.ms-outlook": [".msg"], + "application/vnd.oasis.opendocument.text": [".odt"], + "text/x-org": [".org"], + "application/pkcs7-signature": [".p7s"], + "application/pdf": [".pdf"], + "application/vnd.ms-powerpoint": [".ppt"], + "application/vnd.openxmlformats-officedocument.presentationml.presentation": [".pptx"], + "text/x-rst": [".rst"], + "application/rtf": [".rtf"], + "image/tiff": [".tiff"], + "text/tab-separated-values": [".tsv"], + "application/vnd.ms-excel": [".xls"], + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [".xlsx"], + "application/xml": [".xml"], + ...audioFileTypes, + }; + } + }; + + const acceptedFileTypes = getAcceptedFileTypes(); + const supportedExtensions = Array.from(new Set(Object.values(acceptedFileTypes).flat())).sort(); + + const onDrop = useCallback((acceptedFiles: File[]) => { + setFiles((prevFiles) => [...prevFiles, ...acceptedFiles]); + }, []); + + const { getRootProps, getInputProps, isDragActive } = useDropzone({ + onDrop, + accept: acceptedFileTypes, + maxSize: 50 * 1024 * 1024, + noClick: false, + noKeyboard: false, + }); + + const removeFile = (index: number) => { + setFiles((prevFiles) => prevFiles.filter((_, i) => i !== index)); + }; + + const formatFileSize = (bytes: number) => { + if (bytes === 0) return "0 Bytes"; + const k = 1024; + const sizes = ["Bytes", "KB", "MB", "GB", "TB"]; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return `${parseFloat((bytes / k ** i).toFixed(2))} ${sizes[i]}`; + }; + + const handleUpload = async () => { + setIsUploading(true); + setUploadProgress(0); + + const formData = new FormData(); + files.forEach((file) => { + formData.append("files", file); + }); + formData.append("search_space_id", searchSpaceId); + + try { + const progressInterval = setInterval(() => { + setUploadProgress((prev) => { + if (prev >= 90) return prev; + return prev + Math.random() * 10; + }); + }, 200); + + const response = await fetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/fileupload`, + { + method: "POST", + headers: { + Authorization: `Bearer ${window.localStorage.getItem("surfsense_bearer_token")}`, + }, + body: formData, + } + ); + + clearInterval(progressInterval); + setUploadProgress(100); + + if (!response.ok) { + throw new Error("Upload failed"); + } + + await response.json(); + + toast(t("upload_initiated"), { + description: t("upload_initiated_desc"), + }); + + router.push(`/dashboard/${searchSpaceId}/documents`); + } catch (error: any) { + setIsUploading(false); + setUploadProgress(0); + toast(t("upload_error"), { + description: `${t("upload_error_desc")}: ${error.message}`, + }); + } + }; + + const getTotalFileSize = () => { + return files.reduce((total, file) => total + file.size, 0); + }; + + return ( + + + + {t("file_size_limit")} + + + +
+ +
+ + +
+ + + {isDragActive ? ( + + +

{t("drop_files")}

+
+ ) : ( + + +
+

{t("drag_drop")}

+

{t("or_browse")}

+
+
+ )} + +
+ +
+
+
+
+ + + {files.length > 0 && ( + + + +
+
+ {t("selected_files", { count: files.length })} + + {t("total_size")}: {formatFileSize(getTotalFileSize())} + +
+ +
+
+ +
+ + {files.map((file, index) => ( + +
+ +
+

{file.name}

+
+ + {formatFileSize(file.size)} + + + {file.type || "Unknown type"} + +
+
+
+ +
+ ))} +
+
+ + {isUploading && ( + + +
+
+ {t("uploading_files")} + {Math.round(uploadProgress)}% +
+ +
+
+ )} + + + + +
+
+
+ )} +
+ + + + + + {t("supported_file_types")} + + {t("file_types_desc")} + + +
+ {supportedExtensions.map((ext) => ( + + {ext} + + ))} +
+
+
+
+ ); +} diff --git a/surfsense_web/components/sources/GridPattern.tsx b/surfsense_web/components/sources/GridPattern.tsx new file mode 100644 index 000000000..3f12ad13f --- /dev/null +++ b/surfsense_web/components/sources/GridPattern.tsx @@ -0,0 +1,23 @@ +export function GridPattern() { + const columns = 41; + const rows = 11; + return ( +
+ {Array.from({ length: rows }).map((_, row) => + Array.from({ length: columns }).map((_, col) => { + const index = row * columns + col; + return ( +
+ ); + }) + )} +
+ ); +} diff --git a/surfsense_web/components/sources/YouTubeTab.tsx b/surfsense_web/components/sources/YouTubeTab.tsx new file mode 100644 index 000000000..717a4266d --- /dev/null +++ b/surfsense_web/components/sources/YouTubeTab.tsx @@ -0,0 +1,250 @@ +"use client"; + +import { IconBrandYoutube } from "@tabler/icons-react"; +import { TagInput, type Tag as TagType } from "emblor"; +import { Loader2 } from "lucide-react"; +import { motion } from "motion/react"; +import { useRouter } from "next/navigation"; +import { useTranslations } from "next-intl"; +import { useState } from "react"; +import { toast } from "sonner"; + +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Label } from "@/components/ui/label"; + +const youtubeRegex = + /^(https:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})$/; + +interface YouTubeTabProps { + searchSpaceId: string; +} + +export function YouTubeTab({ searchSpaceId }: YouTubeTabProps) { + const t = useTranslations("add_youtube"); + const router = useRouter(); + const [videoTags, setVideoTags] = useState([]); + const [activeTagIndex, setActiveTagIndex] = useState(null); + const [isSubmitting, setIsSubmitting] = useState(false); + const [error, setError] = useState(null); + + const isValidYoutubeUrl = (url: string): boolean => { + return youtubeRegex.test(url); + }; + + const extractVideoId = (url: string): string | null => { + const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})/); + return match ? match[1] : null; + }; + + const handleSubmit = async () => { + if (videoTags.length === 0) { + setError(t("error_no_video")); + return; + } + + const invalidUrls = videoTags.filter((tag) => !isValidYoutubeUrl(tag.text)); + if (invalidUrls.length > 0) { + setError(t("error_invalid_urls", { urls: invalidUrls.map((tag) => tag.text).join(", ") })); + return; + } + + setError(null); + setIsSubmitting(true); + + try { + toast(t("processing_toast"), { + description: t("processing_toast_desc"), + }); + + const videoUrls = videoTags.map((tag) => tag.text); + + const response = await fetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`, + }, + body: JSON.stringify({ + document_type: "YOUTUBE_VIDEO", + content: videoUrls, + search_space_id: parseInt(searchSpaceId), + }), + } + ); + + if (!response.ok) { + throw new Error("Failed to process YouTube videos"); + } + + await response.json(); + + toast(t("success_toast"), { + description: t("success_toast_desc"), + }); + + router.push(`/dashboard/${searchSpaceId}/documents`); + } catch (error: any) { + setError(error.message || t("error_generic")); + toast(t("error_toast"), { + description: `${t("error_toast_desc")}: ${error.message}`, + }); + } finally { + setIsSubmitting(false); + } + }; + + const handleAddTag = (text: string) => { + if (!isValidYoutubeUrl(text)) { + toast(t("invalid_url_toast"), { + description: t("invalid_url_toast_desc"), + }); + return; + } + + if (videoTags.some((tag) => tag.text === text)) { + toast(t("duplicate_url_toast"), { + description: t("duplicate_url_toast_desc"), + }); + return; + } + + const newTag: TagType = { + id: Date.now().toString(), + text: text, + }; + + setVideoTags([...videoTags, newTag]); + }; + + return ( + + + + + + {t("title")} + + {t("subtitle")} + + + +
+
+ + +

{t("hint")}

+
+ + {error && ( + + {error} + + )} + +
+

{t("tips_title")}

+
    +
  • {t("tip_1")}
  • +
  • {t("tip_2")}
  • +
  • {t("tip_3")}
  • +
  • {t("tip_4")}
  • +
+
+ + {videoTags.length > 0 && ( +
+

{t("preview")}:

+
+ {videoTags.map((tag, index) => { + const videoId = extractVideoId(tag.text); + return videoId ? ( + +