- Connect Your Tools
-
+
+ Connect Your Tools
+
+
Integrate with your favorite services to enhance your research capabilities.
-
- {connectorCategories.map((category, categoryIndex) => (
-
+ {connectorCategories.map((category) => (
+ toggleCategory(category.id)}
- className="border rounded-lg overflow-hidden bg-card"
+ variants={fadeIn}
+ className="rounded-lg border bg-card text-card-foreground shadow-sm"
>
-
-
-
-
- {category.icon}
-
-
-
{category.title}
-
{category.description}
-
-
-
- {category.connectors.map((connector, index) => (
+ toggleCategory(category.id)}
+ className="w-full"
+ >
+
+
{category.title}
+
+
+
+
+ {connector.icon}
+
+
+
+
+
{connector.title}
+ {connector.status === "coming-soon" && (
+
+ Coming soon
+
+ )}
+ {connector.status === "connected" && (
+
+ Connected
+
+ )}
+
+
+ {connector.description}
+
+
+
+
+ {connector.status === 'available' && (
+
+
+
+ )}
+ {connector.status === 'coming-soon' && (
+
+ )}
+ {connector.status === 'connected' && (
+
+ )}
+
+
+
+ ))}
+
+
+
+
+
))}
-
);
}
diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
index 66f8b0810..b7b4bf3ff 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
@@ -1,6 +1,7 @@
"use client";
-import { cn } from "@/lib/utils";
+import { DocumentViewer } from "@/components/document-viewer";
+import { JsonMetadataViewer } from "@/components/json-metadata-viewer";
import {
AlertDialog,
AlertDialogAction,
@@ -12,7 +13,6 @@ import {
AlertDialogTitle,
AlertDialogTrigger,
} from "@/components/ui/alert-dialog";
-import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import {
@@ -43,6 +43,9 @@ import {
TableHeader,
TableRow,
} from "@/components/ui/table";
+import { useDocuments } from "@/hooks/use-documents";
+import { cn } from "@/lib/utils";
+import { IconBrandGithub, IconBrandNotion, IconBrandSlack, IconBrandYoutube, IconLayoutKanban } from "@tabler/icons-react";
import {
ColumnDef,
ColumnFiltersState,
@@ -59,6 +62,7 @@ import {
getSortedRowModel,
useReactTable,
} from "@tanstack/react-table";
+import { AnimatePresence, motion } from "framer-motion";
import {
AlertCircle,
ChevronDown,
@@ -70,31 +74,22 @@ import {
CircleAlert,
CircleX,
Columns3,
- Filter,
- ListFilter,
- Plus,
- FileText,
- Globe,
- MessageSquare,
- FileX,
File,
- Trash,
+ FileX,
+ Filter,
+ Globe,
+ ListFilter,
MoreHorizontal,
- Webhook,
+ Trash,
+ Webhook
} from "lucide-react";
-import { useEffect, useId, useMemo, useRef, useState, useContext } from "react";
-import { motion, AnimatePresence } from "framer-motion";
import { useParams } from "next/navigation";
-import { useDocuments } from "@/hooks/use-documents";
-import React from "react";
-import { toast } from "sonner";
+import React, { useContext, useEffect, useId, useMemo, useRef, useState } from "react";
import ReactMarkdown from "react-markdown";
import rehypeRaw from "rehype-raw";
import rehypeSanitize from "rehype-sanitize";
import remarkGfm from "remark-gfm";
-import { DocumentViewer } from "@/components/document-viewer";
-import { JsonMetadataViewer } from "@/components/json-metadata-viewer";
-import { IconBrandNotion, IconBrandSlack, IconBrandYoutube } from "@tabler/icons-react";
+import { toast } from "sonner";
// Define animation variants for reuse
const fadeInScale = {
@@ -114,7 +109,7 @@ const fadeInScale = {
type Document = {
id: number;
title: string;
- document_type: "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE" | "YOUTUBE_VIDEO";
+ document_type: "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE" | "YOUTUBE_VIDEO" | "LINEAR_CONNECTOR";
document_metadata: any;
content: string;
created_at: string;
@@ -142,6 +137,8 @@ const documentTypeIcons = {
NOTION_CONNECTOR: IconBrandNotion,
FILE: File,
YOUTUBE_VIDEO: IconBrandYoutube,
+ GITHUB_CONNECTOR: IconBrandGithub,
+ LINEAR_CONNECTOR: IconLayoutKanban,
} as const;
const columns: ColumnDef
[] = [
@@ -1028,4 +1025,5 @@ function RowActions({ row }: { row: Row }) {
);
}
-export { DocumentsTable }
\ No newline at end of file
+export { DocumentsTable };
+
diff --git a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx
index 8156f6d2a..d371e9e53 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx
@@ -240,7 +240,7 @@ const SourcesDialogContent = ({
const ChatPage = () => {
const [token, setToken] = React.useState(null);
const [activeTab, setActiveTab] = useState("");
- const [dialogOpen, setDialogOpen] = useState(false);
+ const [dialogOpenId, setDialogOpenId] = useState(null);
const [sourcesPage, setSourcesPage] = useState(1);
const [expandedSources, setExpandedSources] = useState(false);
const [canScrollLeft, setCanScrollLeft] = useState(false);
@@ -260,6 +260,13 @@ const ChatPage = () => {
const { search_space_id, chat_id } = useParams();
+ // Function to scroll terminal to bottom
+ const scrollTerminalToBottom = () => {
+ if (terminalMessagesRef.current) {
+ terminalMessagesRef.current.scrollTop = terminalMessagesRef.current.scrollHeight;
+ }
+ };
+
// Get token from localStorage on client side only
React.useEffect(() => {
setToken(localStorage.getItem('surfsense_bearer_token'));
@@ -469,54 +476,60 @@ const ChatPage = () => {
updateChat();
}, [messages, status, chat_id, researchMode, selectedConnectors, search_space_id]);
- // Log messages whenever they update and extract annotations from the latest assistant message if available
- useEffect(() => {
- console.log('Messages updated:', messages);
-
- // Extract annotations from the latest assistant message if available
+ // Memoize connector sources to prevent excessive re-renders
+ const processedConnectorSources = React.useMemo(() => {
+ if (messages.length === 0) return connectorSources;
+
+ // Only process when we have a complete message (not streaming)
+ if (status !== 'ready') return connectorSources;
+
+ // Find the latest assistant message
const assistantMessages = messages.filter(msg => msg.role === 'assistant');
- if (assistantMessages.length > 0) {
- const latestAssistantMessage = assistantMessages[assistantMessages.length - 1];
- if (latestAssistantMessage?.annotations) {
- const annotations = latestAssistantMessage.annotations as any[];
-
- // Debug log to track streaming annotations
- if (process.env.NODE_ENV === 'development') {
- console.log('Streaming annotations:', annotations);
-
- // Log counts of each annotation type
- const terminalInfoCount = annotations.filter(a => a.type === 'TERMINAL_INFO').length;
- const sourcesCount = annotations.filter(a => a.type === 'SOURCES').length;
- const answerCount = annotations.filter(a => a.type === 'ANSWER').length;
-
- console.log(`Annotation counts - Terminal: ${terminalInfoCount}, Sources: ${sourcesCount}, Answer: ${answerCount}`);
- }
-
- // Process SOURCES annotation - get the last one to ensure we have the latest
- const sourcesAnnotations = annotations.filter(
- (annotation) => annotation.type === 'SOURCES'
- );
-
- if (sourcesAnnotations.length > 0) {
- // Get the last SOURCES annotation to ensure we have the most recent one
- const latestSourcesAnnotation = sourcesAnnotations[sourcesAnnotations.length - 1];
- if (latestSourcesAnnotation.content) {
- setConnectorSources(latestSourcesAnnotation.content);
- }
- }
-
- // Check for terminal info annotations and scroll terminal to bottom if they exist
- const terminalInfoAnnotations = annotations.filter(
- (annotation) => annotation.type === 'TERMINAL_INFO'
- );
-
- if (terminalInfoAnnotations.length > 0) {
- // Schedule scrolling after the DOM has been updated
- setTimeout(scrollTerminalToBottom, 100);
- }
- }
+ if (assistantMessages.length === 0) return connectorSources;
+
+ const latestAssistantMessage = assistantMessages[assistantMessages.length - 1];
+ if (!latestAssistantMessage?.annotations) return connectorSources;
+
+ // Find the latest SOURCES annotation
+ const annotations = latestAssistantMessage.annotations as any[];
+ const sourcesAnnotations = annotations.filter(a => a.type === 'SOURCES');
+
+ if (sourcesAnnotations.length === 0) return connectorSources;
+
+ const latestSourcesAnnotation = sourcesAnnotations[sourcesAnnotations.length - 1];
+ if (!latestSourcesAnnotation.content) return connectorSources;
+
+ // Use this content if it differs from current
+ return latestSourcesAnnotation.content;
+ }, [messages, status, connectorSources]);
+
+ // Update connector sources when processed value changes
+ useEffect(() => {
+ if (processedConnectorSources !== connectorSources) {
+ setConnectorSources(processedConnectorSources);
}
- }, [messages]);
+ }, [processedConnectorSources, connectorSources]);
+
+ // Check and scroll terminal when terminal info is available
+ useEffect(() => {
+ if (messages.length === 0 || status !== 'ready') return;
+
+ // Find the latest assistant message
+ const assistantMessages = messages.filter(msg => msg.role === 'assistant');
+ if (assistantMessages.length === 0) return;
+
+ const latestAssistantMessage = assistantMessages[assistantMessages.length - 1];
+ if (!latestAssistantMessage?.annotations) return;
+
+ // Check for terminal info annotations
+ const annotations = latestAssistantMessage.annotations as any[];
+ const terminalInfoAnnotations = annotations.filter(a => a.type === 'TERMINAL_INFO');
+
+ if (terminalInfoAnnotations.length > 0) {
+ // Schedule scrolling after the DOM has been updated
+ setTimeout(scrollTerminalToBottom, 100);
+ }
+ }, [messages, status]);
// Custom handleSubmit function to include selected connectors and answer type
const handleSubmit = (e: React.FormEvent) => {
@@ -543,24 +556,22 @@ const ChatPage = () => {
messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
};
- // Function to scroll terminal to bottom
- const scrollTerminalToBottom = () => {
- if (terminalMessagesRef.current) {
- terminalMessagesRef.current.scrollTop = terminalMessagesRef.current.scrollHeight;
- }
- };
-
// Scroll to bottom when messages change
useEffect(() => {
scrollToBottom();
}, [messages]);
- // Set activeTab when connectorSources change
- useEffect(() => {
- if (connectorSources.length > 0) {
- setActiveTab(connectorSources[0].type);
- }
+ // Set activeTab when connectorSources change using a memoized value
+ const activeTabValue = React.useMemo(() => {
+ return connectorSources.length > 0 ? connectorSources[0].type : "";
}, [connectorSources]);
+
+ // Update activeTab when the memoized value changes
+ useEffect(() => {
+ if (activeTabValue && activeTabValue !== activeTab) {
+ setActiveTab(activeTabValue);
+ }
+ }, [activeTabValue, activeTab]);
// Scroll terminal to bottom when expanded
useEffect(() => {
@@ -617,49 +628,89 @@ const ChatPage = () => {
};
// Function to get a citation source by ID
- const getCitationSource = (citationId: number): Source | null => {
+ const getCitationSource = React.useCallback((citationId: number, messageIndex?: number): Source | null => {
if (!messages || messages.length === 0) return null;
- // Find the latest assistant message
- const assistantMessages = messages.filter(msg => msg.role === 'assistant');
- if (assistantMessages.length === 0) return null;
+ // If no specific message index is provided, use the latest assistant message
+ if (messageIndex === undefined) {
+ // Find the latest assistant message
+ const assistantMessages = messages.filter(msg => msg.role === 'assistant');
+ if (assistantMessages.length === 0) return null;
- const latestAssistantMessage = assistantMessages[assistantMessages.length - 1];
- if (!latestAssistantMessage?.annotations) return null;
+ const latestAssistantMessage = assistantMessages[assistantMessages.length - 1];
+ if (!latestAssistantMessage?.annotations) return null;
- // Find all SOURCES annotations
- const annotations = latestAssistantMessage.annotations as any[];
- const sourcesAnnotations = annotations.filter(
- (annotation) => annotation.type === 'SOURCES'
- );
+ // Find all SOURCES annotations
+ const annotations = latestAssistantMessage.annotations as any[];
+ const sourcesAnnotations = annotations.filter(
+ (annotation) => annotation.type === 'SOURCES'
+ );
- // Get the latest SOURCES annotation
- if (sourcesAnnotations.length === 0) return null;
- const latestSourcesAnnotation = sourcesAnnotations[sourcesAnnotations.length - 1];
+ // Get the latest SOURCES annotation
+ if (sourcesAnnotations.length === 0) return null;
+ const latestSourcesAnnotation = sourcesAnnotations[sourcesAnnotations.length - 1];
- if (!latestSourcesAnnotation.content) return null;
+ if (!latestSourcesAnnotation.content) return null;
- // Flatten all sources from all connectors
- const allSources: Source[] = [];
- latestSourcesAnnotation.content.forEach((connector: ConnectorSource) => {
- if (connector.sources && Array.isArray(connector.sources)) {
- connector.sources.forEach((source: SourceItem) => {
- allSources.push({
- id: source.id,
- title: source.title,
- description: source.description,
- url: source.url,
- connectorType: connector.type
+ // Flatten all sources from all connectors
+ const allSources: Source[] = [];
+ latestSourcesAnnotation.content.forEach((connector: ConnectorSource) => {
+ if (connector.sources && Array.isArray(connector.sources)) {
+ connector.sources.forEach((source: SourceItem) => {
+ allSources.push({
+ id: source.id,
+ title: source.title,
+ description: source.description,
+ url: source.url,
+ connectorType: connector.type
+ });
});
- });
- }
- });
+ }
+ });
- // Find the source with the matching ID
- const foundSource = allSources.find(source => source.id === citationId);
+ // Find the source with the matching ID
+ const foundSource = allSources.find(source => source.id === citationId);
- return foundSource || null;
- };
+ return foundSource || null;
+ } else {
+ // Use the specific message by index
+ const message = messages[messageIndex];
+ if (!message || message.role !== 'assistant' || !message.annotations) return null;
+
+ // Find all SOURCES annotations
+ const annotations = message.annotations as any[];
+ const sourcesAnnotations = annotations.filter(
+ (annotation) => annotation.type === 'SOURCES'
+ );
+
+ // Get the latest SOURCES annotation
+ if (sourcesAnnotations.length === 0) return null;
+ const latestSourcesAnnotation = sourcesAnnotations[sourcesAnnotations.length - 1];
+
+ if (!latestSourcesAnnotation.content) return null;
+
+ // Flatten all sources from all connectors
+ const allSources: Source[] = [];
+ latestSourcesAnnotation.content.forEach((connector: ConnectorSource) => {
+ if (connector.sources && Array.isArray(connector.sources)) {
+ connector.sources.forEach((source: SourceItem) => {
+ allSources.push({
+ id: source.id,
+ title: source.title,
+ description: source.description,
+ url: source.url,
+ connectorType: connector.type
+ });
+ });
+ }
+ });
+
+ // Find the source with the matching ID
+ const foundSource = allSources.find(source => source.id === citationId);
+
+ return foundSource || null;
+ }
+ }, [messages]);
return (
<>
@@ -685,7 +736,11 @@ const ChatPage = () => {
-
+ getCitationSource(id, index)}
+ className="text-sm"
+ />
@@ -856,7 +911,7 @@ const ChatPage = () => {
))}
{connector.sources.length > INITIAL_SOURCES_DISPLAY && (
-
}
diff --git a/surfsense_web/app/docs/[[...slug]]/page.tsx b/surfsense_web/app/docs/[[...slug]]/page.tsx
new file mode 100644
index 000000000..6c8574d87
--- /dev/null
+++ b/surfsense_web/app/docs/[[...slug]]/page.tsx
@@ -0,0 +1,46 @@
+import { source } from '@/lib/source';
+import {
+ DocsBody,
+ DocsDescription,
+ DocsPage,
+ DocsTitle,
+} from 'fumadocs-ui/page';
+import { notFound } from 'next/navigation';
+import { getMDXComponents } from '@/mdx-components';
+
+export default async function Page(props: {
+ params: Promise<{ slug?: string[] }>;
+}) {
+ const params = await props.params;
+ const page = source.getPage(params.slug);
+ if (!page) notFound();
+
+ const MDX = page.data.body;
+
+ return (
+
+ );
+}
+
+export async function generateStaticParams() {
+ return source.generateParams();
+}
+
+export async function generateMetadata(props: {
+ params: Promise<{ slug?: string[] }>;
+}) {
+ const params = await props.params;
+ const page = source.getPage(params.slug);
+ if (!page) notFound();
+
+ return {
+ title: page.data.title,
+ description: page.data.description,
+ };
+}
\ No newline at end of file
diff --git a/surfsense_web/app/docs/layout.tsx b/surfsense_web/app/docs/layout.tsx
new file mode 100644
index 000000000..e818c1f68
--- /dev/null
+++ b/surfsense_web/app/docs/layout.tsx
@@ -0,0 +1,12 @@
+import { source } from '@/lib/source';
+import { DocsLayout } from 'fumadocs-ui/layouts/docs';
+import type { ReactNode } from 'react';
+import { baseOptions } from '@/app/layout.config';
+
+export default function Layout({ children }: { children: ReactNode }) {
+ return (
+
+ );
+}
\ No newline at end of file
diff --git a/surfsense_web/app/globals.css b/surfsense_web/app/globals.css
index 8fdefeca5..98e4411fb 100644
--- a/surfsense_web/app/globals.css
+++ b/surfsense_web/app/globals.css
@@ -1,4 +1,6 @@
-@import "tailwindcss";
+@import 'tailwindcss';
+@import 'fumadocs-ui/css/neutral.css';
+@import 'fumadocs-ui/css/preset.css';
@plugin "tailwindcss-animate";
diff --git a/surfsense_web/app/layout.config.tsx b/surfsense_web/app/layout.config.tsx
new file mode 100644
index 000000000..ef0500157
--- /dev/null
+++ b/surfsense_web/app/layout.config.tsx
@@ -0,0 +1,7 @@
+import { BaseLayoutProps } from 'fumadocs-ui/layouts/shared';
+
+export const baseOptions: BaseLayoutProps = {
+ nav: {
+ title: 'SurfSense Documentation',
+ },
+};
\ No newline at end of file
diff --git a/surfsense_web/app/layout.tsx b/surfsense_web/app/layout.tsx
index be0b18079..6b60891a4 100644
--- a/surfsense_web/app/layout.tsx
+++ b/surfsense_web/app/layout.tsx
@@ -5,6 +5,7 @@ import { Roboto } from "next/font/google";
import { Toaster } from "@/components/ui/sonner";
import { ThemeProvider } from "@/components/theme/theme-provider";
+import { RootProvider } from 'fumadocs-ui/provider';
const roboto = Roboto({
subsets: ["latin"],
@@ -64,8 +65,10 @@ export default async function RootLayout({
disableTransitionOnChange
defaultTheme="light"
>
- {children}
-