apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-13 17:52:38 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

resolve conflicts

Browse source
This commit is contained in:
Manoj Aggarwal 2026-01-16 15:39:57 -08:00
commit 11b160cee9
85 changed files with 8411 additions and 4637 deletions
Download patch file Download diff file Expand all files Collapse all files

5
surfsense_web/.env.example
View file

@ -1,5 +1,10 @@
NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000
NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL or GOOGLE
NEXT_PUBLIC_ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING
# Electric SQL
NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133
NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure
# Contact Form Vars - OPTIONAL
DATABASE_URL=postgresql://postgres:[YOUR-PASSWORD]@db.sdsf.supabase.co:5432/postgres

4
surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx
View file

@ -1,6 +1,6 @@
"use client";
import { ChevronDown, ChevronUp, FileX, Plus } from "lucide-react";
import { ChevronDown, ChevronUp, FileX, Loader2, Plus } from "lucide-react";
import { motion } from "motion/react";
import { useParams } from "next/navigation";
import { useTranslations } from "next-intl";
@ -114,7 +114,7 @@ export function DocumentsTableShell({
{loading ? (
<div className="flex h-[400px] w-full items-center justify-center">
<div className="flex flex-col items-center gap-2">
<div className="h-8 w-8 animate-spin rounded-full border-b-2 border-primary"></div>
<Loader2 className="h-8 w-8 animate-spin text-primary" />
<p className="text-sm text-muted-foreground">{t("loading")}</p>
</div>
</div>

44
surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/ProcessingIndicator.tsx
View file

@ -1,44 +0,0 @@
"use client";
import { Loader2 } from "lucide-react";
import { AnimatePresence, motion } from "motion/react";
import { useTranslations } from "next-intl";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
interface ProcessingIndicatorProps {
documentProcessorTasksCount: number;
}
export function ProcessingIndicator({ documentProcessorTasksCount }: ProcessingIndicatorProps) {
const t = useTranslations("documents");
// Only show when there are document_processor tasks (uploads), not connector_indexing_task (periodic reindexing)
if (documentProcessorTasksCount === 0) return null;
return (
<AnimatePresence>
<motion.div
initial={{ opacity: 0, height: 0, marginBottom: 0 }}
animate={{ opacity: 1, height: "auto", marginBottom: 24 }}
exit={{ opacity: 0, height: 0, marginBottom: 0 }}
transition={{ duration: 0.3 }}
>
<Alert className="border-border bg-primary/5">
<div className="flex items-center gap-4">
<div className="flex h-10 w-10 items-center justify-center rounded-full bg-primary/10">
<Loader2 className="h-5 w-5 animate-spin text-primary" />
</div>
<div className="flex-1">
<AlertTitle className="text-primary font-semibold">
{t("processing_documents")}
</AlertTitle>
<AlertDescription className="text-muted-foreground">
{t("active_tasks_count", { count: documentProcessorTasksCount })}
</AlertDescription>
</div>
</div>
</Alert>
</motion.div>
</AnimatePresence>
);
}

76
surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
View file

@ -6,20 +6,18 @@ import { RefreshCw, SquarePlus, Upload } from "lucide-react";
import { motion } from "motion/react";
import { useParams, useRouter } from "next/navigation";
import { useTranslations } from "next-intl";
import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react";
import { useCallback, useEffect, useId, useMemo, useState } from "react";
import { toast } from "sonner";
import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup";
import { Button } from "@/components/ui/button";
import type { DocumentTypeEnum } from "@/contracts/types/document.types";
import { useLogsSummary } from "@/hooks/use-logs";
import { documentsApiService } from "@/lib/apis/documents-api.service";
import { cacheKeys } from "@/lib/query-client/cache-keys";
import { DocumentsFilters } from "./components/DocumentsFilters";
import { DocumentsTableShell, type SortKey } from "./components/DocumentsTableShell";
import { PaginationControls } from "./components/PaginationControls";
import { ProcessingIndicator } from "./components/ProcessingIndicator";
import type { ColumnVisibility } from "./components/types";
function useDebounced<T>(value: T, delay = 250) {
@ -109,6 +107,52 @@ export default function DocumentsTable() {
enabled: !!searchSpaceId && !!debouncedSearch.trim(),
});
// Determine if we should show SurfSense docs (when no type filter or SURFSENSE_DOCS is selected)
const showSurfsenseDocs =
activeTypes.length === 0 || activeTypes.includes("SURFSENSE_DOCS" as DocumentTypeEnum);
// Use query for fetching SurfSense docs
const {
data: surfsenseDocsResponse,
isLoading: isSurfsenseDocsLoading,
refetch: refetchSurfsenseDocs,
} = useQuery({
queryKey: ["surfsense-docs", debouncedSearch, pageIndex, pageSize],
queryFn: () =>
documentsApiService.getSurfsenseDocs({
queryParams: {
page: pageIndex,
page_size: pageSize,
title: debouncedSearch.trim() || undefined,
},
}),
staleTime: 3 * 60 * 1000, // 3 minutes
enabled: showSurfsenseDocs,
});
// Transform SurfSense docs to match the Document type
const surfsenseDocsAsDocuments: Document[] = useMemo(() => {
if (!surfsenseDocsResponse?.items) return [];
return surfsenseDocsResponse.items.map((doc) => ({
id: doc.id,
title: doc.title,
document_type: "SURFSENSE_DOCS",
document_metadata: { source: doc.source },
content: doc.content,
created_at: new Date().toISOString(),
search_space_id: -1, // Special value for global docs
}));
}, [surfsenseDocsResponse]);
// Merge type counts with SURFSENSE_DOCS count
const typeCounts = useMemo(() => {
const counts = { ...(rawTypeCounts || {}) };
if (surfsenseDocsResponse?.total) {
counts.SURFSENSE_DOCS = surfsenseDocsResponse.total;
}
return counts;
}, [rawTypeCounts, surfsenseDocsResponse?.total]);
// Extract documents and total based on search state
const documents = debouncedSearch.trim()
? searchResponse?.items || []
@ -150,30 +194,6 @@ export default function DocumentsTable() {
}
}, [debouncedSearch, refetchSearch, refetchDocuments, t, isRefreshing]);
// Set up smart polling for active tasks - only polls when tasks are in progress
const { summary } = useLogsSummary(searchSpaceId, 24, {
enablePolling: true,
refetchInterval: 5000, // Poll every 5 seconds when tasks are active
});
// Filter active tasks to only include document_processor tasks (uploads via "add sources")
// Exclude connector_indexing_task tasks (periodic reindexing)
const documentProcessorTasks =
summary?.active_tasks.filter((task) => task.source === "document_processor") || [];
const documentProcessorTasksCount = documentProcessorTasks.length;
const activeTasksCount = summary?.active_tasks.length || 0;
const prevActiveTasksCount = useRef(activeTasksCount);
// Auto-refresh when a task finishes
useEffect(() => {
if (prevActiveTasksCount.current > activeTasksCount) {
// A task has finished!
refreshCurrentView();
}
prevActiveTasksCount.current = activeTasksCount;
}, [activeTasksCount, refreshCurrentView]);
// Create a delete function for single document deletion
const deleteDocument = useCallback(
async (id: number) => {
@ -262,8 +282,6 @@ export default function DocumentsTable() {
</div>
</motion.div>
<ProcessingIndicator documentProcessorTasksCount={documentProcessorTasksCount} />
<DocumentsFilters
typeCounts={rawTypeCounts ?? {}}
selectedIds={selectedIds}

4
surfsense_web/app/dashboard/[search_space_id]/editor/[documentId]/page.tsx
View file

@ -438,9 +438,7 @@ export default function EditorPage() {
{saving ? (
<>
<Loader2 className="h-3.5 w-3.5 md:h-4 md:w-4 animate-spin" />
<span className="text-xs md:text-sm">
{isNewNote ? "Creating..." : "Saving..."}
</span>
<span className="text-xs md:text-sm">{isNewNote ? "Creating" : "Saving"}</span>
</>
) : (
<>

4
surfsense_web/app/dashboard/[search_space_id]/team/page.tsx
View file

@ -1294,7 +1294,7 @@ function CreateInviteDialog({
{creating ? (
<>
<Loader2 className="h-4 w-4 mr-2 animate-spin" />
Creating...
Creating
</>
) : (
"Create Invite"
@ -1471,7 +1471,7 @@ function CreateRoleDialog({
{creating ? (
<>
<Loader2 className="h-4 w-4 mr-2 animate-spin" />
Creating...
Creating
</>
) : (
"Create Role"

5
surfsense_web/app/layout.tsx
View file

@ -2,6 +2,7 @@ import type { Metadata } from "next";
import "./globals.css";
import { RootProvider } from "fumadocs-ui/provider/next";
import { Roboto } from "next/font/google";
import { ElectricProvider } from "@/components/providers/ElectricProvider";
import { I18nProvider } from "@/components/providers/I18nProvider";
import { PostHogProvider } from "@/components/providers/PostHogProvider";
import { ThemeProvider } from "@/components/theme/theme-provider";
@ -102,7 +103,9 @@ export default function RootLayout({
defaultTheme="light"
>
<RootProvider>
<ReactQueryClientProvider>{children}</ReactQueryClientProvider>
<ReactQueryClientProvider>
<ElectricProvider>{children}</ElectricProvider>
</ReactQueryClientProvider>
<Toaster />
</RootProvider>
</ThemeProvider>

13
surfsense_web/components/UserDropdown.tsx
View file

@ -13,6 +13,7 @@ import {
DropdownMenuSeparator,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { cleanupElectric } from "@/lib/electric/client";
import { resetUser, trackLogout } from "@/lib/posthog/events";
export function UserDropdown({
@ -26,12 +27,20 @@ export function UserDropdown({
}) {
const router = useRouter();
const handleLogout = () => {
const handleLogout = async () => {
try {
// Track logout event and reset PostHog identity
trackLogout();
resetUser();
// Best-effort cleanup of Electric SQL / PGlite
// Even if this fails, login-time cleanup will handle it
try {
await cleanupElectric();
} catch (err) {
console.warn("[Logout] Electric cleanup failed (will be handled on next login):", err);
}
if (typeof window !== "undefined") {
localStorage.removeItem("surfsense_bearer_token");
window.location.href = "/";
@ -40,7 +49,7 @@ export function UserDropdown({
console.error("Error during logout:", error);
// Optionally, provide user feedback
if (typeof window !== "undefined") {
alert("Logout failed. Please try again.");
localStorage.removeItem("surfsense_bearer_token");
window.location.href = "/";
}
}

2
surfsense_web/components/assistant-ui/attachment.tsx
View file

@ -357,7 +357,7 @@ export const ComposerAddAttachment: FC = () => {
</DropdownMenuItem>
<DropdownMenuItem onClick={handleFileUpload} className="cursor-pointer">
<Upload className="size-4" />
<span>Upload Files</span>
<span>Upload Documents</span>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>

300
surfsense_web/components/assistant-ui/composer-action.tsx
View file

@ -1,300 +0,0 @@
import { AssistantIf, ComposerPrimitive, useAssistantState } from "@assistant-ui/react";
import { useAtomValue } from "jotai";
import {
AlertCircle,
ArrowUpIcon,
ChevronRightIcon,
Loader2,
Plug2,
Plus,
SquareIcon,
} from "lucide-react";
import type { FC } from "react";
import { useCallback, useMemo, useRef, useState } from "react";
import { getDocumentTypeLabel } from "@/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentTypeIcon";
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
import {
globalNewLLMConfigsAtom,
llmPreferencesAtom,
newLLMConfigsAtom,
} from "@/atoms/new-llm-config/new-llm-config-query.atoms";
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
import { ComposerAddAttachment } from "@/components/assistant-ui/attachment";
import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button";
import { Button } from "@/components/ui/button";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
import { cn } from "@/lib/utils";
const ConnectorIndicator: FC = () => {
const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom);
const { connectors, isLoading: connectorsLoading } = useSearchSourceConnectors(
false,
searchSpaceId ? Number(searchSpaceId) : undefined
);
const { data: documentTypeCounts, isLoading: documentTypesLoading } =
useAtomValue(documentTypeCountsAtom);
const [isOpen, setIsOpen] = useState(false);
const closeTimeoutRef = useRef<NodeJS.Timeout | null>(null);
const isLoading = connectorsLoading || documentTypesLoading;
const activeDocumentTypes = documentTypeCounts
? Object.entries(documentTypeCounts).filter(([_, count]) => count > 0)
: [];
// Count only active connectors (matching what's shown in the Active tab)
const activeConnectorsCount = connectors.length;
const hasConnectors = activeConnectorsCount > 0;
const hasSources = hasConnectors || activeDocumentTypes.length > 0;
const handleMouseEnter = useCallback(() => {
// Clear any pending close timeout
if (closeTimeoutRef.current) {
clearTimeout(closeTimeoutRef.current);
closeTimeoutRef.current = null;
}
setIsOpen(true);
}, []);
const handleMouseLeave = useCallback(() => {
// Delay closing by 150ms for better UX
closeTimeoutRef.current = setTimeout(() => {
setIsOpen(false);
}, 150);
}, []);
if (!searchSpaceId) return null;
return (
<Popover open={isOpen} onOpenChange={setIsOpen}>
<PopoverTrigger asChild>
<button
type="button"
className={cn(
"size-[34px] rounded-full p-1 flex items-center justify-center transition-colors relative",
"hover:bg-muted-foreground/15 dark:hover:bg-muted-foreground/30",
"outline-none focus:outline-none focus-visible:outline-none",
"border-0 ring-0 focus:ring-0 shadow-none focus:shadow-none",
"data-[state=open]:bg-transparent data-[state=open]:shadow-none data-[state=open]:ring-0",
"text-muted-foreground"
)}
aria-label={
hasConnectors
? `View ${activeConnectorsCount} active connectors`
: "Add your first connector"
}
onMouseEnter={handleMouseEnter}
onMouseLeave={handleMouseLeave}
>
{isLoading ? (
<Loader2 className="size-4 animate-spin" />
) : (
<>
<Plug2 className="size-4" />
{activeConnectorsCount > 0 && (
<span className="absolute -top-0.5 -right-0.5 flex items-center justify-center min-w-[16px] h-4 px-1 text-[10px] font-medium rounded-full bg-primary text-primary-foreground shadow-sm">
{activeConnectorsCount > 99 ? "99+" : activeConnectorsCount}
</span>
)}
</>
)}
</button>
</PopoverTrigger>
<PopoverContent
side="bottom"
align="start"
className="w-64 p-3"
onMouseEnter={handleMouseEnter}
onMouseLeave={handleMouseLeave}
>
{hasSources ? (
<div className="space-y-3">
{activeConnectorsCount > 0 && (
<div className="flex items-center justify-between">
<p className="text-xs font-medium text-muted-foreground">Active Connectors</p>
<span className="text-xs font-medium bg-muted px-1.5 py-0.5 rounded">
{activeConnectorsCount}
</span>
</div>
)}
{activeConnectorsCount > 0 && (
<div className="flex flex-wrap gap-2">
{connectors.map((connector) => (
<div
key={`connector-${connector.id}`}
className="flex items-center gap-1.5 rounded-md bg-muted/80 px-2.5 py-1.5 text-xs border border-border/50"
>
{getConnectorIcon(connector.connector_type, "size-3.5")}
<span className="truncate max-w-[100px]">{connector.name}</span>
</div>
))}
</div>
)}
{activeDocumentTypes.length > 0 && (
<>
{activeConnectorsCount > 0 && (
<div className="pt-2 border-t border-border/50">
<p className="text-xs font-medium text-muted-foreground mb-2">Documents</p>
</div>
)}
<div className="flex flex-wrap gap-2">
{activeDocumentTypes.map(([docType, count]) => (
<div
key={docType}
className="flex items-center gap-1.5 rounded-md bg-muted/80 px-2.5 py-1.5 text-xs border border-border/50"
>
{getConnectorIcon(docType, "size-3.5")}
<span className="truncate max-w-[100px]">
{getDocumentTypeLabel(docType)}
</span>
<span className="flex items-center justify-center min-w-[18px] h-[18px] px-1 text-[10px] font-medium rounded-full bg-primary/10 text-primary">
{count > 999 ? "999+" : count}
</span>
</div>
))}
</div>
</>
)}
<div className="pt-1 border-t border-border/50">
<button
type="button"
className="inline-flex items-center gap-1.5 text-xs text-muted-foreground hover:text-foreground transition-colors"
onClick={() => {
/* Connector popup should be opened via the connector indicator button */
}}
>
<Plus className="size-3" />
Add more sources
<ChevronRightIcon className="size-3" />
</button>
</div>
</div>
) : (
<div className="space-y-2">
<p className="text-sm font-medium">No sources yet</p>
<p className="text-xs text-muted-foreground">
Add documents or connect data sources to enhance search results.
</p>
<button
type="button"
className="inline-flex items-center gap-1.5 rounded-md bg-primary px-3 py-1.5 text-xs font-medium text-primary-foreground hover:bg-primary/90 transition-colors mt-1"
onClick={() => {
/* Connector popup should be opened via the connector indicator button */
}}
>
<Plus className="size-3" />
Add Connector
</button>
</div>
)}
</PopoverContent>
</Popover>
);
};
export const ComposerAction: FC = () => {
// Check if any attachments are still being processed (running AND progress < 100)
// When progress is 100, processing is done but waiting for send()
const hasProcessingAttachments = useAssistantState(({ composer }) =>
composer.attachments?.some((att) => {
const status = att.status;
if (status?.type !== "running") return false;
const progress = (status as { type: "running"; progress?: number }).progress;
return progress === undefined || progress < 100;
})
);
// Check if composer text is empty
const isComposerEmpty = useAssistantState(({ composer }) => {
const text = composer.text?.trim() || "";
return text.length === 0;
});
// Check if a model is configured
const { data: userConfigs } = useAtomValue(newLLMConfigsAtom);
const { data: globalConfigs } = useAtomValue(globalNewLLMConfigsAtom);
const { data: preferences } = useAtomValue(llmPreferencesAtom);
const hasModelConfigured = useMemo(() => {
if (!preferences) return false;
const agentLlmId = preferences.agent_llm_id;
if (agentLlmId === null || agentLlmId === undefined) return false;
// Check if the configured model actually exists
if (agentLlmId < 0) {
return globalConfigs?.some((c) => c.id === agentLlmId) ?? false;
}
return userConfigs?.some((c) => c.id === agentLlmId) ?? false;
}, [preferences, globalConfigs, userConfigs]);
const isSendDisabled = hasProcessingAttachments || isComposerEmpty || !hasModelConfigured;
return (
<div className="aui-composer-action-wrapper relative mx-2 mb-2 flex items-center justify-between">
<div className="flex items-center gap-1">
<ComposerAddAttachment />
<ConnectorIndicator />
</div>
{/* Show processing indicator when attachments are being processed */}
{hasProcessingAttachments && (
<div className="flex items-center gap-1.5 text-muted-foreground text-xs">
<Loader2 className="size-3 animate-spin" />
<span>Processing...</span>
</div>
)}
{/* Show warning when no model is configured */}
{!hasModelConfigured && !hasProcessingAttachments && (
<div className="flex items-center gap-1.5 text-amber-600 dark:text-amber-400 text-xs">
<AlertCircle className="size-3" />
<span>Select a model</span>
</div>
)}
<AssistantIf condition={({ thread }) => !thread.isRunning}>
<ComposerPrimitive.Send asChild disabled={isSendDisabled}>
<TooltipIconButton
tooltip={
!hasModelConfigured
? "Please select a model from the header to start chatting"
: hasProcessingAttachments
? "Wait for attachments to process"
: isComposerEmpty
? "Enter a message to send"
: "Send message"
}
side="bottom"
type="submit"
variant="default"
size="icon"
className={cn(
"aui-composer-send size-8 rounded-full",
isSendDisabled && "cursor-not-allowed opacity-50"
)}
aria-label="Send message"
disabled={isSendDisabled}
>
<ArrowUpIcon className="aui-composer-send-icon size-4" />
</TooltipIconButton>
</ComposerPrimitive.Send>
</AssistantIf>
<AssistantIf condition={({ thread }) => thread.isRunning}>
<ComposerPrimitive.Cancel asChild>
<Button
type="button"
variant="default"
size="icon"
className="aui-composer-cancel size-8 rounded-full"
aria-label="Stop generating"
>
<SquareIcon className="aui-composer-cancel-icon size-3 fill-current" />
</Button>
</ComposerPrimitive.Cancel>
</AssistantIf>
</div>
);
};

257
surfsense_web/components/assistant-ui/composer.tsx
View file

@ -1,257 +0,0 @@
import { ComposerPrimitive, useAssistantState, useComposerRuntime } from "@assistant-ui/react";
import { useAtom, useSetAtom } from "jotai";
import { useParams } from "next/navigation";
import type { FC } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import { createPortal } from "react-dom";
import {
mentionedDocumentIdsAtom,
mentionedDocumentsAtom,
} from "@/atoms/chat/mentioned-documents.atom";
import { ComposerAddAttachment, ComposerAttachments } from "@/components/assistant-ui/attachment";
import { ComposerAction } from "@/components/assistant-ui/composer-action";
import {
InlineMentionEditor,
type InlineMentionEditorRef,
} from "@/components/assistant-ui/inline-mention-editor";
import {
DocumentMentionPicker,
type DocumentMentionPickerRef,
} from "@/components/new-chat/document-mention-picker";
import type { Document } from "@/contracts/types/document.types";
export const Composer: FC = () => {
// ---- State for document mentions (using atoms to persist across remounts) ----
const [mentionedDocuments, setMentionedDocuments] = useAtom(mentionedDocumentsAtom);
const [showDocumentPopover, setShowDocumentPopover] = useState(false);
const [mentionQuery, setMentionQuery] = useState("");
const editorRef = useRef<InlineMentionEditorRef>(null);
const editorContainerRef = useRef<HTMLDivElement>(null);
const documentPickerRef = useRef<DocumentMentionPickerRef>(null);
const { search_space_id } = useParams();
const setMentionedDocumentIds = useSetAtom(mentionedDocumentIdsAtom);
const composerRuntime = useComposerRuntime();
const hasAutoFocusedRef = useRef(false);
// Check if thread is empty (new chat)
const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty);
// Check if thread is currently running (streaming response)
const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning);
// Auto-focus editor when on new chat page
useEffect(() => {
if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) {
// Small delay to ensure the editor is fully mounted
const timeoutId = setTimeout(() => {
editorRef.current?.focus();
hasAutoFocusedRef.current = true;
}, 100);
return () => clearTimeout(timeoutId);
}
}, [isThreadEmpty]);
// Sync mentioned document IDs to atom for use in chat request
useEffect(() => {
setMentionedDocumentIds({
surfsense_doc_ids: mentionedDocuments
.filter((doc) => doc.document_type === "SURFSENSE_DOCS")
.map((doc) => doc.id),
document_ids: mentionedDocuments
.filter((doc) => doc.document_type !== "SURFSENSE_DOCS")
.map((doc) => doc.id),
});
}, [mentionedDocuments, setMentionedDocumentIds]);
// Handle text change from inline editor - sync with assistant-ui composer
const handleEditorChange = useCallback(
(text: string) => {
composerRuntime.setText(text);
},
[composerRuntime]
);
// Handle @ mention trigger from inline editor
const handleMentionTrigger = useCallback((query: string) => {
setShowDocumentPopover(true);
setMentionQuery(query);
}, []);
// Handle mention close
const handleMentionClose = useCallback(() => {
if (showDocumentPopover) {
setShowDocumentPopover(false);
setMentionQuery("");
}
}, [showDocumentPopover]);
// Handle keyboard navigation when popover is open
const handleKeyDown = useCallback(
(e: React.KeyboardEvent) => {
if (showDocumentPopover) {
if (e.key === "ArrowDown") {
e.preventDefault();
documentPickerRef.current?.moveDown();
return;
}
if (e.key === "ArrowUp") {
e.preventDefault();
documentPickerRef.current?.moveUp();
return;
}
if (e.key === "Enter") {
e.preventDefault();
documentPickerRef.current?.selectHighlighted();
return;
}
if (e.key === "Escape") {
e.preventDefault();
setShowDocumentPopover(false);
setMentionQuery("");
return;
}
}
},
[showDocumentPopover]
);
// Handle submit from inline editor (Enter key)
const handleSubmit = useCallback(() => {
// Prevent sending while a response is still streaming
if (isThreadRunning) {
return;
}
if (!showDocumentPopover) {
composerRuntime.send();
// Clear the editor after sending
editorRef.current?.clear();
setMentionedDocuments([]);
setMentionedDocumentIds({
surfsense_doc_ids: [],
document_ids: [],
});
}
}, [
showDocumentPopover,
isThreadRunning,
composerRuntime,
setMentionedDocuments,
setMentionedDocumentIds,
]);
const handleDocumentRemove = useCallback(
(docId: number, docType?: string) => {
setMentionedDocuments((prev) => {
const updated = prev.filter((doc) => !(doc.id === docId && doc.document_type === docType));
setMentionedDocumentIds({
surfsense_doc_ids: updated
.filter((doc) => doc.document_type === "SURFSENSE_DOCS")
.map((doc) => doc.id),
document_ids: updated
.filter((doc) => doc.document_type !== "SURFSENSE_DOCS")
.map((doc) => doc.id),
});
return updated;
});
},
[setMentionedDocuments, setMentionedDocumentIds]
);
const handleDocumentsMention = useCallback(
(documents: Pick<Document, "id" | "title" | "document_type">[]) => {
const existingKeys = new Set(mentionedDocuments.map((d) => `${d.document_type}:${d.id}`));
const newDocs = documents.filter(
(doc) => !existingKeys.has(`${doc.document_type}:${doc.id}`)
);
for (const doc of newDocs) {
editorRef.current?.insertDocumentChip(doc);
}
setMentionedDocuments((prev) => {
const existingKeySet = new Set(prev.map((d) => `${d.document_type}:${d.id}`));
const uniqueNewDocs = documents.filter(
(doc) => !existingKeySet.has(`${doc.document_type}:${doc.id}`)
);
const updated = [...prev, ...uniqueNewDocs];
setMentionedDocumentIds({
surfsense_doc_ids: updated
.filter((doc) => doc.document_type === "SURFSENSE_DOCS")
.map((doc) => doc.id),
document_ids: updated
.filter((doc) => doc.document_type !== "SURFSENSE_DOCS")
.map((doc) => doc.id),
});
return updated;
});
setMentionQuery("");
},
[mentionedDocuments, setMentionedDocuments, setMentionedDocumentIds]
);
return (
<ComposerPrimitive.Root className="aui-composer-root relative flex w-full flex-col">
<ComposerPrimitive.AttachmentDropzone className="aui-composer-attachment-dropzone flex w-full flex-col rounded-2xl border-input bg-muted px-1 pt-2 outline-none transition-shadow data-[dragging=true]:border-ring data-[dragging=true]:border-dashed data-[dragging=true]:bg-accent/50">
<ComposerAttachments />
{/* -------- Inline Mention Editor -------- */}
<div ref={editorContainerRef} className="aui-composer-input-wrapper px-3 pt-3 pb-6">
<InlineMentionEditor
ref={editorRef}
placeholder="Ask SurfSense or @mention docs"
onMentionTrigger={handleMentionTrigger}
onMentionClose={handleMentionClose}
onChange={handleEditorChange}
onDocumentRemove={handleDocumentRemove}
onSubmit={handleSubmit}
onKeyDown={handleKeyDown}
className="min-h-[24px]"
/>
</div>
{/* -------- Document mention popover (rendered via portal) -------- */}
{showDocumentPopover &&
typeof document !== "undefined" &&
createPortal(
<>
{/* Backdrop */}
<button
type="button"
className="fixed inset-0 cursor-default"
style={{ zIndex: 9998 }}
onClick={() => setShowDocumentPopover(false)}
aria-label="Close document picker"
/>
{/* Popover positioned above input */}
<div
className="fixed shadow-2xl rounded-lg border border-border overflow-hidden bg-popover"
style={{
zIndex: 9999,
bottom: editorContainerRef.current
? `${window.innerHeight - editorContainerRef.current.getBoundingClientRect().top + 8}px`
: "200px",
left: editorContainerRef.current
? `${editorContainerRef.current.getBoundingClientRect().left}px`
: "50%",
}}
>
<DocumentMentionPicker
ref={documentPickerRef}
searchSpaceId={Number(search_space_id)}
onSelectionChange={handleDocumentsMention}
onDone={() => {
setShowDocumentPopover(false);
setMentionQuery("");
}}
initialSelectedDocuments={mentionedDocuments}
externalSearch={mentionQuery}
/>
</div>
</>,
document.body
)}
<ComposerAction />
</ComposerPrimitive.AttachmentDropzone>
</ComposerPrimitive.Root>
);
};

123
surfsense_web/components/assistant-ui/connector-popup.tsx
View file

@ -1,19 +1,16 @@
"use client";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { useAtomValue } from "jotai";
import { Cable, Loader2 } from "lucide-react";
import { useSearchParams } from "next/navigation";
import { type FC, useEffect, useMemo } from "react";
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
import type { FC } from "react";
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button";
import { Dialog, DialogContent } from "@/components/ui/dialog";
import { Tabs, TabsContent } from "@/components/ui/tabs";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { useLogsSummary } from "@/hooks/use-logs";
import { connectorsApiService } from "@/lib/apis/connectors-api.service";
import { cacheKeys } from "@/lib/query-client/cache-keys";
import { useConnectorsElectric } from "@/hooks/use-connectors-electric";
import { useDocumentsElectric } from "@/hooks/use-documents-electric";
import { cn } from "@/lib/utils";
import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header";
import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view";
@ -21,6 +18,7 @@ import { ConnectorEditView } from "./connector-popup/connector-configs/views/con
import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view";
import { OAUTH_CONNECTORS } from "./connector-popup/constants/connector-constants";
import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog";
import { useIndexingConnectors } from "./connector-popup/hooks/use-indexing-connectors";
import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab";
import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab";
import { ConnectorAccountsListView } from "./connector-popup/views/connector-accounts-list-view";
@ -30,18 +28,13 @@ import { YouTubeCrawlerView } from "./connector-popup/views/youtube-crawler-view
export const ConnectorIndicator: FC = () => {
const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom);
const searchParams = useSearchParams();
const { data: documentTypeCounts, isLoading: documentTypesLoading } =
useAtomValue(documentTypeCountsAtom);
// Fetch document type counts using Electric SQL + PGlite for real-time updates
const { documentTypeCounts, loading: documentTypesLoading } = useDocumentsElectric(searchSpaceId);
// Check if YouTube view is active
const isYouTubeView = searchParams.get("view") === "youtube";
// Track active indexing tasks
const { summary: logsSummary } = useLogsSummary(searchSpaceId ? Number(searchSpaceId) : 0, 24, {
enablePolling: true,
refetchInterval: 5000,
});
// Use the custom hook for dialog state management
const {
isOpen,
@ -97,57 +90,35 @@ export const ConnectorIndicator: FC = () => {
setConnectorName,
} = useConnectorDialog();
// Fetch connectors using React Query with conditional refetchInterval
// This automatically refetches when mutations invalidate the cache (event-driven)
// and also polls when dialog is open to catch external changes
// Fetch connectors using Electric SQL + PGlite for real-time updates
// This provides instant updates when connectors change, without polling
const {
data: connectors = [],
isLoading: connectorsLoading,
refetch: refreshConnectors,
} = useQuery({
queryKey: cacheKeys.connectors.all(searchSpaceId || ""),
queryFn: () =>
connectorsApiService.getConnectors({
queryParams: {
search_space_id: searchSpaceId ? Number(searchSpaceId) : undefined,
},
}),
enabled: !!searchSpaceId,
staleTime: 5 * 60 * 1000, // 5 minutes (same as connectorsAtom)
// Poll when dialog is open to catch external changes
refetchInterval: isOpen ? 5000 : false, // 5 seconds when open, no polling when closed
});
connectors: connectorsFromElectric = [],
loading: connectorsLoading,
error: connectorsError,
refreshConnectors: refreshConnectorsElectric,
} = useConnectorsElectric(searchSpaceId);
const queryClient = useQueryClient();
// Fallback to API if Electric fails or is not available
const connectors =
connectorsFromElectric.length > 0 || !connectorsError
? connectorsFromElectric
: allConnectors || [];
// Also refresh document type counts when dialog is open
useEffect(() => {
if (!isOpen || !searchSpaceId) return;
// Manual refresh function that works with both Electric and API
const refreshConnectors = async () => {
if (connectorsFromElectric.length > 0 || !connectorsError) {
await refreshConnectorsElectric();
} else {
// Fallback: use allConnectors from useConnectorDialog (which uses connectorsAtom)
// The connectorsAtom will handle refetching if needed
}
};
const POLL_INTERVAL = 5000; // 5 seconds, same as connectors
const intervalId = setInterval(() => {
// Invalidate document type counts to refresh active document types
queryClient.invalidateQueries({
queryKey: cacheKeys.documents.typeCounts(searchSpaceId),
});
}, POLL_INTERVAL);
// Cleanup interval on unmount or when dialog closes
return () => {
clearInterval(intervalId);
};
}, [isOpen, searchSpaceId, queryClient]);
// Get connector IDs that are currently being indexed
const indexingConnectorIds = useMemo(() => {
if (!logsSummary?.active_tasks) return new Set<number>();
return new Set(
logsSummary.active_tasks
.filter((task) => task.source?.includes("connector_indexing") && task.connector_id != null)
.map((task) => task.connector_id as number)
);
}, [logsSummary?.active_tasks]);
// Track indexing state locally - clears automatically when Electric SQL detects last_indexed_at changed
const { indexingConnectorIds, startIndexing } = useIndexingConnectors(
connectors as SearchSourceConnector[]
);
const isLoading = connectorsLoading || documentTypesLoading;
@ -163,8 +134,9 @@ export const ConnectorIndicator: FC = () => {
const activeConnectorsCount = connectors.length;
// Check which connectors are already connected
// Using Electric SQL + PGlite for real-time connector updates
const connectedTypes = new Set(
(allConnectors || []).map((c: SearchSourceConnector) => c.connector_type)
(connectors || []).map((c: SearchSourceConnector) => c.connector_type)
);
if (!searchSpaceId) return null;
@ -221,9 +193,8 @@ export const ConnectorIndicator: FC = () => {
<ConnectorAccountsListView
connectorType={viewingAccountsType.connectorType}
connectorTitle={viewingAccountsType.connectorTitle}
connectors={(allConnectors || []) as SearchSourceConnector[]}
connectors={(connectors || []) as SearchSourceConnector[]} // Using Electric SQL + PGlite for real-time connector updates (all connector types)
indexingConnectorIds={indexingConnectorIds}
logsSummary={logsSummary}
onBack={handleBackFromAccountsList}
onManage={handleStartEdit}
onAddAccount={() => {
@ -239,7 +210,7 @@ export const ConnectorIndicator: FC = () => {
) : connectingConnectorType ? (
<ConnectorConnectView
connectorType={connectingConnectorType}
onSubmit={handleSubmitConnectForm}
onSubmit={(formData) => handleSubmitConnectForm(formData, startIndexing)}
onBack={handleBackFromConnect}
isSubmitting={isCreatingConnector}
/>
@ -263,13 +234,18 @@ export const ConnectorIndicator: FC = () => {
onEndDateChange={setEndDate}
onPeriodicEnabledChange={setPeriodicEnabled}
onFrequencyChange={setFrequencyMinutes}
onSave={() => handleSaveConnector(() => refreshConnectors())}
onSave={() => {
startIndexing(editingConnector.id);
handleSaveConnector(() => refreshConnectors());
}}
onDisconnect={() => handleDisconnectConnector(() => refreshConnectors())}
onBack={handleBackFromEdit}
onQuickIndex={
editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR"
? () =>
handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type)
? () => {
startIndexing(editingConnector.id);
handleQuickIndexConnector(editingConnector.id, editingConnector.connector_type);
}
: undefined
}
onConfigChange={setConnectorConfig}
@ -296,7 +272,12 @@ export const ConnectorIndicator: FC = () => {
onPeriodicEnabledChange={setPeriodicEnabled}
onFrequencyChange={setFrequencyMinutes}
onConfigChange={setIndexingConnectorConfig}
onStartIndexing={() => handleStartIndexing(() => refreshConnectors())}
onStartIndexing={() => {
if (indexingConfig.connectorId) {
startIndexing(indexingConfig.connectorId);
}
handleStartIndexing(() => refreshConnectors());
}}
onSkip={handleSkipIndexing}
/>
) : (
@ -325,10 +306,9 @@ export const ConnectorIndicator: FC = () => {
searchSpaceId={searchSpaceId}
connectedTypes={connectedTypes}
connectingId={connectingId}
allConnectors={allConnectors}
allConnectors={connectors}
documentTypeCounts={documentTypeCounts}
indexingConnectorIds={indexingConnectorIds}
logsSummary={logsSummary}
onConnectOAuth={handleConnectOAuth}
onConnectNonOAuth={handleConnectNonOAuth}
onCreateWebcrawler={handleCreateWebcrawler}
@ -345,7 +325,6 @@ export const ConnectorIndicator: FC = () => {
activeDocumentTypes={activeDocumentTypes}
connectors={connectors as SearchSourceConnector[]}
indexingConnectorIds={indexingConnectorIds}
logsSummary={logsSummary}
searchSpaceId={searchSpaceId}
onTabChange={handleTabChange}
onManage={handleStartEdit}

112
surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx
View file

@ -1,12 +1,10 @@
"use client";
import { IconBrandYoutube } from "@tabler/icons-react";
import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns";
import { FileText, Loader2 } from "lucide-react";
import type { FC } from "react";
import { Button } from "@/components/ui/button";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { LogActiveTask } from "@/contracts/types/log.types";
import { cn } from "@/lib/utils";
import { useConnectorStatus } from "../hooks/use-connector-status";
import { ConnectorStatusBadge } from "./connector-status-badge";
@ -20,34 +18,11 @@ interface ConnectorCardProps {
isConnecting?: boolean;
documentCount?: number;
accountCount?: number;
lastIndexedAt?: string | null;
isIndexing?: boolean;
activeTask?: LogActiveTask;
onConnect?: () => void;
onManage?: () => void;
}
/**
* Check if a connector type is indexable (has documents)
* MCP connectors are tools only and don't have indexable content
*/
function isIndexableConnector(connectorType?: string): boolean {
if (!connectorType) return true; // Default to true for unknown types
const nonIndexableTypes = ["MCP_CONNECTOR"];
return !nonIndexableTypes.includes(connectorType);
}
/**
* Extract a number from the active task message for display
* Looks for patterns like "45 indexed", "Processing 123", etc.
*/
function extractIndexedCount(message: string | undefined): number | null {
if (!message) return null;
// Try to find a number in the message
const match = message.match(/(\d+)/);
return match ? parseInt(match[1], 10) : null;
}
/**
* Format document count (e.g., "1.2k docs", "500 docs", "1.5M docs")
*/
@ -62,45 +37,6 @@ function formatDocumentCount(count: number | undefined): string {
return `${m.replace(/\.0$/, "")}M docs`;
}
/**
* Format last indexed date with contextual messages
* Examples: "Just now", "10 minutes ago", "Today at 2:30 PM", "Yesterday at 3:45 PM", "3 days ago", "Jan 15, 2026"
*/
function formatLastIndexedDate(dateString: string): string {
const date = new Date(dateString);
const now = new Date();
const minutesAgo = differenceInMinutes(now, date);
const daysAgo = differenceInDays(now, date);
// Just now (within last minute)
if (minutesAgo < 1) {
return "Just now";
}
// X minutes ago (less than 1 hour)
if (minutesAgo < 60) {
return `${minutesAgo} ${minutesAgo === 1 ? "minute" : "minutes"} ago`;
}
// Today at [time]
if (isToday(date)) {
return `Today at ${format(date, "h:mm a")}`;
}
// Yesterday at [time]
if (isYesterday(date)) {
return `Yesterday at ${format(date, "h:mm a")}`;
}
// X days ago (less than 7 days)
if (daysAgo < 7) {
return `${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago`;
}
// Full date for older entries
return format(date, "MMM d, yyyy");
}
export const ConnectorCard: FC<ConnectorCardProps> = ({
id,
title,
@ -110,9 +46,7 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
isConnecting = false,
documentCount,
accountCount,
lastIndexedAt,
isIndexing = false,
activeTask,
onConnect,
onManage,
}) => {
@ -125,41 +59,11 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
const statusMessage = getConnectorStatusMessage(connectorType);
const showWarnings = shouldShowWarnings();
// Extract count from active task message during indexing
const indexingCount = extractIndexedCount(activeTask?.message);
// Determine the status content to display
const getStatusContent = () => {
if (isIndexing) {
return (
<div className="flex items-center gap-2 w-full max-w-[200px]">
<span className="text-[11px] text-primary font-medium whitespace-nowrap">
{indexingCount !== null ? <>{indexingCount.toLocaleString()} indexed</> : "Syncing..."}
</span>
{/* Indeterminate progress bar with animation */}
<div className="relative flex-1 h-1 overflow-hidden rounded-full bg-primary/20">
<div className="absolute h-full bg-primary rounded-full animate-progress-indeterminate" />
</div>
</div>
);
}
if (isConnected) {
// For non-indexable connectors (like MCP), show description instead of index status
if (!isIndexableConnector(connectorType)) {
return description;
}
// Show last indexed date for connected indexable connectors
if (lastIndexedAt) {
return (
<span className="whitespace-nowrap text-[10px]">
Last indexed: {formatLastIndexedDate(lastIndexedAt)}
</span>
);
}
// Fallback for connected but never indexed
return <span className="whitespace-nowrap text-[10px]">Never indexed</span>;
// Don't show last indexed in overview tabs - only show in accounts list view
return null;
}
return description;
@ -201,9 +105,13 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
/>
)}
</div>
<div className="text-[10px] text-muted-foreground mt-1">{getStatusContent()}</div>
{isConnected && documentCount !== undefined && (
<p className="text-[10px] text-muted-foreground mt-0.5 flex items-center gap-1.5">
{isIndexing ? (
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
<Loader2 className="size-3 animate-spin" />
Syncing
</p>
) : isConnected ? (
<p className="text-[10px] text-muted-foreground mt-1 flex items-center gap-1.5">
<span>{formatDocumentCount(documentCount)}</span>
{accountCount !== undefined && accountCount > 0 && (
<>
@ -214,6 +122,8 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
</>
)}
</p>
) : (
<div className="text-[10px] text-muted-foreground mt-1">{getStatusContent()}</div>
)}
</div>
<Button

2
surfsense_web/components/assistant-ui/connector-popup/components/connector-status-badge.tsx
View file

@ -44,7 +44,7 @@ export const ConnectorStatusBadge: FC<ConnectorStatusBadgeProps> = ({
case "deprecated":
return {
icon: AlertTriangle,
className: "ext-slate-500 dark:text-slate-400",
className: "text-slate-500 dark:text-slate-400",
defaultTitle: "Deprecated",
};
default:

2
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx
View file

@ -136,7 +136,7 @@ export const ConnectorConnectView: FC<ConnectorConnectViewProps> = ({
{isSubmitting ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
Connecting...
Connecting
</>
) : (
<>{connectorType === "MCP_CONNECTOR" ? "Connect" : `Connect ${getConnectorTypeDisplay(connectorType)}`}</>

4
surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
View file

@ -172,7 +172,7 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
{isQuickIndexing || isIndexing ? (
<>
<RefreshCw className="mr-2 h-4 w-4 animate-spin" />
Indexing...
Syncing
</>
) : (
<>
@ -281,7 +281,7 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
{isDisconnecting ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
Disconnecting...
Disconnecting
</>
) : (
"Confirm Disconnect"

36
surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts
View file

@ -508,20 +508,23 @@ export const useConnectorDialog = () => {
// Handle submitting connect form
const handleSubmitConnectForm = useCallback(
async (formData: {
name: string;
connector_type: string;
config: Record<string, unknown>;
is_indexable: boolean;
last_indexed_at: null;
periodic_indexing_enabled: boolean;
indexing_frequency_minutes: number | null;
next_scheduled_at: null;
startDate?: Date;
endDate?: Date;
periodicEnabled?: boolean;
frequencyMinutes?: string;
}) => {
async (
formData: {
name: string;
connector_type: string;
config: Record<string, unknown>;
is_indexable: boolean;
last_indexed_at: null;
periodic_indexing_enabled: boolean;
indexing_frequency_minutes: number | null;
next_scheduled_at: null;
startDate?: Date;
endDate?: Date;
periodicEnabled?: boolean;
frequencyMinutes?: string;
},
onIndexingStart?: (connectorId: number) => void
) => {
if (!searchSpaceId || !connectingConnectorType) return;
// Prevent multiple submissions using ref for immediate check
@ -621,6 +624,11 @@ export const useConnectorDialog = () => {
});
}
// Notify caller that indexing is starting (for UI syncing state)
if (onIndexingStart) {
onIndexingStart(connector.id);
}
// Start indexing (backend will use defaults if dates are undefined)
const startDateStr = startDateForIndexing
? format(startDateForIndexing, "yyyy-MM-dd")

41
surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-status.ts
View file

@ -1,6 +1,6 @@
"use client";
import { useMemo } from "react";
import { useCallback, useMemo } from "react";
import {
type ConnectorStatusConfig,
connectorStatusConfig,
@ -14,34 +14,43 @@ export function useConnectorStatus() {
/**
* Get status configuration for a specific connector type
*/
const getConnectorStatus = (connectorType: string | undefined): ConnectorStatusConfig => {
if (!connectorType) {
return getDefaultConnectorStatus();
}
const getConnectorStatus = useCallback(
(connectorType: string | undefined): ConnectorStatusConfig => {
if (!connectorType) {
return getDefaultConnectorStatus();
}
return connectorStatusConfig.connectorStatuses[connectorType] || getDefaultConnectorStatus();
};
return connectorStatusConfig.connectorStatuses[connectorType] || getDefaultConnectorStatus();
},
[]
);
/**
* Check if a connector is enabled
*/
const isConnectorEnabled = (connectorType: string | undefined): boolean => {
return getConnectorStatus(connectorType).enabled;
};
const isConnectorEnabled = useCallback(
(connectorType: string | undefined): boolean => {
return getConnectorStatus(connectorType).enabled;
},
[getConnectorStatus]
);
/**
* Get status message for a connector
*/
const getConnectorStatusMessage = (connectorType: string | undefined): string | null => {
return getConnectorStatus(connectorType).statusMessage || null;
};
const getConnectorStatusMessage = useCallback(
(connectorType: string | undefined): string | null => {
return getConnectorStatus(connectorType).statusMessage || null;
},
[getConnectorStatus]
);
/**
* Check if warnings should be shown globally
*/
const shouldShowWarnings = (): boolean => {
const shouldShowWarnings = useCallback((): boolean => {
return connectorStatusConfig.globalSettings.showWarnings;
};
}, []);
return useMemo(
() => ({
@ -50,6 +59,6 @@ export function useConnectorStatus() {
getConnectorStatusMessage,
shouldShowWarnings,
}),
[]
[getConnectorStatus, isConnectorEnabled, getConnectorStatusMessage, shouldShowWarnings]
);
}

81
surfsense_web/components/assistant-ui/connector-popup/hooks/use-indexing-connectors.ts Normal file
View file

@ -0,0 +1,81 @@
"use client";
import { useCallback, useEffect, useRef, useState } from "react";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
/**
* Hook to track which connectors are currently indexing using local state.
*
* This provides a better UX than polling by:
* 1. Setting indexing state immediately when user triggers indexing (optimistic)
* 2. Clearing indexing state when Electric SQL detects last_indexed_at changed
*
* The actual `last_indexed_at` value comes from Electric SQL/PGlite, not local state.
*/
export function useIndexingConnectors(connectors: SearchSourceConnector[]) {
// Set of connector IDs that are currently indexing
const [indexingConnectorIds, setIndexingConnectorIds] = useState<Set<number>>(new Set());
// Track previous last_indexed_at values to detect changes
const previousLastIndexedAtRef = useRef<Map<number, string | null>>(new Map());
// Detect when last_indexed_at changes (indexing completed) via Electric SQL
useEffect(() => {
const previousValues = previousLastIndexedAtRef.current;
const newIndexingIds = new Set(indexingConnectorIds);
let hasChanges = false;
for (const connector of connectors) {
const previousValue = previousValues.get(connector.id);
const currentValue = connector.last_indexed_at;
// If last_indexed_at changed and connector was in indexing state, clear it
if (
previousValue !== undefined && // We've seen this connector before
previousValue !== currentValue && // Value changed
indexingConnectorIds.has(connector.id) // It was marked as indexing
) {
newIndexingIds.delete(connector.id);
hasChanges = true;
}
// Update previous value tracking
previousValues.set(connector.id, currentValue);
}
if (hasChanges) {
setIndexingConnectorIds(newIndexingIds);
}
}, [connectors, indexingConnectorIds]);
// Add a connector to the indexing set (called when indexing starts)
const startIndexing = useCallback((connectorId: number) => {
setIndexingConnectorIds((prev) => {
const next = new Set(prev);
next.add(connectorId);
return next;
});
}, []);
// Remove a connector from the indexing set (called manually if needed)
const stopIndexing = useCallback((connectorId: number) => {
setIndexingConnectorIds((prev) => {
const next = new Set(prev);
next.delete(connectorId);
return next;
});
}, []);
// Check if a connector is currently indexing
const isIndexing = useCallback(
(connectorId: number) => indexingConnectorIds.has(connectorId),
[indexingConnectorIds]
);
return {
indexingConnectorIds,
startIndexing,
stopIndexing,
isIndexing,
};
}

123
surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx
View file

@ -1,6 +1,5 @@
"use client";
import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns";
import { ArrowRight, Cable, Loader2 } from "lucide-react";
import { useRouter } from "next/navigation";
import type { FC } from "react";
@ -24,7 +23,6 @@ interface ActiveConnectorsTabProps {
activeDocumentTypes: Array<[string, number]>;
connectors: SearchSourceConnector[];
indexingConnectorIds: Set<number>;
logsSummary: LogSummary | undefined;
searchSpaceId: string;
onTabChange: (value: string) => void;
onManage?: (connector: SearchSourceConnector) => void;
@ -45,7 +43,6 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
activeDocumentTypes,
connectors,
indexingConnectorIds,
logsSummary,
searchSpaceId,
onTabChange,
onManage,
@ -78,32 +75,6 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
return `${m.replace(/\.0$/, "")}M docs`;
};
// Format last indexed date with contextual messages
const formatLastIndexedDate = (dateString: string): string => {
const date = new Date(dateString);
const now = new Date();
const minutesAgo = differenceInMinutes(now, date);
const daysAgo = differenceInDays(now, date);
if (minutesAgo < 1) return "Just now";
if (minutesAgo < 60) return `${minutesAgo} ${minutesAgo === 1 ? "minute" : "minutes"} ago`;
if (isToday(date)) return `Today at ${format(date, "h:mm a")}`;
if (isYesterday(date)) return `Yesterday at ${format(date, "h:mm a")}`;
if (daysAgo < 7) return `${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago`;
return format(date, "MMM d, yyyy");
};
// Get most recent last indexed date from a list of connectors
const getMostRecentLastIndexed = (
connectorsList: SearchSourceConnector[]
): string | undefined => {
return connectorsList.reduce<string | undefined>((latest, c) => {
if (!c.last_indexed_at) return latest;
if (!latest) return c.last_indexed_at;
return new Date(c.last_indexed_at) > new Date(latest) ? c.last_indexed_at : latest;
}, undefined);
};
// Document types that should be shown as standalone cards (not from connectors)
const standaloneDocumentTypes = ["EXTENSION", "FILE", "NOTE", "YOUTUBE_VIDEO", "CRAWLED_URL"];
@ -202,24 +173,25 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
const documentCount = getDocumentCountForConnector(
connectorType,
documentTypeCounts
);
const accountCount = typeConnectors.length;
const mostRecentLastIndexed = getMostRecentLastIndexed(typeConnectors);
const handleManageClick = () => {
if (onViewAccountsList) {
onViewAccountsList(connectorType, title);
} else if (onManage && typeConnectors[0]) {
onManage(typeConnectors[0]);
}
};
);
const accountCount = typeConnectors.length;
const handleManageClick = () => {
if (onViewAccountsList) {
onViewAccountsList(connectorType, title);
} else if (onManage && typeConnectors[0]) {
onManage(typeConnectors[0]);
}
};
return (
<div
key={`oauth-type-${connectorType}`}
className={cn(
"relative flex items-center gap-4 p-4 rounded-xl border border-border transition-all",
"relative flex items-center gap-4 p-4 rounded-xl transition-all",
isAnyIndexing
? "bg-primary/5 border-primary/20"
: "bg-slate-400/5 dark:bg-white/5 hover:bg-slate-400/10 dark:hover:bg-white/10"
? "bg-primary/5 border-0"
: "bg-slate-400/5 dark:bg-white/5 hover:bg-slate-400/10 dark:hover:bg-white/10 border border-border"
)}
>
<div
@ -237,30 +209,17 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
{isAnyIndexing ? (
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
<Loader2 className="size-3 animate-spin" />
Indexing...
Syncing
</p>
) : (
<>
{isIndexableConnector(connectorType) && (
<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap">
{mostRecentLastIndexed
? `Last indexed: ${formatLastIndexedDate(mostRecentLastIndexed)}`
: "Never indexed"}
</p>
)}
</>
<p className="text-[10px] text-muted-foreground mt-1 flex items-center gap-1.5">
<span>{formatDocumentCount(documentCount)}</span>
<span className="text-muted-foreground/50">•</span>
<span>
{accountCount} {accountCount === 1 ? "Account" : "Accounts"}
</span>
</p>
)}
<p className="text-[10px] text-muted-foreground mt-0.5 flex items-center gap-1.5">
{isIndexableConnector(connectorType) && (
<>
<span>{formatDocumentCount(documentCount)}</span>
<span className="text-muted-foreground/50">•</span>
</>
)}
<span>
{accountCount} {accountCount === 1 ? "Account" : "Accounts"}
</span>
</p>
</div>
<Button
variant="secondary"
@ -277,9 +236,6 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
{/* Non-OAuth Connectors - Individual Cards */}
{filteredNonOAuthConnectors.map((connector) => {
const isIndexing = indexingConnectorIds.has(connector.id);
const activeTask = logsSummary?.active_tasks?.find(
(task: LogActiveTask) => task.connector_id === connector.id
);
const documentCount = getDocumentCountForConnector(
connector.connector_type,
documentTypeCounts
@ -288,10 +244,10 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
<div
key={`connector-${connector.id}`}
className={cn(
"flex items-center gap-4 p-4 rounded-xl border border-border transition-all",
"flex items-center gap-4 p-4 rounded-xl transition-all",
isIndexing
? "bg-primary/5 border-primary/20"
: "bg-slate-400/5 dark:bg-white/5 hover:bg-slate-400/10 dark:hover:bg-white/10"
? "bg-primary/5 border-0"
: "bg-slate-400/5 dark:bg-white/5 hover:bg-slate-400/10 dark:hover:bg-white/10 border border-border"
)}
>
<div
@ -313,26 +269,10 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
{isIndexing ? (
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
<Loader2 className="size-3 animate-spin" />
Indexing...
{activeTask?.message && (
<span className="text-muted-foreground truncate max-w-[150px]">
• {activeTask.message}
</span>
)}
Syncing
</p>
) : (
<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap">
{isIndexableConnector(connector.connector_type)
? connector.last_indexed_at
? `Last indexed: ${formatLastIndexedDate(connector.last_indexed_at)}`
: "Never indexed"
: connector.connector_type === "MCP_CONNECTOR"
? ""
: "Active"}
</p>
)}
{isIndexableConnector(connector.connector_type) && (
<p className="text-[10px] text-muted-foreground mt-0.5">
<p className="text-[10px] text-muted-foreground mt-1">
{formatDocumentCount(documentCount)}
</p>
)}
@ -389,19 +329,12 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
) : (
<div className="flex flex-col items-center justify-center py-20 text-center">
<div className="flex h-16 w-16 items-center justify-center rounded-full bg-muted mb-4">
<Cable className="size-8 text-muted-foreground/50" />
<Cable className="size-8 text-muted-foreground" />
</div>
<h4 className="text-lg font-semibold">No active sources</h4>
<p className="text-sm text-muted-foreground mt-1 max-w-[280px]">
Connect your first service to start searching across all your data.
</p>
<Button
variant="link"
className="mt-6 text-primary hover:underline"
onClick={() => onTabChange("all")}
>
Browse available connectors
</Button>
</div>
)}
</TabsContent>

30
surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx
View file

@ -1,10 +1,7 @@
"use client";
import { Plus } from "lucide-react";
import type { FC } from "react";
import { Button } from "@/components/ui/button";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
import { ConnectorCard } from "../components/connector-card";
import { CRAWLERS, OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants";
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
@ -30,7 +27,6 @@ interface AllConnectorsTabProps {
allConnectors: SearchSourceConnector[] | undefined;
documentTypeCounts?: Record<string, number>;
indexingConnectorIds?: Set<number>;
logsSummary?: LogSummary;
onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[number]) => void;
onConnectNonOAuth?: (connectorType: string) => void;
onCreateWebcrawler?: () => void;
@ -41,13 +37,11 @@ interface AllConnectorsTabProps {
export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
searchQuery,
searchSpaceId,
connectedTypes,
connectingId,
allConnectors,
documentTypeCounts,
indexingConnectorIds,
logsSummary,
onConnectOAuth,
onConnectNonOAuth,
onCreateWebcrawler,
@ -55,14 +49,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
onManage,
onViewAccountsList,
}) => {
// Helper to find active task for a connector
const getActiveTaskForConnector = (connectorId: number): LogActiveTask | undefined => {
if (!logsSummary?.active_tasks) return undefined;
return logsSummary.active_tasks.find(
(task: LogActiveTask) => task.connector_id === connectorId
);
};
// Filter connectors based on search
const filteredOAuth = OAUTH_CONNECTORS.filter(
(c) =>
@ -125,11 +111,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
// Check if any account is currently indexing
const isIndexing = typeConnectors.some((c) => indexingConnectorIds?.has(c.id));
// Get active task from any indexing account
const activeTask = typeConnectors
.map((c) => getActiveTaskForConnector(c.id))
.find((task) => task !== undefined);
return (
<ConnectorCard
key={connector.id}
@ -143,7 +124,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
accountCount={accountCount}
lastIndexedAt={mostRecentLastIndexed}
isIndexing={isIndexing}
activeTask={activeTask}
onConnect={() => onConnectOAuth(connector)}
onManage={
isConnected && onViewAccountsList
@ -181,9 +161,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
documentTypeCounts
);
const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id);
const activeTask = actualConnector
? getActiveTaskForConnector(actualConnector.id)
: undefined;
const handleConnect = onConnectNonOAuth
? () => onConnectNonOAuth(connector.connectorType)
@ -199,9 +176,7 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
isConnected={isConnected}
isConnecting={isConnecting}
documentCount={documentCount}
lastIndexedAt={actualConnector?.last_indexed_at}
isIndexing={isIndexing}
activeTask={activeTask}
onConnect={handleConnect}
onManage={
actualConnector && onManage ? () => onManage(actualConnector) : undefined
@ -242,9 +217,6 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
? getDocumentCountForConnector(crawler.connectorType, documentTypeCounts)
: undefined;
const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id);
const activeTask = actualConnector
? getActiveTaskForConnector(actualConnector.id)
: undefined;
const handleConnect =
isYouTube && onCreateYouTubeCrawler
@ -269,9 +241,7 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
isConnected={isConnected}
isConnecting={isConnecting}
documentCount={documentCount}
lastIndexedAt={actualConnector?.last_indexed_at}
isIndexing={isIndexing}
activeTask={activeTask}
onConnect={handleConnect}
onManage={
actualConnector && onManage ? () => onManage(actualConnector) : undefined

113
surfsense_web/components/assistant-ui/connector-popup/utils/mcp-config-validator.ts Normal file
View file

@ -0,0 +1,113 @@
import type { MCPServerConfig, MCPToolDefinition } from "@/contracts/types/mcp.types";
import { connectorsApiService } from "@/lib/apis/connectors-api.service";
/**
* Shared MCP configuration validation result
*/
export interface MCPConfigValidationResult {
config: MCPServerConfig | null;
error: string | null;
}
/**
* Shared MCP connection test result
*/
export interface MCPConnectionTestResult {
status: "success" | "error";
message: string;
tools: MCPToolDefinition[];
}
/**
* Parse and validate MCP server configuration from JSON string
* @param configJson - JSON string containing MCP server configuration
* @returns Validation result with parsed config or error message
*/
export const parseMCPConfig = (configJson: string): MCPConfigValidationResult => {
try {
const parsed = JSON.parse(configJson);
// Validate that it's an object, not an array
if (Array.isArray(parsed)) {
return {
config: null,
error: "Please provide a single server configuration object, not an array",
};
}
// Validate required fields
if (!parsed.command || typeof parsed.command !== "string") {
return {
config: null,
error: "'command' field is required and must be a string",
};
}
const config: MCPServerConfig = {
command: parsed.command,
args: parsed.args || [],
env: parsed.env || {},
transport: parsed.transport || "stdio",
};
return {
config,
error: null,
};
} catch (error) {
return {
config: null,
error: error instanceof Error ? error.message : "Invalid JSON",
};
}
};
/**
* Test connection to MCP server
* @param serverConfig - MCP server configuration to test
* @returns Connection test result with status, message, and available tools
*/
export const testMCPConnection = async (
serverConfig: MCPServerConfig
): Promise<MCPConnectionTestResult> => {
try {
const result = await connectorsApiService.testMCPConnection(serverConfig);
if (result.status === "success") {
return {
status: "success",
message: `Successfully connected. Found ${result.tools.length} tool${result.tools.length !== 1 ? "s" : ""}.`,
tools: result.tools,
};
}
return {
status: "error",
message: result.message || "Failed to connect",
tools: [],
};
} catch (error) {
return {
status: "error",
message: error instanceof Error ? error.message : "Failed to connect",
tools: [],
};
}
};
/**
* Extract server name from MCP config JSON
* @param configJson - JSON string containing MCP server configuration
* @returns Server name if found, otherwise default name
*/
export const extractServerName = (configJson: string): string => {
try {
const parsed = JSON.parse(configJson);
if (parsed.name && typeof parsed.name === "string") {
return parsed.name;
}
} catch {
// Return default if parsing fails
}
return "MCP Server";
};

21
surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx
View file

@ -6,7 +6,6 @@ import type { FC } from "react";
import { Button } from "@/components/ui/button";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
import { cn } from "@/lib/utils";
import { useConnectorStatus } from "../hooks/use-connector-status";
import { getConnectorDisplayName } from "../tabs/all-connectors-tab";
@ -16,7 +15,6 @@ interface ConnectorAccountsListViewProps {
connectorTitle: string;
connectors: SearchSourceConnector[];
indexingConnectorIds: Set<number>;
logsSummary: LogSummary | undefined;
onBack: () => void;
onManage: (connector: SearchSourceConnector) => void;
onAddAccount: () => void;
@ -68,7 +66,6 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
connectorTitle,
connectors,
indexingConnectorIds,
logsSummary,
onBack,
onManage,
onAddAccount,
@ -133,7 +130,7 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
)}
</div>
<span className="text-[11px] sm:text-[12px] font-medium">
{isConnecting ? "Connecting..." : "Add Account"}
{isConnecting ? "Connecting" : "Add Account"}
</span>
</button>
</div>
@ -145,18 +142,15 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">
{typeConnectors.map((connector) => {
const isIndexing = indexingConnectorIds.has(connector.id);
const activeTask = logsSummary?.active_tasks?.find(
(task: LogActiveTask) => task.connector_id === connector.id
);
return (
<div
key={connector.id}
className={cn(
"flex items-center gap-4 p-4 rounded-xl border border-border transition-all",
"flex items-center gap-4 p-4 rounded-xl transition-all",
isIndexing
? "bg-primary/5 border-primary/20"
: "bg-slate-400/5 dark:bg-white/5 hover:bg-slate-400/10 dark:hover:bg-white/10"
? "bg-primary/5 border-0"
: "bg-slate-400/5 dark:bg-white/5 hover:bg-slate-400/10 dark:hover:bg-white/10 border border-border"
)}
>
<div
@ -176,12 +170,7 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
{isIndexing ? (
<p className="text-[11px] text-primary mt-1 flex items-center gap-1.5">
<Loader2 className="size-3 animate-spin" />
Indexing...
{activeTask?.message && (
<span className="text-muted-foreground truncate max-w-[100px]">
• {activeTask.message}
</span>
)}
Syncing
</p>
) : (
<p className="text-[10px] text-muted-foreground mt-1 whitespace-nowrap truncate">

3
surfsense_web/components/assistant-ui/document-upload-popup.tsx
View file

@ -2,7 +2,6 @@
import { useAtomValue } from "jotai";
import { Upload } from "lucide-react";
import { useRouter } from "next/navigation";
import {
createContext,
type FC,
@ -85,13 +84,11 @@ const DocumentUploadPopupContent: FC<{
onOpenChange: (open: boolean) => void;
}> = ({ isOpen, onOpenChange }) => {
const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom);
const router = useRouter();
if (!searchSpaceId) return null;
const handleSuccess = () => {
onOpenChange(false);
router.push(`/dashboard/${searchSpaceId}/documents`);
};
return (

71
surfsense_web/components/assistant-ui/thread-welcome.tsx
View file

@ -1,71 +0,0 @@
import { useAtomValue } from "jotai";
import type { FC } from "react";
import { useMemo } from "react";
import { currentUserAtom } from "@/atoms/user/user-query.atoms";
import { Composer } from "@/components/assistant-ui/composer";
const getTimeBasedGreeting = (userEmail?: string): string => {
const hour = new Date().getHours();
// Extract first name from email if available
const firstName = userEmail
? userEmail.split("@")[0].split(".")[0].charAt(0).toUpperCase() +
userEmail.split("@")[0].split(".")[0].slice(1)
: null;
// Array of greeting variations for each time period
const morningGreetings = ["Good morning", "Fresh start today", "Morning", "Hey there"];
const afternoonGreetings = ["Good afternoon", "Afternoon", "Hey there", "Hi there"];
const eveningGreetings = ["Good evening", "Evening", "Hey there", "Hi there"];
const nightGreetings = ["Good night", "Evening", "Hey there", "Winding down"];
const lateNightGreetings = ["Still up", "Night owl mode", "Up past bedtime", "Hi there"];
// Select a random greeting based on time
let greeting: string;
if (hour < 5) {
// Late night: midnight to 5 AM
greeting = lateNightGreetings[Math.floor(Math.random() * lateNightGreetings.length)];
} else if (hour < 12) {
greeting = morningGreetings[Math.floor(Math.random() * morningGreetings.length)];
} else if (hour < 18) {
greeting = afternoonGreetings[Math.floor(Math.random() * afternoonGreetings.length)];
} else if (hour < 22) {
greeting = eveningGreetings[Math.floor(Math.random() * eveningGreetings.length)];
} else {
// Night: 10 PM to midnight
greeting = nightGreetings[Math.floor(Math.random() * nightGreetings.length)];
}
// Add personalization with first name if available
if (firstName) {
return `${greeting}, ${firstName}!`;
}
return `${greeting}!`;
};
export const ThreadWelcome: FC = () => {
const { data: user } = useAtomValue(currentUserAtom);
// Memoize greeting so it doesn't change on re-renders (only on user change)
const greeting = useMemo(() => getTimeBasedGreeting(user?.email), [user?.email]);
return (
<div className="aui-thread-welcome-root mx-auto flex w-full max-w-(--thread-max-width) grow flex-col items-center px-4 relative">
{/* Greeting positioned above the composer - fixed position */}
<div className="aui-thread-welcome-message absolute bottom-[calc(50%+5rem)] left-0 right-0 flex flex-col items-center text-center">
<h1 className="aui-thread-welcome-message-inner fade-in slide-in-from-bottom-2 animate-in text-3xl md:text-5xl delay-100 duration-500 ease-out fill-mode-both">
{greeting}
</h1>
</div>
{/* Composer - top edge fixed, expands downward only */}
<div className="fade-in slide-in-from-bottom-3 animate-in delay-200 duration-500 ease-out fill-mode-both w-full flex items-start justify-center absolute top-[calc(50%-3.5rem)] left-0 right-0">
<Composer />
</div>
</div>
);
};

23
surfsense_web/components/assistant-ui/thread.tsx
View file

@ -124,14 +124,23 @@ const ThreadScrollToBottom: FC = () => {
);
};
const getTimeBasedGreeting = (userEmail?: string): string => {
const getTimeBasedGreeting = (user?: { display_name?: string | null; email?: string }): string => {
const hour = new Date().getHours();
// Extract first name from email if available
const firstName = userEmail
? userEmail.split("@")[0].split(".")[0].charAt(0).toUpperCase() +
userEmail.split("@")[0].split(".")[0].slice(1)
: null;
// Extract first name: prefer display_name, fall back to email extraction
let firstName: string | null = null;
if (user?.display_name?.trim()) {
// Use display_name if available and not empty
// Extract first name from display_name (take first word)
const nameParts = user.display_name.trim().split(/\s+/);
firstName = nameParts[0].charAt(0).toUpperCase() + nameParts[0].slice(1).toLowerCase();
} else if (user?.email) {
// Fall back to email extraction if display_name is not available
firstName =
user.email.split("@")[0].split(".")[0].charAt(0).toUpperCase() +
user.email.split("@")[0].split(".")[0].slice(1);
}
// Array of greeting variations for each time period
const morningGreetings = ["Good morning", "Fresh start today", "Morning", "Hey there"];
@ -172,7 +181,7 @@ const ThreadWelcome: FC = () => {
const { data: user } = useAtomValue(currentUserAtom);
// Memoize greeting so it doesn't change on re-renders (only on user change)
const greeting = useMemo(() => getTimeBasedGreeting(user?.email), [user?.email]);
const greeting = useMemo(() => getTimeBasedGreeting(user), [user]);
return (
<div className="aui-thread-welcome-root mx-auto flex w-full max-w-(--thread-max-width) grow flex-col items-center px-4 relative">

7
surfsense_web/components/assistant-ui/tooltip-icon-button.tsx
View file

@ -27,12 +27,7 @@ export const TooltipIconButton = forwardRef<HTMLButtonElement, TooltipIconButton
<span className="aui-sr-only sr-only">{tooltip}</span>
</Button>
</TooltipTrigger>
<TooltipContent
side={side}
className="bg-black text-white font-medium shadow-xl px-3 py-1.5 dark:bg-zinc-800 dark:text-zinc-50 border-none"
>
{tooltip}
</TooltipContent>
<TooltipContent side={side}>{tooltip}</TooltipContent>
</Tooltip>
);
}

2
surfsense_web/components/homepage/hero-section.tsx
View file

@ -184,7 +184,7 @@ function GetStartedButton() {
return (
<motion.div whileHover={{ scale: 1.02, y: -2 }} whileTap={{ scale: 0.98 }}>
<Link
href="/login"
href="/register"
className="group relative z-20 flex h-11 w-full cursor-pointer items-center justify-center gap-2 rounded-xl bg-black px-6 py-2.5 text-sm font-semibold text-white shadow-lg transition-shadow duration-300 hover:shadow-xl sm:w-56 dark:bg-white dark:text-black"
>
Get Started

12
surfsense_web/components/layout/providers/LayoutDataProvider.tsx
View file

@ -21,6 +21,7 @@ import {
} from "@/components/ui/dialog";
import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service";
import { deleteThread, fetchThreads } from "@/lib/chat/thread-persistence";
import { cleanupElectric } from "@/lib/electric/client";
import { resetUser, trackLogout } from "@/lib/posthog/events";
import { cacheKeys } from "@/lib/query-client/cache-keys";
import type { ChatItem, NavItem, SearchSpace } from "../types/layout.types";
@ -278,10 +279,19 @@ export function LayoutDataProvider({
router.push(`/dashboard/${searchSpaceId}/team`);
}, [router, searchSpaceId]);
const handleLogout = useCallback(() => {
const handleLogout = useCallback(async () => {
try {
trackLogout();
resetUser();
// Best-effort cleanup of Electric SQL / PGlite
// Even if this fails, login-time cleanup will handle it
try {
await cleanupElectric();
} catch (err) {
console.warn("[Logout] Electric cleanup failed (will be handled on next login):", err);
}
if (typeof window !== "undefined") {
localStorage.removeItem("surfsense_bearer_token");
router.push("/");

4
surfsense_web/components/layout/ui/header/Header.tsx
View file

@ -3,6 +3,7 @@
import { Moon, Sun } from "lucide-react";
import { Button } from "@/components/ui/button";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { NotificationButton } from "@/components/notifications/NotificationButton";
interface HeaderProps {
breadcrumb?: React.ReactNode;
@ -29,6 +30,9 @@ export function Header({
{/* Right side - Actions */}
<div className="flex items-center gap-2">
{/* Notifications */}
<NotificationButton />
{/* Theme toggle */}
{onToggleTheme && (
<Tooltip>

5
surfsense_web/components/layout/ui/sidebar/AllPrivateChatsSidebar.tsx
View file

@ -34,7 +34,6 @@ import {
deleteThread,
fetchThreads,
searchThreads,
type ThreadListItem,
updateThread,
} from "@/lib/chat/thread-persistence";
import { cn } from "@/lib/utils";
@ -410,7 +409,7 @@ export function AllPrivateChatsSidebar({
</div>
) : isSearchMode ? (
<div className="text-center py-8">
<Search className="h-12 w-12 mx-auto text-muted-foreground/50 mb-3" />
<Search className="h-12 w-12 mx-auto text-muted-foreground mb-3" />
<p className="text-sm text-muted-foreground">
{t("no_chats_found") || "No chats found"}
</p>
@ -420,7 +419,7 @@ export function AllPrivateChatsSidebar({
</div>
) : (
<div className="text-center py-8">
<Lock className="h-12 w-12 mx-auto text-muted-foreground/50 mb-3" />
<Lock className="h-12 w-12 mx-auto text-muted-foreground mb-3" />
<p className="text-sm text-muted-foreground">
{showArchived
? t("no_archived_chats") || "No archived chats"

5
surfsense_web/components/layout/ui/sidebar/AllSharedChatsSidebar.tsx
View file

@ -34,7 +34,6 @@ import {
deleteThread,
fetchThreads,
searchThreads,
type ThreadListItem,
updateThread,
} from "@/lib/chat/thread-persistence";
import { cn } from "@/lib/utils";
@ -410,7 +409,7 @@ export function AllSharedChatsSidebar({
</div>
) : isSearchMode ? (
<div className="text-center py-8">
<Search className="h-12 w-12 mx-auto text-muted-foreground/50 mb-3" />
<Search className="h-12 w-12 mx-auto text-muted-foreground mb-3" />
<p className="text-sm text-muted-foreground">
{t("no_chats_found") || "No chats found"}
</p>
@ -420,7 +419,7 @@ export function AllSharedChatsSidebar({
</div>
) : (
<div className="text-center py-8">
<Users className="h-12 w-12 mx-auto text-muted-foreground/50 mb-3" />
<Users className="h-12 w-12 mx-auto text-muted-foreground mb-3" />
<p className="text-sm text-muted-foreground">
{showArchived
? t("no_archived_chats") || "No archived chats"

8
surfsense_web/components/layout/ui/sidebar/Sidebar.tsx
View file

@ -156,10 +156,10 @@ export function Sidebar({
<Button
variant="ghost"
size="icon"
className="h-5 w-5"
className="h-8 w-8 shrink-0 hover:bg-transparent hover:text-current focus-visible:ring-0"
onClick={onViewAllSharedChats}
>
<FolderOpen className="h-3.5 w-3.5" />
<FolderOpen className="h-4 w-4" />
</Button>
</TooltipTrigger>
<TooltipContent side="top">
@ -197,10 +197,10 @@ export function Sidebar({
<Button
variant="ghost"
size="icon"
className="h-5 w-5"
className="h-8 w-8 shrink-0 hover:bg-transparent hover:text-current focus-visible:ring-0"
onClick={onViewAllPrivateChats}
>
<FolderOpen className="h-3.5 w-3.5" />
<FolderOpen className="h-4 w-4" />
</Button>
</TooltipTrigger>
<TooltipContent side="top">

4
surfsense_web/components/layout/ui/sidebar/SidebarSection.tsx
View file

@ -37,14 +37,14 @@ export function SidebarSection({
{/* Action button - visible on hover (always visible on mobile) */}
{action && (
<div className="shrink-0 opacity-0 group-hover/section:opacity-100 transition-opacity pr-1 flex items-center gap-0.5">
<div className="shrink-0 opacity-0 group-hover/section:opacity-100 transition-opacity pr-1 flex items-center">
{action}
</div>
)}
{/* Persistent action - always visible */}
{persistentAction && (
<div className="shrink-0 pr-1 flex items-center gap-0.5">{persistentAction}</div>
<div className="shrink-0 pr-1 flex items-center">{persistentAction}</div>
)}
</div>

36
surfsense_web/components/new-chat/chat-share-button.tsx
View file

@ -1,7 +1,7 @@
"use client";
import { useQueryClient } from "@tanstack/react-query";
import { Globe, Loader2, Lock, Share2, Users } from "lucide-react";
import { Loader2, Lock, Share2, Users } from "lucide-react";
import { useCallback, useState } from "react";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
@ -92,8 +92,7 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS
variant="ghost"
size="sm"
className={cn(
"h-7 md:h-9 gap-1.5 md:gap-2 px-2 md:px-3 rounded-lg md:rounded-xl",
"border border-border/80 bg-background/50 backdrop-blur-sm",
"h-7 md:h-9 gap-1 md:gap-2 px-2 md:px-3 rounded-lg md:rounded-xl border border-border/80 bg-background/50 backdrop-blur-sm",
"hover:bg-muted/80 hover:border-border/30 transition-all duration-200",
"text-xs md:text-sm font-medium text-foreground",
"focus-visible:ring-0 focus-visible:ring-offset-0",
@ -104,7 +103,6 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS
<span className="hidden md:inline">
{currentVisibility === "PRIVATE" ? "Private" : "Shared"}
</span>
<Share2 className="size-3 md:size-3.5 text-muted-foreground" />
</Button>
</PopoverTrigger>
@ -113,25 +111,13 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS
align="end"
sideOffset={8}
>
<div className="p-3 md:p-4 border-b border-border/30">
<div className="flex items-center gap-2">
<Share2 className="size-4 md:size-5 text-primary" />
<div>
<h4 className="text-sm font-semibold">Share Chat</h4>
<p className="text-xs text-muted-foreground">
Control who can access this conversation
</p>
</div>
</div>
</div>
<div className="p-1.5 space-y-1">
{/* Updating overlay */}
{isUpdating && (
<div className="absolute inset-0 z-10 flex items-center justify-center bg-background/80 backdrop-blur-sm rounded-xl">
<div className="flex items-center gap-2 text-sm text-muted-foreground">
<Loader2 className="size-4 animate-spin" />
<span>Updating...</span>
<span>Updating</span>
</div>
</div>
)}
@ -149,8 +135,8 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS
className={cn(
"w-full flex items-start gap-2.5 px-2.5 py-2 rounded-md transition-all",
"hover:bg-accent/50 cursor-pointer",
"focus:outline-none focus:ring-2 focus:ring-primary/20",
isSelected && "bg-accent/80 ring-1 ring-primary/20"
"focus:outline-none",
isSelected && "bg-accent/80"
)}
>
<div
@ -185,18 +171,6 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS
);
})}
</div>
{/* Info footer */}
<div className="p-3 bg-muted/30 border-t border-border/30 rounded-b-xl">
<div className="flex items-start gap-2">
<Globe className="size-3.5 text-muted-foreground mt-0.5 shrink-0" />
<p className="text-[11px] text-muted-foreground leading-relaxed">
{currentVisibility === "PRIVATE"
? "This chat is private. Only you can view and interact with it."
: "This chat is shared. All search space members can view, continue the conversation, and delete it."}
</p>
</div>
</div>
</PopoverContent>
</Popover>
);

6
surfsense_web/components/new-chat/document-mention-picker.tsx
View file

@ -329,7 +329,7 @@ export const DocumentMentionPicker = forwardRef<
</div>
) : actualDocuments.length === 0 ? (
<div className="flex flex-col items-center justify-center py-4 text-center px-4">
<FileText className="h-5 w-5 text-muted-foreground/50 mb-1" />
<FileText className="h-5 w-5 text-muted-foreground/55 mb-1" />
<p className="text-sm text-muted-foreground">No documents found</p>
</div>
) : (
@ -337,7 +337,7 @@ export const DocumentMentionPicker = forwardRef<
{/* SurfSense Documentation Section */}
{surfsenseDocsList.length > 0 && (
<>
<div className="sticky top-0 z-10 px-3 py-2 text-xs font-bold uppercase tracking-wider bg-muted text-foreground/80 border-b border-border">
<div className="px-3 py-2 text-xs font-bold text-muted-foreground/55">
SurfSense Docs
</div>
{surfsenseDocsList.map((doc) => {
@ -385,7 +385,7 @@ export const DocumentMentionPicker = forwardRef<
{/* User Documents Section */}
{userDocsList.length > 0 && (
<>
<div className="sticky top-0 z-10 px-3 py-2 text-xs font-bold uppercase tracking-wider bg-muted text-foreground/80 border-b border-border">
<div className="px-3 py-2 text-xs font-bold text-muted-foreground/55">
Your Documents
</div>
{userDocsList.map((doc) => {

61
surfsense_web/components/notifications/NotificationButton.tsx Normal file
View file

@ -0,0 +1,61 @@
"use client";
import { Bell } from "lucide-react";
import { Button } from "@/components/ui/button";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { useNotifications } from "@/hooks/use-notifications";
import { useAtomValue } from "jotai";
import { currentUserAtom } from "@/atoms/user/user-query.atoms";
import { NotificationPopup } from "./NotificationPopup";
import { cn } from "@/lib/utils";
import { useParams } from "next/navigation";
export function NotificationButton() {
const { data: user } = useAtomValue(currentUserAtom);
const params = useParams();
const userId = user?.id ? String(user.id) : null;
// Get searchSpaceId from URL params - the component is rendered within /dashboard/[search_space_id]/
const searchSpaceId = params?.search_space_id ? Number(params.search_space_id) : null;
const { notifications, unreadCount, loading, markAsRead, markAllAsRead } = useNotifications(
userId,
searchSpaceId
);
return (
<Popover>
<Tooltip>
<TooltipTrigger asChild>
<PopoverTrigger asChild>
<Button variant="outline" size="icon" className="h-8 w-8 relative">
<Bell className="h-4 w-4" />
{unreadCount > 0 && (
<span
className={cn(
"absolute -top-1 -right-1 flex h-5 w-5 items-center justify-center rounded-full bg-black text-[10px] font-medium text-white dark:bg-zinc-800 dark:text-zinc-50",
unreadCount > 9 && "px-1"
)}
>
{unreadCount > 99 ? "99+" : unreadCount}
</span>
)}
<span className="sr-only">Notifications</span>
</Button>
</PopoverTrigger>
</TooltipTrigger>
<TooltipContent>Notifications</TooltipContent>
</Tooltip>
<PopoverContent align="end" className="w-80 p-0">
<NotificationPopup
notifications={notifications}
unreadCount={unreadCount}
loading={loading}
markAsRead={markAsRead}
markAllAsRead={markAllAsRead}
/>
</PopoverContent>
</Popover>
);
}

127
surfsense_web/components/notifications/NotificationPopup.tsx Normal file
View file

@ -0,0 +1,127 @@
"use client";
import { Bell, CheckCheck, Loader2, AlertCircle, CheckCircle2 } from "lucide-react";
import { Button } from "@/components/ui/button";
import { ScrollArea } from "@/components/ui/scroll-area";
import { Separator } from "@/components/ui/separator";
import type { Notification } from "@/hooks/use-notifications";
import { formatDistanceToNow } from "date-fns";
import { cn } from "@/lib/utils";
interface NotificationPopupProps {
notifications: Notification[];
unreadCount: number;
loading: boolean;
markAsRead: (id: number) => Promise<boolean>;
markAllAsRead: () => Promise<boolean>;
}
export function NotificationPopup({
notifications,
unreadCount,
loading,
markAsRead,
markAllAsRead,
}: NotificationPopupProps) {
const handleMarkAsRead = async (id: number) => {
await markAsRead(id);
};
const handleMarkAllAsRead = async () => {
await markAllAsRead();
};
const formatTime = (dateString: string) => {
try {
return formatDistanceToNow(new Date(dateString), { addSuffix: true });
} catch {
return "Recently";
}
};
const getStatusIcon = (notification: Notification) => {
const status = notification.metadata?.status as string | undefined;
switch (status) {
case "in_progress":
return <Loader2 className="h-4 w-4 text-foreground animate-spin" />;
case "completed":
return <CheckCircle2 className="h-4 w-4 text-green-500" />;
case "failed":
return <AlertCircle className="h-4 w-4 text-red-500" />;
default:
return <Bell className="h-4 w-4 text-muted-foreground" />;
}
};
return (
<div className="flex flex-col w-80 max-w-[calc(100vw-2rem)]">
{/* Header */}
<div className="flex items-center justify-between px-4 py-3 border-b">
<div className="flex items-center gap-2">
<h3 className="font-semibold text-sm">Notifications</h3>
</div>
{unreadCount > 0 && (
<Button variant="ghost" size="sm" onClick={handleMarkAllAsRead} className="h-7 text-xs">
<CheckCheck className="h-3.5 w-3.5 mr-0" />
Mark all read
</Button>
)}
</div>
{/* Notifications List */}
<ScrollArea className="h-[400px]">
{loading ? (
<div className="flex items-center justify-center py-8">
<Loader2 className="h-5 w-5 animate-spin text-foreground" />
</div>
) : notifications.length === 0 ? (
<div className="flex flex-col items-center justify-center py-8 px-4 text-center">
<Bell className="h-8 w-8 text-muted-foreground mb-2" />
<p className="text-sm text-muted-foreground">No notifications</p>
</div>
) : (
<div className="pt-0 pb-2">
{notifications.map((notification, index) => (
<div key={notification.id}>
<button
type="button"
onClick={() => !notification.read && handleMarkAsRead(notification.id)}
className={cn(
"w-full px-4 py-3 text-left hover:bg-accent transition-colors",
!notification.read && "bg-accent/50"
)}
>
<div className="flex items-start gap-3 overflow-hidden">
<div className="flex-shrink-0 mt-0.5">{getStatusIcon(notification)}</div>
<div className="flex-1 min-w-0 overflow-hidden">
<div className="flex items-start justify-between gap-2 mb-1">
<p
className={cn(
"text-xs font-medium break-all",
!notification.read && "font-semibold"
)}
>
{notification.title}
</p>
</div>
<p className="text-[11px] text-muted-foreground break-all line-clamp-2">
{notification.message}
</p>
<div className="flex items-center justify-between mt-2">
<span className="text-[10px] text-muted-foreground">
{formatTime(notification.created_at)}
</span>
</div>
</div>
</div>
</button>
{index < notifications.length - 1 && <Separator />}
</div>
))}
</div>
)}
</ScrollArea>
</div>
);
}

132
surfsense_web/components/providers/ElectricProvider.tsx Normal file
View file

@ -0,0 +1,132 @@
"use client";
import { useEffect, useState, useRef } from "react";
import { useAtomValue } from "jotai";
import { currentUserAtom } from "@/atoms/user/user-query.atoms";
import {
initElectric,
cleanupElectric,
isElectricInitialized,
type ElectricClient,
} from "@/lib/electric/client";
import { ElectricContext } from "@/lib/electric/context";
interface ElectricProviderProps {
children: React.ReactNode;
}
/**
* ElectricProvider initializes the Electric SQL client with user-specific PGlite database
* and provides it to children via context.
*
* KEY BEHAVIORS:
* 1. Single initialization point - only this provider creates the Electric client
* 2. Creates user-specific database (isolated per user)
* 3. Cleans up other users' databases on login
* 4. Re-initializes when user changes
* 5. Provides client via context - hooks should use useElectricClient()
*/
export function ElectricProvider({ children }: ElectricProviderProps) {
const [electricClient, setElectricClient] = useState<ElectricClient | null>(null);
const [error, setError] = useState<Error | null>(null);
const {
data: user,
isSuccess: isUserLoaded,
isError: isUserError,
} = useAtomValue(currentUserAtom);
const previousUserIdRef = useRef<string | null>(null);
const initializingRef = useRef(false);
useEffect(() => {
// Skip on server side
if (typeof window === "undefined") return;
// If no user is logged in, don't initialize Electric
// The app can still function without real-time sync for non-authenticated pages
if (!isUserLoaded || !user?.id) {
// If we had a previous user and now logged out, cleanup
if (previousUserIdRef.current && isElectricInitialized()) {
console.log("[ElectricProvider] User logged out, cleaning up...");
cleanupElectric().then(() => {
previousUserIdRef.current = null;
setElectricClient(null);
});
}
return;
}
const userId = String(user.id);
// If already initialized for THIS user, skip
if (electricClient && previousUserIdRef.current === userId) {
return;
}
// Prevent concurrent initialization attempts
if (initializingRef.current) {
return;
}
// User changed or first initialization
initializingRef.current = true;
let mounted = true;
async function init() {
try {
console.log(`[ElectricProvider] Initializing for user: ${userId}`);
// If different user was previously initialized, cleanup will happen inside initElectric
const client = await initElectric(userId);
if (mounted) {
previousUserIdRef.current = userId;
setElectricClient(client);
setError(null);
console.log(`[ElectricProvider] ✅ Ready for user: ${userId}`);
}
} catch (err) {
console.error("[ElectricProvider] Failed to initialize:", err);
if (mounted) {
setError(err instanceof Error ? err : new Error("Failed to initialize Electric SQL"));
// Set client to null so hooks know initialization failed
setElectricClient(null);
}
} finally {
if (mounted) {
initializingRef.current = false;
}
}
}
init();
return () => {
mounted = false;
};
}, [user?.id, isUserLoaded, electricClient]);
// For non-authenticated pages (like landing page), render immediately with null context
// Also render immediately if user query failed (e.g., token expired)
if (!isUserLoaded || !user?.id || isUserError) {
return <ElectricContext.Provider value={null}>{children}</ElectricContext.Provider>;
}
// Show loading state while initializing for authenticated users
if (!electricClient && !error) {
return (
<ElectricContext.Provider value={null}>
<div className="flex items-center justify-center min-h-screen">
<div className="text-muted-foreground">Initializing...</div>
</div>
</ElectricContext.Provider>
);
}
// If there's an error, still render but warn
if (error) {
console.warn("[ElectricProvider] Initialization failed, sync may not work:", error.message);
}
// Provide the Electric client to children
return <ElectricContext.Provider value={electricClient}>{children}</ElectricContext.Provider>;
}

2
surfsense_web/components/shared/llm-config-form.tsx
View file

@ -560,7 +560,7 @@ export function LLMConfigForm({
{isSubmitting ? (
<>
<Loader2 className="h-3.5 w-3.5 sm:h-4 sm:w-4 animate-spin" />
{mode === "edit" ? "Updating..." : "Creating..."}
{mode === "edit" ? "Updating..." : "Creating"}
</>
) : (
<>

4
surfsense_web/components/sources/DocumentUploadTab.tsx
View file

@ -3,7 +3,6 @@
import { useAtom } from "jotai";
import { CheckCircle2, FileType, Info, Loader2, Tag, Upload, X } from "lucide-react";
import { AnimatePresence, motion } from "motion/react";
import { useRouter } from "next/navigation";
import { useTranslations } from "next-intl";
import { useCallback, useMemo, useRef, useState } from "react";
import { useDropzone } from "react-dropzone";
@ -121,7 +120,6 @@ export function DocumentUploadTab({
onAccordionStateChange,
}: DocumentUploadTabProps) {
const t = useTranslations("upload_documents");
const router = useRouter();
const [files, setFiles] = useState<File[]>([]);
const [uploadProgress, setUploadProgress] = useState(0);
const [accordionValue, setAccordionValue] = useState<string>("");
@ -224,7 +222,7 @@ export function DocumentUploadTab({
setUploadProgress(100);
trackDocumentUploadSuccess(Number(searchSpaceId), files.length);
toast(t("upload_initiated"), { description: t("upload_initiated_desc") });
onSuccess?.() || router.push(`/dashboard/${searchSpaceId}/documents`);
onSuccess?.();
},
onError: (error: unknown) => {
clearInterval(progressInterval);

2
surfsense_web/components/tool-ui/generate-podcast.tsx
View file

@ -98,7 +98,7 @@ function PodcastGeneratingState({ title }: { title: string }) {
<h3 className="font-semibold text-foreground text-lg">{title}</h3>
<div className="mt-2 flex items-center gap-2 text-muted-foreground">
<Loader2Icon className="size-4 animate-spin" />
<span className="text-sm">Generating podcast... This may take a few minutes</span>
<span className="text-sm">Generating podcast. This may take a few minutes</span>
</div>
<div className="mt-3">
<div className="h-1.5 w-full overflow-hidden rounded-full bg-primary/10">

2
surfsense_web/components/ui/tooltip.tsx
View file

@ -42,7 +42,7 @@ function TooltipContent({
data-slot="tooltip-content"
sideOffset={sideOffset}
className={cn(
"bg-popover text-popover-foreground fill-popover shadow-md animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit rounded-md px-3 py-1.5 text-xs text-balance",
"bg-black text-white font-medium shadow-xl px-3 py-1.5 dark:bg-zinc-800 dark:text-zinc-50 border-none animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit rounded-md text-xs text-balance",
className
)}
{...props}

2
surfsense_web/content/docs/connectors/meta.json
View file

@ -19,5 +19,5 @@
"elasticsearch",
"bookstack"
],
"defaultOpen": true
"defaultOpen": false
}

275
surfsense_web/content/docs/how-to/electric-sql.mdx Normal file
View file

@ -0,0 +1,275 @@
---
title: Electric SQL
description: Setting up Electric SQL for real-time data synchronization in SurfSense
---
# Electric SQL
[Electric SQL](https://electric-sql.com/) enables real-time data synchronization in SurfSense, providing instant updates for notifications, document indexing status, and connector sync progress without manual refresh. The frontend uses [PGlite](https://pglite.dev/) (a lightweight PostgreSQL in the browser) to maintain a local database that syncs with the backend via Electric SQL.
## What Does Electric SQL Do?
When you index documents or receive notifications, Electric SQL pushes updates to your browser in real-time. The data flows like this:
1. Backend writes data to PostgreSQL
2. Electric SQL detects changes and streams them to the frontend
3. PGlite (running in your browser) receives and stores the data locally in IndexedDB
4. Your UI updates instantly without refreshing
This means:
- **Notifications appear instantly** - No need to refresh the page
- **Document indexing progress updates live** - Watch your documents get processed
- **Connector status syncs automatically** - See when connectors finish syncing
- **Offline support** - PGlite caches data locally, so previously loaded data remains accessible
## Docker Setup
### All-in-One Quickstart
The simplest way to run SurfSense with Electric SQL is using the all-in-one Docker image. This bundles everything into a single container:
- PostgreSQL + pgvector (vector database)
- Redis (task queue)
- Electric SQL (real-time sync)
- Backend API
- Frontend
```bash
docker run -d \
-p 3000:3000 \
-p 8000:8000 \
-p 5133:5133 \
-v surfsense-data:/data \
--name surfsense \
ghcr.io/modsetter/surfsense:latest
```
**With custom Electric SQL credentials:**
```bash
docker run -d \
-p 3000:3000 \
-p 8000:8000 \
-p 5133:5133 \
-v surfsense-data:/data \
-e ELECTRIC_DB_USER=your_electric_user \
-e ELECTRIC_DB_PASSWORD=your_electric_password \
--name surfsense \
ghcr.io/modsetter/surfsense:latest
```
Access SurfSense at `http://localhost:3000`. Electric SQL is automatically configured and running on port 5133.
### Docker Compose
For more control over individual services, use Docker Compose.
**Quickstart (all-in-one image):**
```bash
docker compose -f docker-compose.quickstart.yml up -d
```
**Standard setup (separate services):**
The `docker-compose.yml` includes the Electric SQL service configuration:
```yaml
electric:
image: electricsql/electric:latest
ports:
- "${ELECTRIC_PORT:-5133}:3000"
environment:
- DATABASE_URL=${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-surfsense}?sslmode=disable}
- ELECTRIC_INSECURE=true
- ELECTRIC_WRITE_TO_PG_MODE=direct
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"]
interval: 10s
timeout: 5s
retries: 5
```
No additional configuration is required - Electric SQL is pre-configured to work with the Docker PostgreSQL instance.
## Manual Setup
Follow the steps below based on your PostgreSQL setup.
### Step 1: Configure Environment Variables
Ensure your environment files are configured. If you haven't set up SurfSense yet, follow the [Manual Installation Guide](/docs/manual-installation) first.
For Electric SQL, verify these variables are set:
**Root `.env`:**
```bash
ELECTRIC_PORT=5133
POSTGRES_HOST=host.docker.internal # Use 'db' for Docker PostgreSQL instance
ELECTRIC_DB_USER=electric
ELECTRIC_DB_PASSWORD=electric_password
NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133
```
**Frontend `.env` (`surfsense_web/.env`):**
```bash
NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133
NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure
```
---
### Option A: Using Docker PostgreSQL
If you're using the Docker-managed PostgreSQL instance, follow these steps:
**1. Update environment variable:**
In your root `.env` file, set:
```bash
POSTGRES_HOST=db
```
**2. Start PostgreSQL and Electric SQL:**
```bash
docker-compose up -d db electric
```
**3. Run database migration:**
```bash
cd surfsense_backend
uv run alembic upgrade head
```
**4. Start the backend:**
```bash
uv run main.py
```
Electric SQL is now configured and connected to your Docker PostgreSQL database.
---
### Option B: Using Local PostgreSQL
If you're using a local PostgreSQL installation, follow these steps:
**1. Enable logical replication in PostgreSQL:**
Open your `postgresql.conf` file using vim (or your preferred editor):
```bash
# Common locations:
# macOS (Homebrew): /opt/homebrew/var/postgresql@15/postgresql.conf
# Linux: /etc/postgresql/15/main/postgresql.conf
# Windows: C:\Program Files\PostgreSQL\15\data\postgresql.conf
sudo vim /path/to/postgresql.conf
```
Add the following settings:
```ini
# Enable logical replication (required for Electric SQL)
wal_level = logical
max_replication_slots = 10
max_wal_senders = 10
```
After saving the changes (`:wq` in vim), restart your PostgreSQL server for the configuration to take effect.
**2. Update environment variable:**
In your root `.env` file, set:
```bash
POSTGRES_HOST=host.docker.internal
```
**3. Start Electric SQL:**
```bash
docker-compose up -d electric
```
**4. Run database migration:**
```bash
cd surfsense_backend
uv run alembic upgrade head
```
**5. Start the backend:**
```bash
uv run main.py
```
Electric SQL is now configured and connected to your local PostgreSQL database.
## Environment Variables Reference
| Variable | Location | Description | Default |
|----------|----------|-------------|---------|
| `ELECTRIC_PORT` | Root `.env` | Port to expose Electric SQL | `5133` |
| `POSTGRES_HOST` | Root `.env` | PostgreSQL host (`db` for Docker, `host.docker.internal` for local) | `host.docker.internal` |
| `ELECTRIC_DB_USER` | Root `.env` | Database user for Electric | `electric` |
| `ELECTRIC_DB_PASSWORD` | Root `.env` | Database password for Electric | `electric_password` |
| `NEXT_PUBLIC_ELECTRIC_URL` | Frontend `.env` | Electric SQL server URL (PGlite connects to this) | `http://localhost:5133` |
| `NEXT_PUBLIC_ELECTRIC_AUTH_MODE` | Frontend `.env` | Authentication mode (`insecure` for dev, `secure` for production) | `insecure` |
## Verify Setup
To verify Electric SQL is running correctly:
```bash
curl http://localhost:5133/v1/health
```
You should receive:
```json
{"status":"active"}
```
## Troubleshooting
### Electric SQL Server Not Starting
**Check PostgreSQL settings:**
- Ensure `wal_level = logical` is set
- Verify the Electric user has replication permissions
- Check database connectivity from Electric container
### Real-time Updates Not Working
1. Open browser DevTools → Console
2. Look for errors containing `[Electric]`
3. Check Network tab for WebSocket connections to the Electric URL
### Connection Refused Errors
- Verify Electric SQL server is running: `docker ps | grep electric`
- Check the `NEXT_PUBLIC_ELECTRIC_URL` matches your Electric server address
- For Docker setups, ensure the frontend can reach the Electric container
### Data Not Syncing
- Check Electric SQL logs: `docker logs electric`
- Verify PostgreSQL replication is working
- Ensure the Electric user has proper table permissions
### PGlite/IndexedDB Issues
If data appears stale or corrupted in the browser:
1. Open browser DevTools → Application → IndexedDB
2. Delete databases starting with `surfsense-`
3. Refresh the page - PGlite will recreate the local database and resync

5
surfsense_web/content/docs/how-to/meta.json Normal file
View file

@ -0,0 +1,5 @@
{
"title": "How to",
"pages": ["electric-sql"],
"defaultOpen": false
}

3
surfsense_web/content/docs/meta.json
View file

@ -8,6 +8,7 @@
"installation",
"docker-installation",
"manual-installation",
"connectors"
"connectors",
"how-to"
]
}

1
surfsense_web/contracts/enums/connector.ts
View file

@ -1,4 +1,5 @@
export enum EnumConnectorName {
SERPER_API = "SERPER_API",
TAVILY_API = "TAVILY_API",
SEARXNG_API = "SEARXNG_API",
LINKUP_API = "LINKUP_API",

6
surfsense_web/contracts/types/document.types.ts
View file

@ -6,9 +6,11 @@ export const documentTypeEnum = z.enum([
"CRAWLED_URL",
"FILE",
"SLACK_CONNECTOR",
"TEAMS_CONNECTOR",
"NOTION_CONNECTOR",
"YOUTUBE_VIDEO",
"GITHUB_CONNECTOR",
"LINEAR_CONNECTOR",
"DISCORD_CONNECTOR",
"JIRA_CONNECTOR",
"CONFLUENCE_CONNECTOR",
@ -19,10 +21,10 @@ export const documentTypeEnum = z.enum([
"AIRTABLE_CONNECTOR",
"LUMA_CONNECTOR",
"ELASTICSEARCH_CONNECTOR",
"LINEAR_CONNECTOR",
"NOTE",
"BOOKSTACK_CONNECTOR",
"CIRCLEBACK",
"SURFSENSE_DOCS",
"NOTE",
]);
export const document = z.object({

120
surfsense_web/contracts/types/notification.types.ts Normal file
View file

@ -0,0 +1,120 @@
import { z } from "zod";
import { searchSourceConnectorTypeEnum } from "./connector.types";
import { documentTypeEnum } from "./document.types";
/**
* Notification type enum - matches backend notification types
*/
export const notificationTypeEnum = z.enum(["connector_indexing", "document_processing"]);
/**
* Notification status enum - used in metadata
*/
export const notificationStatusEnum = z.enum(["in_progress", "completed", "failed"]);
/**
* Document processing stage enum
*/
export const documentProcessingStageEnum = z.enum([
"queued",
"parsing",
"chunking",
"embedding",
"storing",
"completed",
"failed",
]);
/**
* Base metadata schema shared across notification types
*/
export const baseNotificationMetadata = z.object({
operation_id: z.string().optional(),
status: notificationStatusEnum.optional(),
started_at: z.string().optional(),
completed_at: z.string().optional(),
});
/**
* Connector indexing metadata schema
*/
export const connectorIndexingMetadata = baseNotificationMetadata.extend({
connector_id: z.number(),
connector_name: z.string(),
connector_type: searchSourceConnectorTypeEnum,
start_date: z.string().nullable().optional(),
end_date: z.string().nullable().optional(),
indexed_count: z.number(),
total_count: z.number().optional(),
progress_percent: z.number().optional(),
error_message: z.string().nullable().optional(),
// Google Drive specific fields
folder_count: z.number().optional(),
file_count: z.number().optional(),
folder_names: z.array(z.string()).optional(),
file_names: z.array(z.string()).optional(),
});
/**
* Document processing metadata schema
*/
export const documentProcessingMetadata = baseNotificationMetadata.extend({
document_type: documentTypeEnum,
document_name: z.string(),
processing_stage: documentProcessingStageEnum,
file_size: z.number().optional(),
chunks_count: z.number().optional(),
document_id: z.number().optional(),
error_message: z.string().nullable().optional(),
});
/**
* Union of all notification metadata types
* Use this when the notification type is unknown
*/
export const notificationMetadata = z.union([
connectorIndexingMetadata,
documentProcessingMetadata,
baseNotificationMetadata,
]);
/**
* Main notification schema
*/
export const notification = z.object({
id: z.number(),
user_id: z.string(),
search_space_id: z.number().nullable(),
type: notificationTypeEnum,
title: z.string(),
message: z.string(),
read: z.boolean(),
metadata: z.record(z.string(), z.unknown()),
created_at: z.string(),
updated_at: z.string().nullable(),
});
/**
* Typed notification schemas for specific notification types
*/
export const connectorIndexingNotification = notification.extend({
type: z.literal("connector_indexing"),
metadata: connectorIndexingMetadata,
});
export const documentProcessingNotification = notification.extend({
type: z.literal("document_processing"),
metadata: documentProcessingMetadata,
});
// Inferred types
export type NotificationTypeEnum = z.infer<typeof notificationTypeEnum>;
export type NotificationStatusEnum = z.infer<typeof notificationStatusEnum>;
export type DocumentProcessingStageEnum = z.infer<typeof documentProcessingStageEnum>;
export type BaseNotificationMetadata = z.infer<typeof baseNotificationMetadata>;
export type ConnectorIndexingMetadata = z.infer<typeof connectorIndexingMetadata>;
export type DocumentProcessingMetadata = z.infer<typeof documentProcessingMetadata>;
export type NotificationMetadata = z.infer<typeof notificationMetadata>;
export type Notification = z.infer<typeof notification>;
export type ConnectorIndexingNotification = z.infer<typeof connectorIndexingNotification>;
export type DocumentProcessingNotification = z.infer<typeof documentProcessingNotification>;

203
surfsense_web/hooks/use-connectors-electric.ts Normal file
View file

@ -0,0 +1,203 @@
"use client";
import { useEffect, useState, useCallback, useRef } from "react";
import { useElectricClient } from "@/lib/electric/context";
import type { SyncHandle } from "@/lib/electric/client";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
/**
* Hook for managing connectors with Electric SQL real-time sync
*
* Uses the Electric client from context (provided by ElectricProvider)
* instead of initializing its own - prevents race conditions and memory leaks
*/
export function useConnectorsElectric(searchSpaceId: number | string | null) {
// Get Electric client from context - ElectricProvider handles initialization
const electricClient = useElectricClient();
const [connectors, setConnectors] = useState<SearchSourceConnector[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<Error | null>(null);
const syncHandleRef = useRef<SyncHandle | null>(null);
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null);
const syncKeyRef = useRef<string | null>(null);
// Transform connector data from Electric SQL/PGlite to match expected format
function transformConnector(connector: any): SearchSourceConnector {
return {
...connector,
last_indexed_at: connector.last_indexed_at
? typeof connector.last_indexed_at === "string"
? connector.last_indexed_at
: new Date(connector.last_indexed_at).toISOString()
: null,
next_scheduled_at: connector.next_scheduled_at
? typeof connector.next_scheduled_at === "string"
? connector.next_scheduled_at
: new Date(connector.next_scheduled_at).toISOString()
: null,
created_at: connector.created_at
? typeof connector.created_at === "string"
? connector.created_at
: new Date(connector.created_at).toISOString()
: new Date().toISOString(),
};
}
// Start syncing when Electric client is available
useEffect(() => {
// Wait for both searchSpaceId and Electric client to be available
if (!searchSpaceId || !electricClient) {
setLoading(!electricClient); // Still loading if waiting for Electric
if (!searchSpaceId) {
setConnectors([]);
}
return;
}
// Create a unique key for this sync to prevent duplicate subscriptions
const syncKey = `connectors_${searchSpaceId}`;
if (syncKeyRef.current === syncKey) {
// Already syncing for this search space
return;
}
let mounted = true;
syncKeyRef.current = syncKey;
async function startSync() {
try {
console.log("[useConnectorsElectric] Starting sync for search space:", searchSpaceId);
const handle = await electricClient.syncShape({
table: "search_source_connectors",
where: `search_space_id = ${searchSpaceId}`,
primaryKey: ["id"],
});
console.log("[useConnectorsElectric] Sync started:", {
isUpToDate: handle.isUpToDate,
});
// Wait for initial sync with timeout
if (!handle.isUpToDate && handle.initialSyncPromise) {
try {
await Promise.race([
handle.initialSyncPromise,
new Promise((resolve) => setTimeout(resolve, 2000)),
]);
} catch (syncErr) {
console.error("[useConnectorsElectric] Initial sync failed:", syncErr);
}
}
if (!mounted) {
handle.unsubscribe();
return;
}
syncHandleRef.current = handle;
setLoading(false);
setError(null);
// Fetch initial connectors
await fetchConnectors();
// Set up live query for real-time updates
await setupLiveQuery();
} catch (err) {
if (!mounted) return;
console.error("[useConnectorsElectric] Failed to start sync:", err);
setError(err instanceof Error ? err : new Error("Failed to sync connectors"));
setLoading(false);
}
}
async function fetchConnectors() {
try {
const result = await electricClient.db.query(
`SELECT * FROM search_source_connectors WHERE search_space_id = $1 ORDER BY created_at DESC`,
[searchSpaceId]
);
if (mounted) {
setConnectors((result.rows || []).map(transformConnector));
}
} catch (err) {
console.error("[useConnectorsElectric] Failed to fetch:", err);
}
}
async function setupLiveQuery() {
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const db = electricClient.db as any;
if (db.live?.query && typeof db.live.query === "function") {
const liveQuery = await db.live.query(
`SELECT * FROM search_source_connectors WHERE search_space_id = $1 ORDER BY created_at DESC`,
[searchSpaceId]
);
if (!mounted) {
liveQuery.unsubscribe?.();
return;
}
// Set initial results
if (liveQuery.initialResults?.rows) {
setConnectors(liveQuery.initialResults.rows.map(transformConnector));
} else if (liveQuery.rows) {
setConnectors(liveQuery.rows.map(transformConnector));
}
// Subscribe to changes
if (typeof liveQuery.subscribe === "function") {
liveQuery.subscribe((result: { rows: any[] }) => {
if (mounted && result.rows) {
setConnectors(result.rows.map(transformConnector));
}
});
}
if (typeof liveQuery.unsubscribe === "function") {
liveQueryRef.current = liveQuery;
}
}
} catch (liveErr) {
console.error("[useConnectorsElectric] Failed to set up live query:", liveErr);
}
}
startSync();
return () => {
mounted = false;
syncKeyRef.current = null;
if (syncHandleRef.current) {
syncHandleRef.current.unsubscribe();
syncHandleRef.current = null;
}
if (liveQueryRef.current) {
liveQueryRef.current.unsubscribe();
liveQueryRef.current = null;
}
};
}, [searchSpaceId, electricClient]);
// Manual refresh function (optional, for fallback)
const refreshConnectors = useCallback(async () => {
if (!electricClient) return;
try {
const result = await electricClient.db.query(
`SELECT * FROM search_source_connectors WHERE search_space_id = $1 ORDER BY created_at DESC`,
[searchSpaceId]
);
setConnectors((result.rows || []).map(transformConnector));
} catch (err) {
console.error("[useConnectorsElectric] Failed to refresh:", err);
}
}, [electricClient, searchSpaceId]);
return { connectors, loading, error, refreshConnectors };
}

185
surfsense_web/hooks/use-documents-electric.ts Normal file
View file

@ -0,0 +1,185 @@
"use client";
import { useEffect, useState, useRef, useMemo } from "react";
import { useElectricClient } from "@/lib/electric/context";
import type { SyncHandle } from "@/lib/electric/client";
interface Document {
id: number;
search_space_id: number;
document_type: string;
created_at: string;
}
/**
* Hook for managing documents with Electric SQL real-time sync
*
* Uses the Electric client from context (provided by ElectricProvider)
* instead of initializing its own - prevents race conditions and memory leaks
*/
export function useDocumentsElectric(searchSpaceId: number | string | null) {
// Get Electric client from context - ElectricProvider handles initialization
const electricClient = useElectricClient();
const [documents, setDocuments] = useState<Document[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<Error | null>(null);
const syncHandleRef = useRef<SyncHandle | null>(null);
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null);
const syncKeyRef = useRef<string | null>(null);
// Calculate document type counts from synced documents
const documentTypeCounts = useMemo(() => {
if (!documents.length) return {};
const counts: Record<string, number> = {};
for (const doc of documents) {
counts[doc.document_type] = (counts[doc.document_type] || 0) + 1;
}
return counts;
}, [documents]);
// Start syncing when Electric client is available
useEffect(() => {
// Wait for both searchSpaceId and Electric client to be available
if (!searchSpaceId || !electricClient) {
setLoading(!electricClient); // Still loading if waiting for Electric
if (!searchSpaceId) {
setDocuments([]);
}
return;
}
// Create a unique key for this sync to prevent duplicate subscriptions
const syncKey = `documents_${searchSpaceId}`;
if (syncKeyRef.current === syncKey) {
// Already syncing for this search space
return;
}
let mounted = true;
syncKeyRef.current = syncKey;
async function startSync() {
try {
console.log("[useDocumentsElectric] Starting sync for search space:", searchSpaceId);
const handle = await electricClient.syncShape({
table: "documents",
where: `search_space_id = ${searchSpaceId}`,
columns: ["id", "document_type", "search_space_id", "created_at"],
primaryKey: ["id"],
});
console.log("[useDocumentsElectric] Sync started:", {
isUpToDate: handle.isUpToDate,
});
// Wait for initial sync with timeout
if (!handle.isUpToDate && handle.initialSyncPromise) {
try {
await Promise.race([
handle.initialSyncPromise,
new Promise((resolve) => setTimeout(resolve, 2000)),
]);
} catch (syncErr) {
console.error("[useDocumentsElectric] Initial sync failed:", syncErr);
}
}
if (!mounted) {
handle.unsubscribe();
return;
}
syncHandleRef.current = handle;
setLoading(false);
setError(null);
// Fetch initial documents
await fetchDocuments();
// Set up live query for real-time updates
await setupLiveQuery();
} catch (err) {
if (!mounted) return;
console.error("[useDocumentsElectric] Failed to start sync:", err);
setError(err instanceof Error ? err : new Error("Failed to sync documents"));
setLoading(false);
}
}
async function fetchDocuments() {
try {
const result = await electricClient.db.query<Document>(
`SELECT id, document_type, search_space_id, created_at FROM documents WHERE search_space_id = $1 ORDER BY created_at DESC`,
[searchSpaceId]
);
if (mounted) {
setDocuments(result.rows || []);
}
} catch (err) {
console.error("[useDocumentsElectric] Failed to fetch:", err);
}
}
async function setupLiveQuery() {
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const db = electricClient.db as any;
if (db.live?.query && typeof db.live.query === "function") {
const liveQuery = await db.live.query(
`SELECT id, document_type, search_space_id, created_at FROM documents WHERE search_space_id = $1 ORDER BY created_at DESC`,
[searchSpaceId]
);
if (!mounted) {
liveQuery.unsubscribe?.();
return;
}
// Set initial results
if (liveQuery.initialResults?.rows) {
setDocuments(liveQuery.initialResults.rows);
} else if (liveQuery.rows) {
setDocuments(liveQuery.rows);
}
// Subscribe to changes
if (typeof liveQuery.subscribe === "function") {
liveQuery.subscribe((result: { rows: Document[] }) => {
if (mounted && result.rows) {
setDocuments(result.rows);
}
});
}
if (typeof liveQuery.unsubscribe === "function") {
liveQueryRef.current = liveQuery;
}
}
} catch (liveErr) {
console.error("[useDocumentsElectric] Failed to set up live query:", liveErr);
}
}
startSync();
return () => {
mounted = false;
syncKeyRef.current = null;
if (syncHandleRef.current) {
syncHandleRef.current.unsubscribe();
syncHandleRef.current = null;
}
if (liveQueryRef.current) {
liveQueryRef.current.unsubscribe();
liveQueryRef.current = null;
}
};
}, [searchSpaceId, electricClient]);
return { documentTypeCounts, loading, error };
}

248
surfsense_web/hooks/use-notifications.ts Normal file
View file

@ -0,0 +1,248 @@
"use client";
import { useEffect, useState, useCallback, useRef } from "react";
import { useElectricClient } from "@/lib/electric/context";
import type { SyncHandle } from "@/lib/electric/client";
import type { Notification } from "@/contracts/types/notification.types";
import { authenticatedFetch } from "@/lib/auth-utils";
export type { Notification } from "@/contracts/types/notification.types";
/**
* Hook for managing notifications with Electric SQL real-time sync
*
* Uses the Electric client from context (provided by ElectricProvider)
* instead of initializing its own - prevents race conditions and memory leaks
*
* Architecture:
* - User-level sync: Syncs ALL notifications for a user (runs once per user)
* - Search-space-level query: Filters notifications by searchSpaceId (updates on search space change)
*
* This separation ensures smooth transitions when switching search spaces (no flash).
*
* @param userId - The user ID to fetch notifications for
* @param searchSpaceId - The search space ID to filter notifications (null shows global notifications only)
*/
export function useNotifications(userId: string | null, searchSpaceId: number | null) {
// Get Electric client from context - ElectricProvider handles initialization
const electricClient = useElectricClient();
const [notifications, setNotifications] = useState<Notification[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<Error | null>(null);
const syncHandleRef = useRef<SyncHandle | null>(null);
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null);
// Track user-level sync key to prevent duplicate sync subscriptions
const userSyncKeyRef = useRef<string | null>(null);
// EFFECT 1: User-level sync - runs once per user, syncs ALL notifications
useEffect(() => {
if (!userId || !electricClient) {
setLoading(!electricClient);
return;
}
const userSyncKey = `notifications_${userId}`;
if (userSyncKeyRef.current === userSyncKey) {
// Already syncing for this user
return;
}
let mounted = true;
userSyncKeyRef.current = userSyncKey;
async function startUserSync() {
try {
console.log("[useNotifications] Starting user-level sync for:", userId);
// Sync ALL notifications for this user (cached via syncShape caching)
const handle = await electricClient.syncShape({
table: "notifications",
where: `user_id = '${userId}'`,
primaryKey: ["id"],
});
console.log("[useNotifications] User sync started:", {
isUpToDate: handle.isUpToDate,
});
// Wait for initial sync with timeout
if (!handle.isUpToDate && handle.initialSyncPromise) {
try {
await Promise.race([
handle.initialSyncPromise,
new Promise((resolve) => setTimeout(resolve, 2000)),
]);
} catch (syncErr) {
console.error("[useNotifications] Initial sync failed:", syncErr);
}
}
if (!mounted) {
handle.unsubscribe();
return;
}
syncHandleRef.current = handle;
setLoading(false);
setError(null);
} catch (err) {
if (!mounted) return;
console.error("[useNotifications] Failed to start user sync:", err);
setError(err instanceof Error ? err : new Error("Failed to sync notifications"));
setLoading(false);
}
}
startUserSync();
return () => {
mounted = false;
userSyncKeyRef.current = null;
if (syncHandleRef.current) {
syncHandleRef.current.unsubscribe();
syncHandleRef.current = null;
}
};
}, [userId, electricClient]);
// EFFECT 2: Search-space-level query - updates when searchSpaceId changes
// This runs independently of sync, allowing smooth transitions between search spaces
useEffect(() => {
if (!userId || !electricClient) {
return;
}
let mounted = true;
async function updateQuery() {
// Clean up previous live query (but DON'T clear notifications - keep showing old until new arrive)
if (liveQueryRef.current) {
liveQueryRef.current.unsubscribe();
liveQueryRef.current = null;
}
try {
console.log("[useNotifications] Updating query for searchSpace:", searchSpaceId);
// Fetch notifications for current search space immediately
const result = await electricClient.db.query<Notification>(
`SELECT * FROM notifications
WHERE user_id = $1
AND (search_space_id = $2 OR search_space_id IS NULL)
ORDER BY created_at DESC`,
[userId, searchSpaceId]
);
if (mounted) {
setNotifications(result.rows || []);
}
// Set up live query for real-time updates
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const db = electricClient.db as any;
if (db.live?.query && typeof db.live.query === "function") {
const liveQuery = await db.live.query(
`SELECT * FROM notifications
WHERE user_id = $1
AND (search_space_id = $2 OR search_space_id IS NULL)
ORDER BY created_at DESC`,
[userId, searchSpaceId]
);
if (!mounted) {
liveQuery.unsubscribe?.();
return;
}
// Set initial results from live query
if (liveQuery.initialResults?.rows) {
setNotifications(liveQuery.initialResults.rows);
} else if (liveQuery.rows) {
setNotifications(liveQuery.rows);
}
// Subscribe to changes
if (typeof liveQuery.subscribe === "function") {
liveQuery.subscribe((result: { rows: Notification[] }) => {
if (mounted && result.rows) {
setNotifications(result.rows);
}
});
}
if (typeof liveQuery.unsubscribe === "function") {
liveQueryRef.current = liveQuery;
}
}
} catch (err) {
console.error("[useNotifications] Failed to update query:", err);
}
}
updateQuery();
return () => {
mounted = false;
if (liveQueryRef.current) {
liveQueryRef.current.unsubscribe();
liveQueryRef.current = null;
}
};
}, [userId, searchSpaceId, electricClient]);
// Mark notification as read via backend API
const markAsRead = useCallback(async (notificationId: number) => {
try {
const response = await authenticatedFetch(
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/notifications/${notificationId}/read`,
{ method: "PATCH" }
);
if (!response.ok) {
const error = await response.json().catch(() => ({ detail: "Failed to mark as read" }));
throw new Error(error.detail || "Failed to mark notification as read");
}
return true;
} catch (err) {
console.error("Failed to mark notification as read:", err);
return false;
}
}, []);
// Mark all notifications as read via backend API
const markAllAsRead = useCallback(async () => {
try {
const response = await authenticatedFetch(
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/notifications/read-all`,
{ method: "PATCH" }
);
if (!response.ok) {
const error = await response.json().catch(() => ({ detail: "Failed to mark all as read" }));
throw new Error(error.detail || "Failed to mark all notifications as read");
}
return true;
} catch (err) {
console.error("Failed to mark all notifications as read:", err);
return false;
}
}, []);
// Get unread count
const unreadCount = notifications.filter((n) => !n.read).length;
return {
notifications,
unreadCount,
markAsRead,
markAllAsRead,
loading,
error,
};
}

20
surfsense_web/lib/electric/auth.ts Normal file
View file

@ -0,0 +1,20 @@
/**
* Get auth token for Electric SQL
* In production, this should get the token from your auth system
*/
export async function getElectricAuthToken(): Promise<string> {
// For insecure mode (development), return empty string
if (process.env.NEXT_PUBLIC_ELECTRIC_AUTH_MODE === "insecure") {
return "";
}
// In production, get token from your auth system
// This should match your backend auth token
if (typeof window !== "undefined") {
const token = localStorage.getItem("surfsense_bearer_token");
return token || "";
}
return "";
}

665
surfsense_web/lib/electric/client.ts Normal file
View file

@ -0,0 +1,665 @@
/**
* Electric SQL client setup for ElectricSQL 1.x with PGlite
*
* USER-SPECIFIC DATABASE ARCHITECTURE:
* - Each user gets their own IndexedDB database: idb://surfsense-{userId}-v{version}
* - On login: cleanup databases from other users, then initialize current user's DB
* - On logout: best-effort cleanup (not relied upon)
*
* This ensures:
* 1. Complete user isolation (data can never leak between users)
* 2. Self-healing on login (stale databases are cleaned up)
* 3. Works even if logout cleanup fails
*/
import { PGlite } from "@electric-sql/pglite";
import { electricSync } from "@electric-sql/pglite-sync";
import { live } from "@electric-sql/pglite/live";
// Types
export interface ElectricClient {
db: PGlite;
userId: string;
syncShape: (options: SyncShapeOptions) => Promise<SyncHandle>;
}
export interface SyncShapeOptions {
table: string;
where?: string;
columns?: string[];
primaryKey?: string[];
}
export interface SyncHandle {
unsubscribe: () => void;
readonly isUpToDate: boolean;
// The stream property contains the ShapeStreamInterface from pglite-sync
stream?: unknown;
// Promise that resolves when initial sync is complete
initialSyncPromise?: Promise<void>;
}
// Singleton state - now tracks the user ID
let electricClient: ElectricClient | null = null;
let currentUserId: string | null = null;
let isInitializing = false;
let initPromise: Promise<ElectricClient> | null = null;
// Cache for sync handles to prevent duplicate subscriptions (memory optimization)
const activeSyncHandles = new Map<string, SyncHandle>();
// Version for sync state - increment this to force fresh sync when Electric config changes
// Set to v2 for user-specific database architecture
const SYNC_VERSION = 2;
// Database name prefix for identifying SurfSense databases
const DB_PREFIX = "surfsense-";
// Get Electric URL from environment
function getElectricUrl(): string {
if (typeof window !== "undefined") {
return process.env.NEXT_PUBLIC_ELECTRIC_URL || "http://localhost:5133";
}
return "http://localhost:5133";
}
/**
* Get the database name for a specific user
*/
function getDbName(userId: string): string {
return `idb://${DB_PREFIX}${userId}-v${SYNC_VERSION}`;
}
/**
* Clean up databases from OTHER users (not the current user)
* This is called on login to ensure clean state
*/
async function cleanupOtherUserDatabases(currentUserId: string): Promise<void> {
if (typeof window === "undefined" || !window.indexedDB) {
return;
}
try {
// Try to list all databases (not supported in all browsers)
if (typeof window.indexedDB.databases === "function") {
const databases = await window.indexedDB.databases();
for (const dbInfo of databases) {
const dbName = dbInfo.name;
if (!dbName) continue;
// Check if this is a SurfSense database
if (dbName.startsWith(DB_PREFIX) || dbName.includes("surfsense")) {
// Don't delete current user's database
if (dbName.includes(currentUserId)) {
console.log(`[Electric] Keeping current user's database: ${dbName}`);
continue;
}
// Delete databases from other users
try {
console.log(`[Electric] Deleting stale database: ${dbName}`);
window.indexedDB.deleteDatabase(dbName);
} catch (deleteErr) {
console.warn(`[Electric] Failed to delete database ${dbName}:`, deleteErr);
}
}
}
}
} catch (err) {
// indexedDB.databases() not supported - that's okay, login cleanup is best-effort
console.warn("[Electric] Could not enumerate databases for cleanup:", err);
}
}
/**
* Initialize the Electric SQL client for a specific user
*
* KEY BEHAVIORS:
* 1. If already initialized for the SAME user, returns existing client
* 2. If initialized for a DIFFERENT user, closes old client and creates new one
* 3. On first init, cleans up databases from other users
*
* @param userId - The current user's ID (required)
*/
export async function initElectric(userId: string): Promise<ElectricClient> {
if (!userId) {
throw new Error("userId is required for Electric initialization");
}
// If already initialized for this user, return existing client
if (electricClient && currentUserId === userId) {
return electricClient;
}
// If initialized for a different user, close the old client first
if (electricClient && currentUserId !== userId) {
console.log(`[Electric] User changed from ${currentUserId} to ${userId}, reinitializing...`);
await cleanupElectric();
}
// If already initializing, wait for it
if (isInitializing && initPromise) {
return initPromise;
}
isInitializing = true;
currentUserId = userId;
initPromise = (async () => {
try {
// STEP 1: Clean up databases from other users (login-time cleanup)
console.log("[Electric] Cleaning up databases from other users...");
await cleanupOtherUserDatabases(userId);
// STEP 2: Create user-specific PGlite database
const dbName = getDbName(userId);
console.log(`[Electric] Initializing database: ${dbName}`);
const db = await PGlite.create({
dataDir: dbName,
relaxedDurability: true,
extensions: {
// Enable debug mode in electricSync to see detailed sync logs
electric: electricSync({ debug: true }),
live, // Enable live queries for real-time updates
},
});
// STEP 3: Create the notifications table schema in PGlite
// This matches the backend schema
await db.exec(`
CREATE TABLE IF NOT EXISTS notifications (
id INTEGER PRIMARY KEY,
user_id TEXT NOT NULL,
search_space_id INTEGER,
type TEXT NOT NULL,
title TEXT NOT NULL,
message TEXT NOT NULL,
read BOOLEAN NOT NULL DEFAULT FALSE,
metadata JSONB DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_notifications_user_id ON notifications(user_id);
CREATE INDEX IF NOT EXISTS idx_notifications_read ON notifications(read);
`);
// Create the search_source_connectors table schema in PGlite
// This matches the backend schema
await db.exec(`
CREATE TABLE IF NOT EXISTS search_source_connectors (
id INTEGER PRIMARY KEY,
search_space_id INTEGER NOT NULL,
user_id TEXT NOT NULL,
connector_type TEXT NOT NULL,
name TEXT NOT NULL,
is_indexable BOOLEAN NOT NULL DEFAULT FALSE,
last_indexed_at TIMESTAMPTZ,
config JSONB DEFAULT '{}',
periodic_indexing_enabled BOOLEAN NOT NULL DEFAULT FALSE,
indexing_frequency_minutes INTEGER,
next_scheduled_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_connectors_search_space_id ON search_source_connectors(search_space_id);
CREATE INDEX IF NOT EXISTS idx_connectors_type ON search_source_connectors(connector_type);
CREATE INDEX IF NOT EXISTS idx_connectors_user_id ON search_source_connectors(user_id);
`);
// Create the documents table schema in PGlite
// Only sync minimal fields needed for type counts: id, document_type, search_space_id
await db.exec(`
CREATE TABLE IF NOT EXISTS documents (
id INTEGER PRIMARY KEY,
search_space_id INTEGER NOT NULL,
document_type TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_documents_search_space_id ON documents(search_space_id);
CREATE INDEX IF NOT EXISTS idx_documents_type ON documents(document_type);
CREATE INDEX IF NOT EXISTS idx_documents_search_space_type ON documents(search_space_id, document_type);
`);
const electricUrl = getElectricUrl();
// STEP 4: Create the client wrapper
electricClient = {
db,
userId,
syncShape: async (options: SyncShapeOptions): Promise<SyncHandle> => {
const { table, where, columns, primaryKey = ["id"] } = options;
// Create cache key for this sync shape
const cacheKey = `${table}_${where || "all"}_${columns?.join(",") || "all"}`;
// Check if we already have an active sync for this shape (memory optimization)
const existingHandle = activeSyncHandles.get(cacheKey);
if (existingHandle) {
console.log(`[Electric] Reusing existing sync handle for: ${cacheKey}`);
return existingHandle;
}
// Build params for the shape request
// Electric SQL expects params as URL query parameters
const params: Record<string, string> = { table };
// Validate and fix WHERE clause to ensure string literals are properly quoted
let validatedWhere = where;
if (where) {
// Check if where uses positional parameters
if (where.includes("$1")) {
// Extract the value from the where clause if it's embedded
// For now, we'll use the where clause as-is and let Electric handle it
params.where = where;
validatedWhere = where;
} else {
// Validate that string literals are properly quoted
// Count single quotes - should be even (pairs) for properly quoted strings
const singleQuoteCount = (where.match(/'/g) || []).length;
if (singleQuoteCount % 2 !== 0) {
// Odd number of quotes means unterminated string literal
console.warn("Where clause has unmatched quotes, fixing:", where);
// Add closing quote at the end
validatedWhere = `${where}'`;
params.where = validatedWhere;
} else {
// Use the where clause directly (already formatted)
params.where = where;
validatedWhere = where;
}
}
}
if (columns) params.columns = columns.join(",");
console.log("[Electric] Syncing shape with params:", params);
console.log("[Electric] Electric URL:", `${electricUrl}/v1/shape`);
console.log("[Electric] Where clause:", where, "Validated:", validatedWhere);
try {
// Debug: Test Electric SQL connection directly first
// Use validatedWhere to ensure proper URL encoding
const testUrl = `${electricUrl}/v1/shape?table=${table}&offset=-1${validatedWhere ? `&where=${encodeURIComponent(validatedWhere)}` : ""}`;
console.log("[Electric] Testing Electric SQL directly:", testUrl);
try {
const testResponse = await fetch(testUrl);
const testHeaders = {
handle: testResponse.headers.get("electric-handle"),
offset: testResponse.headers.get("electric-offset"),
upToDate: testResponse.headers.get("electric-up-to-date"),
};
console.log("[Electric] Direct Electric SQL response headers:", testHeaders);
const testData = await testResponse.json();
console.log(
"[Electric] Direct Electric SQL data count:",
Array.isArray(testData) ? testData.length : "not array",
testData
);
} catch (testErr) {
console.error("[Electric] Direct Electric SQL test failed:", testErr);
}
// Use PGlite's electric sync plugin to sync the shape
// According to Electric SQL docs, the shape config uses params for table, where, columns
// Note: mapColumns is OPTIONAL per pglite-sync types.ts
// Create a promise that resolves when initial sync is complete
// Using recommended approach: check isUpToDate immediately, watch stream, shorter timeout
// IMPORTANT: We don't unsubscribe from the stream - it must stay active for real-time updates
let syncResolved = false;
// Initialize with no-op functions to satisfy TypeScript
let resolveInitialSync: () => void = () => {};
let rejectInitialSync: (error: Error) => void = () => {};
const initialSyncPromise = new Promise<void>((resolve, reject) => {
resolveInitialSync = () => {
if (!syncResolved) {
syncResolved = true;
// DON'T unsubscribe from stream - it needs to stay active for real-time updates
resolve();
}
};
rejectInitialSync = (error: Error) => {
if (!syncResolved) {
syncResolved = true;
// DON'T unsubscribe from stream even on error - let Electric handle it
reject(error);
}
};
// Shorter timeout (5 seconds) as fallback
setTimeout(() => {
if (!syncResolved) {
console.warn(
`[Electric] ⚠️ Sync timeout for ${table} - checking isUpToDate one more time...`
);
// Check isUpToDate one more time before resolving
// This will be checked after shape is created
setTimeout(() => {
if (!syncResolved) {
console.warn(
`[Electric] ⚠️ Sync timeout for ${table} - resolving anyway after 5s`
);
resolveInitialSync();
}
}, 100);
}
}, 5000);
});
// Include userId in shapeKey for user-specific sync state
const shapeConfig = {
shape: {
url: `${electricUrl}/v1/shape`,
params: {
table,
...(validatedWhere ? { where: validatedWhere } : {}),
...(columns ? { columns: columns.join(",") } : {}),
},
},
table,
primaryKey,
shapeKey: `${userId}_v${SYNC_VERSION}_${table}_${where?.replace(/[^a-zA-Z0-9]/g, "_") || "all"}`, // User-specific versioned key
onInitialSync: () => {
console.log(
`[Electric] ✅ Initial sync complete for ${table} - data should now be in PGlite`
);
resolveInitialSync();
},
onError: (error: Error) => {
console.error(`[Electric] ❌ Shape sync error for ${table}:`, error);
console.error(
"[Electric] Error details:",
JSON.stringify(error, Object.getOwnPropertyNames(error))
);
rejectInitialSync(error);
},
};
console.log(
"[Electric] syncShapeToTable config:",
JSON.stringify(shapeConfig, null, 2)
);
// Type assertion to PGlite with electric extension
const pgWithElectric = db as PGlite & {
electric: {
syncShapeToTable: (
config: typeof shapeConfig
) => Promise<{ unsubscribe: () => void; isUpToDate: boolean; stream: unknown }>;
};
};
const shape = await pgWithElectric.electric.syncShapeToTable(shapeConfig);
if (!shape) {
throw new Error("syncShapeToTable returned undefined");
}
// Log the actual shape result structure
console.log("[Electric] Shape sync result (initial):", {
hasUnsubscribe: typeof shape?.unsubscribe === "function",
isUpToDate: shape?.isUpToDate,
hasStream: !!shape?.stream,
streamType: typeof shape?.stream,
});
// Recommended Approach Step 1: Check isUpToDate immediately
if (shape.isUpToDate) {
console.log(
`[Electric] ✅ Sync already up-to-date for ${table} (resuming from previous state)`
);
resolveInitialSync();
} else {
// Recommended Approach Step 2: Subscribe to stream and watch for "up-to-date" message
if (shape?.stream) {
const stream = shape.stream as any;
console.log("[Electric] Shape stream details:", {
shapeHandle: stream?.shapeHandle,
lastOffset: stream?.lastOffset,
isUpToDate: stream?.isUpToDate,
error: stream?.error,
hasSubscribe: typeof stream?.subscribe === "function",
hasUnsubscribe: typeof stream?.unsubscribe === "function",
});
// Subscribe to the stream to watch for "up-to-date" control message
// NOTE: We keep this subscription active - don't unsubscribe!
// The stream is what Electric SQL uses for real-time updates
if (typeof stream?.subscribe === "function") {
console.log(
"[Electric] Subscribing to shape stream to watch for up-to-date message..."
);
// Subscribe but don't store unsubscribe - we want it to stay active
stream.subscribe((messages: unknown[]) => {
// Continue receiving updates even after sync is resolved
if (!syncResolved) {
console.log(
"[Electric] 🔵 Shape stream received messages:",
messages?.length || 0
);
}
// Check if any message indicates sync is complete
if (messages && messages.length > 0) {
for (const message of messages) {
const msg = message as any;
// Check for "up-to-date" control message
if (
msg?.headers?.control === "up-to-date" ||
msg?.headers?.electric_up_to_date === "true" ||
(typeof msg === "object" && "up-to-date" in msg)
) {
if (!syncResolved) {
console.log(`[Electric] ✅ Received up-to-date message for ${table}`);
resolveInitialSync();
}
// Continue listening for real-time updates - don't return!
}
}
if (!syncResolved && messages.length > 0) {
console.log(
"[Electric] First message:",
JSON.stringify(messages[0], null, 2)
);
}
}
// Also check stream's isUpToDate property after receiving messages
if (!syncResolved && stream?.isUpToDate) {
console.log(`[Electric] ✅ Stream isUpToDate is true for ${table}`);
resolveInitialSync();
}
});
// Also check stream's isUpToDate property immediately
if (stream?.isUpToDate) {
console.log(`[Electric] ✅ Stream isUpToDate is true immediately for ${table}`);
resolveInitialSync();
}
}
// Also poll isUpToDate periodically as a backup (every 200ms)
const pollInterval = setInterval(() => {
if (syncResolved) {
clearInterval(pollInterval);
return;
}
if (shape.isUpToDate || stream?.isUpToDate) {
console.log(`[Electric] ✅ Sync completed (detected via polling) for ${table}`);
clearInterval(pollInterval);
resolveInitialSync();
}
}, 200);
// Clean up polling when promise resolves
initialSyncPromise.finally(() => {
clearInterval(pollInterval);
});
} else {
console.warn(
`[Electric] ⚠️ No stream available for ${table}, relying on callback and timeout`
);
}
}
// Create the sync handle with proper cleanup
const syncHandle: SyncHandle = {
unsubscribe: () => {
console.log(`[Electric] Unsubscribing from: ${cacheKey}`);
// Remove from cache first
activeSyncHandles.delete(cacheKey);
// Then unsubscribe from the shape
if (shape && typeof shape.unsubscribe === "function") {
shape.unsubscribe();
}
},
// Use getter to always return current state
get isUpToDate() {
return shape?.isUpToDate ?? false;
},
stream: shape?.stream,
initialSyncPromise, // Expose promise so callers can wait for sync
};
// Cache the sync handle for reuse (memory optimization)
activeSyncHandles.set(cacheKey, syncHandle);
console.log(
`[Electric] Cached sync handle for: ${cacheKey} (total cached: ${activeSyncHandles.size})`
);
return syncHandle;
} catch (error) {
console.error("[Electric] Failed to sync shape:", error);
// Check if Electric SQL server is reachable
try {
const response = await fetch(`${electricUrl}/v1/shape?table=${table}&offset=-1`, {
method: "GET",
});
console.log(
"[Electric] Electric SQL server response:",
response.status,
response.statusText
);
if (!response.ok) {
console.error("[Electric] Electric SQL server error:", await response.text());
}
} catch (fetchError) {
console.error("[Electric] Cannot reach Electric SQL server:", fetchError);
console.error("[Electric] Make sure Electric SQL is running at:", electricUrl);
}
throw error;
}
},
};
console.log(`[Electric] ✅ Initialized successfully for user: ${userId}`);
return electricClient;
} catch (error) {
console.error("[Electric] Failed to initialize:", error);
// Reset state on failure
electricClient = null;
currentUserId = null;
throw error;
} finally {
isInitializing = false;
}
})();
return initPromise;
}
/**
* Cleanup Electric SQL - close database and reset singleton
* Called on logout (best-effort) and when switching users
*/
export async function cleanupElectric(): Promise<void> {
if (!electricClient) {
return;
}
const userIdToClean = currentUserId;
console.log(`[Electric] Cleaning up for user: ${userIdToClean}`);
// Unsubscribe from all active sync handles first (memory cleanup)
console.log(`[Electric] Unsubscribing from ${activeSyncHandles.size} active sync handles`);
// Copy keys to array to avoid mutation during iteration
const handleKeys = Array.from(activeSyncHandles.keys());
for (const key of handleKeys) {
const handle = activeSyncHandles.get(key);
if (handle) {
try {
handle.unsubscribe();
} catch (err) {
console.warn(`[Electric] Failed to unsubscribe from ${key}:`, err);
}
}
}
// Ensure cache is empty
activeSyncHandles.clear();
try {
// Close the PGlite database connection
await electricClient.db.close();
console.log("[Electric] Database closed");
} catch (error) {
console.error("[Electric] Error closing database:", error);
}
// Reset singleton state
electricClient = null;
currentUserId = null;
isInitializing = false;
initPromise = null;
// Delete the user's IndexedDB database (best-effort cleanup on logout)
if (typeof window !== "undefined" && window.indexedDB && userIdToClean) {
try {
const dbName = `${DB_PREFIX}${userIdToClean}-v${SYNC_VERSION}`;
window.indexedDB.deleteDatabase(dbName);
console.log(`[Electric] Deleted database: ${dbName}`);
} catch (err) {
console.warn("[Electric] Failed to delete database:", err);
}
}
console.log("[Electric] Cleanup complete");
}
/**
* Get the Electric client (throws if not initialized)
*/
export function getElectric(): ElectricClient {
if (!electricClient) {
throw new Error("Electric not initialized. Call initElectric(userId) first.");
}
return electricClient;
}
/**
* Check if Electric is initialized for a specific user
*/
export function isElectricInitialized(userId?: string): boolean {
if (!electricClient) return false;
if (userId && currentUserId !== userId) return false;
return true;
}
/**
* Get the current user ID that Electric is initialized for
*/
export function getCurrentElectricUserId(): string | null {
return currentUserId;
}
/**
* Get the PGlite database instance
*/
export function getDb(): PGlite | null {
return electricClient?.db ?? null;
}

36
surfsense_web/lib/electric/context.ts Normal file
View file

@ -0,0 +1,36 @@
"use client";
import { createContext, useContext } from "react";
import type { ElectricClient } from "./client";
/**
* Context for sharing the Electric SQL client across the app
*
* This ensures:
* 1. Single initialization point (ElectricProvider only)
* 2. No race conditions (hooks wait for context)
* 3. Clean cleanup (ElectricProvider manages lifecycle)
*/
export const ElectricContext = createContext<ElectricClient | null>(null);
/**
* Hook to get the Electric client from context
* Returns null if Electric is not initialized yet
*/
export function useElectricClient(): ElectricClient | null {
return useContext(ElectricContext);
}
/**
* Hook to get the Electric client, throwing if not available
* Use this when you're sure Electric should be initialized
*/
export function useElectricClientOrThrow(): ElectricClient {
const client = useContext(ElectricContext);
if (!client) {
throw new Error(
"Electric client not available. Make sure you're inside ElectricProvider and user is authenticated."
);
}
return client;
}

4
surfsense_web/messages/en.json
View file

@ -88,7 +88,7 @@
"description_label": "Description",
"description_placeholder": "What is this search space for?",
"create_button": "Create",
"creating": "Creating...",
"creating": "Creating",
"all_search_spaces": "All Search Spaces",
"search_spaces_count": "{count, plural, =0 {No search spaces} =1 {1 search space} other {# search spaces}}",
"no_search_spaces": "No search spaces yet",
@ -658,7 +658,7 @@
"sidebar": {
"chats": "Private Chats",
"shared_chats": "Shared Chats",
"search_chats": "Search chats...",
"search_chats": "Search chats",
"no_chats_found": "No chats found",
"no_shared_chats": "No shared chats",
"view_all_shared_chats": "View all shared chats",

7
surfsense_web/package.json
View file

@ -29,6 +29,10 @@
"@blocknote/mantine": "^0.45.0",
"@blocknote/react": "^0.45.0",
"@blocknote/server-util": "^0.45.0",
"@electric-sql/client": "^1.4.0",
"@electric-sql/pglite": "^0.3.14",
"@electric-sql/pglite-sync": "^0.4.0",
"@electric-sql/react": "^1.0.26",
"@hookform/resolvers": "^5.2.2",
"@number-flow/react": "^0.5.10",
"@posthog/react": "^1.5.2",
@ -120,6 +124,7 @@
"eslint-config-next": "15.2.0",
"tailwindcss": "^4.1.11",
"tsx": "^4.20.6",
"typescript": "^5.8.3"
"typescript": "^5.8.3",
"vite": "^7.3.1"
}
}

673
surfsense_web/pnpm-lock.yaml generated
View file

File diff suppressed because it is too large Load diff