diff --git a/surfsense_backend/app/routes/logs_routes.py b/surfsense_backend/app/routes/logs_routes.py index e7e00280e..b82e02077 100644 --- a/surfsense_backend/app/routes/logs_routes.py +++ b/surfsense_backend/app/routes/logs_routes.py @@ -322,6 +322,9 @@ async def get_logs_summary( document_id = ( log.log_metadata.get("document_id") if log.log_metadata else None ) + connector_id = ( + log.log_metadata.get("connector_id") if log.log_metadata else None + ) summary["active_tasks"].append( { "id": log.id, @@ -330,6 +333,7 @@ async def get_logs_summary( "started_at": log.created_at, "source": log.source, "document_id": document_id, + "connector_id": connector_id, } ) diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index c61fcfe78..019f5796a 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -125,14 +125,22 @@ export default function DocumentsTable() { setColumnVisibility((prev) => ({ ...prev, [id]: checked })); }; + const [isRefreshing, setIsRefreshing] = useState(false); + const refreshCurrentView = useCallback(async () => { - if (debouncedSearch.trim()) { - await refetchSearch(); - } else { - await refetchDocuments(); + if (isRefreshing) return; + setIsRefreshing(true); + try { + if (debouncedSearch.trim()) { + await refetchSearch(); + } else { + await refetchDocuments(); + } + toast.success(t("refresh_success") || "Documents refreshed"); + } finally { + setIsRefreshing(false); } - toast.success(t("refresh_success") || "Documents refreshed"); - }, [debouncedSearch, refetchSearch, refetchDocuments, t]); + }, [debouncedSearch, refetchSearch, refetchDocuments, t, isRefreshing]); // Set up smart polling for active tasks - only polls when tasks are in progress const { summary } = useLogsSummary(searchSpaceId, 24, { @@ -230,8 +238,8 @@ export default function DocumentsTable() {

{t("title")}

{t("subtitle")}

- diff --git a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx index b6e4ed108..0070b20be 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx @@ -472,9 +472,18 @@ export default function LogsManagePage() { } }; + const [isRefreshing, setIsRefreshing] = useState(false); + const [isSummaryRefreshing, setIsSummaryRefreshing] = useState(false); + const handleRefresh = async () => { - await Promise.all([refreshLogs(), refreshSummary()]); - toast.success("Logs refreshed"); + if (isRefreshing) return; + setIsRefreshing(true); + try { + await Promise.all([refreshLogs(), refreshSummary()]); + toast.success("Logs refreshed"); + } finally { + setIsRefreshing(false); + } }; return ( @@ -495,7 +504,16 @@ export default function LogsManagePage() { summary={summary} loading={summaryLoading} error={summaryError?.message ?? null} - onRefresh={refreshSummary} + onRefresh={async () => { + if (isSummaryRefreshing) return; + setIsSummaryRefreshing(true); + try { + await refreshSummary(); + } finally { + setIsSummaryRefreshing(false); + } + }} + isRefreshing={isSummaryRefreshing} /> {/* Logs Table Header */} @@ -509,8 +527,8 @@ export default function LogsManagePage() {

{t("title")}

{t("subtitle")}

- @@ -546,11 +564,13 @@ function LogsSummaryDashboard({ loading, error, onRefresh, + isRefreshing = false, }: { summary: any; loading: boolean; error: string | null; - onRefresh: () => void; + onRefresh: () => void | Promise; + isRefreshing?: boolean; }) { const t = useTranslations("logs"); if (loading) { @@ -581,7 +601,8 @@ function LogsSummaryDashboard({

{t("failed_load_summary")}

-
diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index f51a80f98..ba80f0d7e 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -136,12 +136,8 @@ export const ConnectorIndicator: FC = () => { if (!logsSummary?.active_tasks) return new Set(); return new Set( logsSummary.active_tasks - .filter((task) => task.source?.includes("connector_indexing")) - .map((task) => { - const match = task.source?.match(/connector[_-]?(\d+)/i); - return match ? parseInt(match[1], 10) : null; - }) - .filter((id): id is number => id !== null) + .filter((task) => task.source?.includes("connector_indexing") && task.connector_id != null) + .map((task) => task.connector_id as number) ); }, [logsSummary?.active_tasks]); @@ -261,19 +257,22 @@ export const ConnectorIndicator: FC = () => {
- - - + + + void; onManage?: () => void; } +/** + * Extract a number from the active task message for display + * Looks for patterns like "45 indexed", "Processing 123", etc. + */ +function extractIndexedCount(message: string | undefined): number | null { + if (!message) return null; + // Try to find a number in the message + const match = message.match(/(\d+)/); + return match ? parseInt(match[1], 10) : null; +} + export const ConnectorCard: FC = ({ id, title, @@ -23,9 +38,53 @@ export const ConnectorCard: FC = ({ connectorType, isConnected = false, isConnecting = false, + documentCount, + isIndexing = false, + activeTask, onConnect, onManage, }) => { + // Extract count from active task message during indexing + const indexingCount = extractIndexedCount(activeTask?.message); + + // Determine the status content to display + const getStatusContent = () => { + if (isIndexing) { + return ( +
+ + {indexingCount !== null ? ( + <>{indexingCount.toLocaleString()} indexed + ) : ( + "Syncing..." + )} + + {/* Indeterminate progress bar with animation */} +
+
+
+
+ ); + } + + if (isConnected) { + if (documentCount !== undefined && documentCount > 0) { + return ( + + + + {documentCount.toLocaleString()} document{documentCount !== 1 ? "s" : ""} + + + ); + } + // Fallback for connected but no documents yet + return No documents indexed; + } + + return description; + }; + return (
@@ -35,19 +94,21 @@ export const ConnectorCard: FC = ({
{title}
-

- {isConnected ? "Connected" : description} -

+
+ {getStatusContent()} +
@@ -69,9 +69,7 @@ export const ActiveConnectorsTab: FC = ({ {connectors.map((connector) => { const isIndexing = indexingConnectorIds.has(connector.id); const activeTask = logsSummary?.active_tasks?.find( - (task: LogActiveTask) => - task.source?.includes(`connector_${connector.id}`) || - task.source?.includes(`connector-${connector.id}`) + (task: LogActiveTask) => task.connector_id === connector.id ); return ( diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index 6143f1d54..74ae3660f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -3,8 +3,10 @@ import { useRouter } from "next/navigation"; import { type FC } from "react"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types"; import { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants"; import { ConnectorCard } from "../components/connector-card"; +import { getDocumentCountForConnector } from "../utils/connector-document-mapping"; interface AllConnectorsTabProps { searchQuery: string; @@ -12,6 +14,9 @@ interface AllConnectorsTabProps { connectedTypes: Set; connectingId: string | null; allConnectors: SearchSourceConnector[] | undefined; + documentTypeCounts?: Record; + indexingConnectorIds?: Set; + logsSummary?: LogSummary; onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[number]) => void; onConnectNonOAuth?: (connectorType: string) => void; onCreateWebcrawler?: () => void; @@ -24,6 +29,9 @@ export const AllConnectorsTab: FC = ({ connectedTypes, connectingId, allConnectors, + documentTypeCounts, + indexingConnectorIds, + logsSummary, onConnectOAuth, onConnectNonOAuth, onCreateWebcrawler, @@ -31,6 +39,14 @@ export const AllConnectorsTab: FC = ({ }) => { const router = useRouter(); + // Helper to find active task for a connector + const getActiveTaskForConnector = (connectorId: number): LogActiveTask | undefined => { + if (!logsSummary?.active_tasks) return undefined; + return logsSummary.active_tasks.find( + (task: LogActiveTask) => task.connector_id === connectorId + ); + }; + // Filter connectors based on search const filteredOAuth = OAUTH_CONNECTORS.filter( (c) => @@ -55,28 +71,35 @@ export const AllConnectorsTab: FC = ({
- {filteredOAuth.map((connector) => { - const isConnected = connectedTypes.has(connector.connectorType); - const isConnecting = connectingId === connector.id; - // Find the actual connector object if connected - const actualConnector = isConnected && allConnectors - ? allConnectors.find((c: SearchSourceConnector) => c.connector_type === connector.connectorType) - : undefined; + {filteredOAuth.map((connector) => { + const isConnected = connectedTypes.has(connector.connectorType); + const isConnecting = connectingId === connector.id; + // Find the actual connector object if connected + const actualConnector = isConnected && allConnectors + ? allConnectors.find((c: SearchSourceConnector) => c.connector_type === connector.connectorType) + : undefined; + + const documentCount = getDocumentCountForConnector(connector.connectorType, documentTypeCounts); + const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id); + const activeTask = actualConnector ? getActiveTaskForConnector(actualConnector.id) : undefined; - return ( - onConnectOAuth(connector)} - onManage={actualConnector && onManage ? () => onManage(actualConnector) : undefined} - /> - ); - })} + return ( + onConnectOAuth(connector)} + onManage={actualConnector && onManage ? () => onManage(actualConnector) : undefined} + /> + ); + })}
)} @@ -109,34 +132,41 @@ export const AllConnectorsTab: FC = ({ const isClickUp = connector.id === "clickup-connector"; const isLuma = connector.id === "luma-connector"; - const isConnected = connectedTypes.has(connector.connectorType); - const isConnecting = connectingId === connector.id; - - // Find the actual connector object if connected - const actualConnector = isConnected && allConnectors - ? allConnectors.find((c: SearchSourceConnector) => c.connector_type === connector.connectorType) - : undefined; + const isConnected = connectedTypes.has(connector.connectorType); + const isConnecting = connectingId === connector.id; + + // Find the actual connector object if connected + const actualConnector = isConnected && allConnectors + ? allConnectors.find((c: SearchSourceConnector) => c.connector_type === connector.connectorType) + : undefined; - const handleConnect = isWebcrawler && onCreateWebcrawler - ? onCreateWebcrawler - : (isTavily || isSearxng || isLinkup || isBaidu || isLinear || isElasticsearch || isSlack || isDiscord || isNotion || isConfluence || isBookStack || isGithub || isJira || isClickUp || isLuma) && onConnectNonOAuth - ? () => onConnectNonOAuth(connector.connectorType) - : () => router.push(`/dashboard/${searchSpaceId}/connectors/add/${connector.id}`); + const documentCount = getDocumentCountForConnector(connector.connectorType, documentTypeCounts); + const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id); + const activeTask = actualConnector ? getActiveTaskForConnector(actualConnector.id) : undefined; - return ( - onManage(actualConnector) : undefined} - /> - ); - })} + const handleConnect = isWebcrawler && onCreateWebcrawler + ? onCreateWebcrawler + : (isTavily || isSearxng || isLinkup || isBaidu || isLinear || isElasticsearch || isSlack || isDiscord || isNotion || isConfluence || isBookStack || isGithub || isJira || isClickUp || isLuma) && onConnectNonOAuth + ? () => onConnectNonOAuth(connector.connectorType) + : () => router.push(`/dashboard/${searchSpaceId}/connectors/add/${connector.id}`); + + return ( + onManage(actualConnector) : undefined} + /> + ); + })}
)} diff --git a/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts b/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts new file mode 100644 index 000000000..a0afa7431 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts @@ -0,0 +1,64 @@ +"use client"; + +/** + * Maps SearchSourceConnectorType to DocumentType for fetching document counts + * + * Note: Some connectors don't have a direct 1:1 mapping to document types: + * - Search API connectors (TAVILY_API, SEARXNG_API, etc.) don't index documents + * - WEBCRAWLER_CONNECTOR maps to CRAWLED_URL document type + * - GOOGLE_DRIVE_CONNECTOR maps to GOOGLE_DRIVE_FILE document type + */ +export const CONNECTOR_TO_DOCUMENT_TYPE: Record = { + // Direct mappings (connector type matches document type) + SLACK_CONNECTOR: "SLACK_CONNECTOR", + NOTION_CONNECTOR: "NOTION_CONNECTOR", + GITHUB_CONNECTOR: "GITHUB_CONNECTOR", + LINEAR_CONNECTOR: "LINEAR_CONNECTOR", + DISCORD_CONNECTOR: "DISCORD_CONNECTOR", + JIRA_CONNECTOR: "JIRA_CONNECTOR", + CONFLUENCE_CONNECTOR: "CONFLUENCE_CONNECTOR", + CLICKUP_CONNECTOR: "CLICKUP_CONNECTOR", + GOOGLE_CALENDAR_CONNECTOR: "GOOGLE_CALENDAR_CONNECTOR", + GOOGLE_GMAIL_CONNECTOR: "GOOGLE_GMAIL_CONNECTOR", + AIRTABLE_CONNECTOR: "AIRTABLE_CONNECTOR", + LUMA_CONNECTOR: "LUMA_CONNECTOR", + ELASTICSEARCH_CONNECTOR: "ELASTICSEARCH_CONNECTOR", + BOOKSTACK_CONNECTOR: "BOOKSTACK_CONNECTOR", + + // Special mappings (connector type differs from document type) + GOOGLE_DRIVE_CONNECTOR: "GOOGLE_DRIVE_FILE", + WEBCRAWLER_CONNECTOR: "CRAWLED_URL", +}; + +/** + * Get the document type for a given connector type + * Returns undefined if the connector doesn't index documents (e.g., search APIs) + */ +export function getDocumentTypeForConnector( + connectorType: string +): string | undefined { + return CONNECTOR_TO_DOCUMENT_TYPE[connectorType]; +} + +/** + * Get document count for a specific connector type from document type counts + */ +export function getDocumentCountForConnector( + connectorType: string, + documentTypeCounts: Record | undefined +): number | undefined { + if (!documentTypeCounts) return undefined; + + const documentType = getDocumentTypeForConnector(connectorType); + if (!documentType) return undefined; + + return documentTypeCounts[documentType]; +} + +/** + * Check if a connector type is indexable (produces documents) + */ +export function isIndexableConnectorType(connectorType: string): boolean { + return connectorType in CONNECTOR_TO_DOCUMENT_TYPE; +} + diff --git a/surfsense_web/contracts/types/log.types.ts b/surfsense_web/contracts/types/log.types.ts index ac81d2d0d..eb9f7fe6c 100644 --- a/surfsense_web/contracts/types/log.types.ts +++ b/surfsense_web/contracts/types/log.types.ts @@ -86,6 +86,7 @@ export const logActiveTask = z.object({ started_at: z.string(), source: z.string().nullable().optional(), document_id: z.number().nullable().optional(), + connector_id: z.number().nullable().optional(), }); export const logFailure = z.object({ id: z.number(), diff --git a/surfsense_web/tailwind.config.js b/surfsense_web/tailwind.config.js index 9c4601f68..8bd78ed2b 100644 --- a/surfsense_web/tailwind.config.js +++ b/surfsense_web/tailwind.config.js @@ -65,10 +65,16 @@ module.exports = { from: { height: "var(--radix-accordion-content-height)" }, to: { height: 0 }, }, + "progress-indeterminate": { + "0%": { left: "-33%", width: "33%" }, + "50%": { width: "50%" }, + "100%": { left: "100%", width: "33%" }, + }, }, animation: { "accordion-down": "accordion-down 0.2s ease-out", "accordion-up": "accordion-up 0.2s ease-out", + "progress-indeterminate": "progress-indeterminate 1.5s ease-in-out infinite", }, }, },