From 5d1859db1748b80a7beb0320b8d6eb83e9dc16c9 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 31 Dec 2025 11:41:57 +0530 Subject: [PATCH] feat: Add Webcrawler and YouTube connector configurations, enhance connector dialog with creation functionality, and improve UI responsiveness and styling across components. --- .../assistant-ui/connector-popup.tsx | 4 + .../components/date-range-selector.tsx | 20 +-- .../components/periodic-sync-config.tsx | 22 +-- .../components/google-drive-config.tsx | 4 +- .../components/webcrawler-config.tsx | 127 ++++++++++++++ .../components/youtube-config.tsx | 148 ++++++++++++++++ .../connector-configs/index.tsx | 6 + .../views/connector-edit-view.tsx | 27 +-- .../views/indexing-configuration-view.tsx | 24 +-- .../constants/connector-constants.ts | 6 + .../hooks/use-connector-dialog.ts | 166 ++++++++++++++++-- .../tabs/all-connectors-tab.tsx | 21 ++- surfsense_web/contracts/enums/connector.ts | 1 + .../contracts/enums/connectorIcons.tsx | 2 + .../contracts/types/connector.types.ts | 1 + 15 files changed, 512 insertions(+), 67 deletions(-) create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/youtube-config.tsx diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index c1ed38e7b..55a11f420 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -88,6 +88,8 @@ export const ConnectorIndicator: FC = () => { handleTabChange, handleScroll, handleConnectOAuth, + handleCreateWebcrawler, + handleCreateYouTube, handleStartIndexing, handleSkipIndexing, handleStartEdit, @@ -212,6 +214,8 @@ export const ConnectorIndicator: FC = () => { connectingId={connectingId} allConnectors={allConnectors} onConnectOAuth={handleConnectOAuth} + onCreateWebcrawler={handleCreateWebcrawler} + onCreateYouTube={handleCreateYouTube} onManage={handleStartEdit} /> diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx index 1112f3f36..5c7870639 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/date-range-selector.tsx @@ -40,23 +40,23 @@ export const DateRangeSelector: FC = ({ }; return ( -
-

Select Date Range

-

+

+

Select Date Range

+

Choose how far back you want to sync your data. You can always re-index later with different dates.

{/* Start Date */}
- + @@ -120,7 +120,7 @@ export const DateRangeSelector: FC = ({ variant="outline" size="sm" onClick={handleLast30Days} - className="text-xs bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10" + className="text-xs sm:text-sm bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10" > Last 30 Days @@ -129,7 +129,7 @@ export const DateRangeSelector: FC = ({ variant="outline" size="sm" onClick={handleLastYear} - className="text-xs bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10" + className="text-xs sm:text-sm bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10" > Last Year diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx index 427a6ac86..f8b869a67 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/periodic-sync-config.tsx @@ -25,11 +25,11 @@ export const PeriodicSyncConfig: FC = ({ onFrequencyChange, }) => { return ( -
+
-

Enable Periodic Sync

-

+

Enable Periodic Sync

+

Automatically re-index at regular intervals

@@ -39,21 +39,21 @@ export const PeriodicSyncConfig: FC = ({ {enabled && (
- +
diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx index 280d6ed23..a30450dad 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/google-drive-config.tsx @@ -91,9 +91,9 @@ export const GoogleDriveConfig: FC = ({ )} - + - + Folder selection is used when indexing. You can change this selection when you start indexing. diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx new file mode 100644 index 000000000..4bb75c58d --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx @@ -0,0 +1,127 @@ +"use client"; + +import { Info } from "lucide-react"; +import { useState, useEffect } from "react"; +import type { FC } from "react"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Textarea } from "@/components/ui/textarea"; +import type { ConnectorConfigProps } from "../index"; + +export const WebcrawlerConfig: FC = ({ + connector, + onConfigChange, +}) => { + // Initialize with existing config values + const existingApiKey = (connector.config?.FIRECRAWL_API_KEY as string | undefined) || ""; + const existingUrls = (connector.config?.INITIAL_URLS as string | undefined) || ""; + + const [apiKey, setApiKey] = useState(existingApiKey); + const [initialUrls, setInitialUrls] = useState(existingUrls); + const [showApiKey, setShowApiKey] = useState(false); + + // Update state when connector config changes + useEffect(() => { + const apiKeyValue = (connector.config?.FIRECRAWL_API_KEY as string | undefined) || ""; + const urlsValue = (connector.config?.INITIAL_URLS as string | undefined) || ""; + setApiKey(apiKeyValue); + setInitialUrls(urlsValue); + }, [connector.config]); + + const handleApiKeyChange = (value: string) => { + setApiKey(value); + if (onConfigChange) { + onConfigChange({ + ...connector.config, + FIRECRAWL_API_KEY: value.trim() || undefined, + }); + } + }; + + const handleUrlsChange = (value: string) => { + setInitialUrls(value); + if (onConfigChange) { + onConfigChange({ + ...connector.config, + INITIAL_URLS: value.trim() || undefined, + }); + } + }; + + return ( +
+
+

Web Crawler Configuration

+

+ Configure your web crawler settings. You can add a Firecrawl API key for enhanced crawling or use the free fallback option. +

+
+ + {/* API Key Field */} +
+ +
+ handleApiKeyChange(e.target.value)} + className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 text-xs sm:text-sm pr-10" + /> + +
+

+ Get your API key from{" "} + + firecrawl.dev + + . If not provided, will use AsyncChromiumLoader as fallback. +

+
+ + {/* Initial URLs Field */} +
+ +