feat: Add Webcrawler and YouTube connector configurations, enhance connector dialog with creation functionality, and improve UI responsiveness and styling across components.

This commit is contained in:
Anish Sarkar 2025-12-31 11:41:57 +05:30
parent ddfbb9509b
commit 5d1859db17
15 changed files with 512 additions and 67 deletions

View file

@ -88,6 +88,8 @@ export const ConnectorIndicator: FC = () => {
handleTabChange,
handleScroll,
handleConnectOAuth,
handleCreateWebcrawler,
handleCreateYouTube,
handleStartIndexing,
handleSkipIndexing,
handleStartEdit,
@ -212,6 +214,8 @@ export const ConnectorIndicator: FC = () => {
connectingId={connectingId}
allConnectors={allConnectors}
onConnectOAuth={handleConnectOAuth}
onCreateWebcrawler={handleCreateWebcrawler}
onCreateYouTube={handleCreateYouTube}
onManage={handleStartEdit}
/>
</TabsContent>

View file

@ -40,23 +40,23 @@ export const DateRangeSelector: FC<DateRangeSelectorProps> = ({
};
return (
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-6">
<h3 className="font-medium mb-4">Select Date Range</h3>
<p className="text-sm text-muted-foreground mb-6">
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6">
<h3 className="font-medium text-xs sm:text-base mb-4">Select Date Range</h3>
<p className="text-xs sm:text-sm text-muted-foreground mb-6">
Choose how far back you want to sync your data. You can always re-index later with different dates.
</p>
<div className="grid grid-cols-1 sm:grid-cols-2 gap-4">
{/* Start Date */}
<div className="space-y-2">
<Label htmlFor="start-date">Start Date</Label>
<Label htmlFor="start-date" className="text-xs sm:text-sm">Start Date</Label>
<Popover>
<PopoverTrigger asChild>
<Button
id="start-date"
variant="outline"
className={cn(
"w-full justify-start text-left font-normal bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20",
"w-full justify-start text-left font-normal bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm",
!startDate && "text-muted-foreground"
)}
>
@ -77,14 +77,14 @@ export const DateRangeSelector: FC<DateRangeSelectorProps> = ({
{/* End Date */}
<div className="space-y-2">
<Label htmlFor="end-date">End Date</Label>
<Label htmlFor="end-date" className="text-xs sm:text-sm">End Date</Label>
<Popover>
<PopoverTrigger asChild>
<Button
id="end-date"
variant="outline"
className={cn(
"w-full justify-start text-left font-normal bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20",
"w-full justify-start text-left font-normal bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm",
!endDate && "text-muted-foreground"
)}
>
@ -111,7 +111,7 @@ export const DateRangeSelector: FC<DateRangeSelectorProps> = ({
variant="outline"
size="sm"
onClick={handleClearDates}
className="text-xs bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10"
className="text-xs sm:text-sm bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10"
>
Clear Dates
</Button>
@ -120,7 +120,7 @@ export const DateRangeSelector: FC<DateRangeSelectorProps> = ({
variant="outline"
size="sm"
onClick={handleLast30Days}
className="text-xs bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10"
className="text-xs sm:text-sm bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10"
>
Last 30 Days
</Button>
@ -129,7 +129,7 @@ export const DateRangeSelector: FC<DateRangeSelectorProps> = ({
variant="outline"
size="sm"
onClick={handleLastYear}
className="text-xs bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10"
className="text-xs sm:text-sm bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 hover:bg-slate-400/10 dark:hover:bg-slate-400/10"
>
Last Year
</Button>

View file

@ -25,11 +25,11 @@ export const PeriodicSyncConfig: FC<PeriodicSyncConfigProps> = ({
onFrequencyChange,
}) => {
return (
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-6">
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6">
<div className="flex items-center justify-between">
<div className="space-y-1">
<h3 className="font-medium">Enable Periodic Sync</h3>
<p className="text-sm text-muted-foreground">
<h3 className="font-medium text-xs sm:text-base">Enable Periodic Sync</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Automatically re-index at regular intervals
</p>
</div>
@ -39,21 +39,21 @@ export const PeriodicSyncConfig: FC<PeriodicSyncConfigProps> = ({
{enabled && (
<div className="mt-4 pt-4 border-t border-border/100 space-y-3">
<div className="space-y-2">
<Label htmlFor="frequency">Sync Frequency</Label>
<Label htmlFor="frequency" className="text-xs sm:text-sm">Sync Frequency</Label>
<Select value={frequencyMinutes} onValueChange={onFrequencyChange}>
<SelectTrigger
id="frequency"
className="w-full bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20"
className="w-full bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
>
<SelectValue placeholder="Select frequency" />
</SelectTrigger>
<SelectContent className="z-[100]">
<SelectItem value="15">Every 15 minutes</SelectItem>
<SelectItem value="60">Every hour</SelectItem>
<SelectItem value="360">Every 6 hours</SelectItem>
<SelectItem value="720">Every 12 hours</SelectItem>
<SelectItem value="1440">Daily</SelectItem>
<SelectItem value="10080">Weekly</SelectItem>
<SelectItem value="15" className="text-xs sm:text-sm">Every 15 minutes</SelectItem>
<SelectItem value="60" className="text-xs sm:text-sm">Every hour</SelectItem>
<SelectItem value="360" className="text-xs sm:text-sm">Every 6 hours</SelectItem>
<SelectItem value="720" className="text-xs sm:text-sm">Every 12 hours</SelectItem>
<SelectItem value="1440" className="text-xs sm:text-sm">Daily</SelectItem>
<SelectItem value="10080" className="text-xs sm:text-sm">Weekly</SelectItem>
</SelectContent>
</Select>
</div>

View file

@ -91,9 +91,9 @@ export const GoogleDriveConfig: FC<ConnectorConfigProps> = ({
</Button>
)}
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 p-2 sm:p-3">
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 p-2 sm:p-3 flex items-center gap-2 [&>svg]:relative [&>svg]:left-0 [&>svg]:top-0 [&>svg+div]:translate-y-0">
<Info className="h-3 w-3 sm:h-4 sm:w-4 shrink-0" />
<AlertDescription className="text-[10px] sm:text-xs">
<AlertDescription className="text-[10px] sm:text-xs !pl-0">
Folder selection is used when indexing. You can change this selection when you start indexing.
</AlertDescription>
</Alert>

View file

@ -0,0 +1,127 @@
"use client";
import { Info } from "lucide-react";
import { useState, useEffect } from "react";
import type { FC } from "react";
import { Alert, AlertDescription } from "@/components/ui/alert";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Textarea } from "@/components/ui/textarea";
import type { ConnectorConfigProps } from "../index";
export const WebcrawlerConfig: FC<ConnectorConfigProps> = ({
connector,
onConfigChange,
}) => {
// Initialize with existing config values
const existingApiKey = (connector.config?.FIRECRAWL_API_KEY as string | undefined) || "";
const existingUrls = (connector.config?.INITIAL_URLS as string | undefined) || "";
const [apiKey, setApiKey] = useState(existingApiKey);
const [initialUrls, setInitialUrls] = useState(existingUrls);
const [showApiKey, setShowApiKey] = useState(false);
// Update state when connector config changes
useEffect(() => {
const apiKeyValue = (connector.config?.FIRECRAWL_API_KEY as string | undefined) || "";
const urlsValue = (connector.config?.INITIAL_URLS as string | undefined) || "";
setApiKey(apiKeyValue);
setInitialUrls(urlsValue);
}, [connector.config]);
const handleApiKeyChange = (value: string) => {
setApiKey(value);
if (onConfigChange) {
onConfigChange({
...connector.config,
FIRECRAWL_API_KEY: value.trim() || undefined,
});
}
};
const handleUrlsChange = (value: string) => {
setInitialUrls(value);
if (onConfigChange) {
onConfigChange({
...connector.config,
INITIAL_URLS: value.trim() || undefined,
});
}
};
return (
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-4 sm:space-y-6">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base">Web Crawler Configuration</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Configure your web crawler settings. You can add a Firecrawl API key for enhanced crawling or use the free fallback option.
</p>
</div>
{/* API Key Field */}
<div className="space-y-2">
<Label htmlFor="api-key" className="text-xs sm:text-sm">
Firecrawl API Key (Optional)
</Label>
<div className="relative">
<Input
id="api-key"
type={showApiKey ? "text" : "password"}
placeholder="fc-xxxxxxxxxxxxx"
value={apiKey}
onChange={(e) => handleApiKeyChange(e.target.value)}
className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 text-xs sm:text-sm pr-10"
/>
<Button
type="button"
variant="ghost"
size="sm"
onClick={() => setShowApiKey(!showApiKey)}
className="absolute right-1 top-1/2 -translate-y-1/2 h-7 px-2 text-xs text-muted-foreground hover:text-foreground"
>
{showApiKey ? "Hide" : "Show"}
</Button>
</div>
<p className="text-[10px] sm:text-xs text-muted-foreground">
Get your API key from{" "}
<a
href="https://firecrawl.dev"
target="_blank"
rel="noopener noreferrer"
className="text-primary hover:underline"
>
firecrawl.dev
</a>
. If not provided, will use AsyncChromiumLoader as fallback.
</p>
</div>
{/* Initial URLs Field */}
<div className="space-y-2">
<Label htmlFor="initial-urls" className="text-xs sm:text-sm">
Initial URLs (Optional)
</Label>
<Textarea
id="initial-urls"
placeholder="https://example.com&#10;https://docs.example.com&#10;https://blog.example.com"
value={initialUrls}
onChange={(e) => handleUrlsChange(e.target.value)}
className="min-h-[100px] font-mono text-xs sm:text-sm bg-slate-400/5 dark:bg-white/5 border-slate-400/20 resize-none"
/>
<p className="text-[10px] sm:text-xs text-muted-foreground">
Enter URLs to crawl (one per line). You can add more URLs later.
</p>
</div>
{/* Info Alert */}
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 p-2 sm:p-3 flex items-center gap-2 [&>svg]:relative [&>svg]:left-0 [&>svg]:top-0 [&>svg+div]:translate-y-0">
<Info className="h-3 w-3 sm:h-4 sm:w-4 shrink-0" />
<AlertDescription className="text-[10px] sm:text-xs !pl-0">
Configuration is saved when you start indexing. You can update these settings anytime from the connector management page.
</AlertDescription>
</Alert>
</div>
);
};

View file

@ -0,0 +1,148 @@
"use client";
import { IconBrandYoutube } from "@tabler/icons-react";
import { Info } from "lucide-react";
import { TagInput, type Tag as TagType } from "emblor";
import { useState, useEffect } from "react";
import type { FC } from "react";
import { Alert, AlertDescription } from "@/components/ui/alert";
import { Button } from "@/components/ui/button";
import { Label } from "@/components/ui/label";
import { toast } from "sonner";
import type { ConnectorConfigProps } from "../index";
const youtubeRegex =
/^(https:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})$/;
export const YouTubeConfig: FC<ConnectorConfigProps> = ({
connector,
onConfigChange,
}) => {
// Initialize with existing YouTube URLs from connector config
const existingUrls = (connector.config?.youtube_urls as string[] | undefined) || [];
const [youtubeTags, setYoutubeTags] = useState<TagType[]>(
existingUrls.map((url) => ({
id: url,
text: url,
}))
);
const [activeTagIndex, setActiveTagIndex] = useState<number | null>(null);
// Update YouTube tags when connector config changes
useEffect(() => {
const urls = (connector.config?.youtube_urls as string[] | undefined) || [];
setYoutubeTags(
urls.map((url) => ({
id: url,
text: url,
}))
);
}, [connector.config]);
const isValidYoutubeUrl = (url: string): boolean => {
return youtubeRegex.test(url);
};
const handleTagsChange = (tags: TagType[]) => {
setYoutubeTags(tags);
if (onConfigChange) {
// Extract URLs from tags and validate
const urls = tags.map((tag) => tag.text).filter(isValidYoutubeUrl);
onConfigChange({
...connector.config,
youtube_urls: urls,
});
}
};
const handleAddTag = (text: string) => {
if (!isValidYoutubeUrl(text)) {
toast("Invalid YouTube URL", {
description: "Please enter a valid YouTube video URL (youtube.com/watch?v= or youtu.be/)",
});
return;
}
if (youtubeTags.some((tag) => tag.text === text)) {
toast("Duplicate URL", {
description: "This YouTube video has already been added",
});
return;
}
const newTag: TagType = {
id: Date.now().toString(),
text: text,
};
handleTagsChange([...youtubeTags, newTag]);
};
return (
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-3 sm:space-y-4">
<div className="space-y-1 sm:space-y-2">
<h3 className="font-medium text-sm sm:text-base flex items-center gap-2">
<IconBrandYoutube className="h-4 w-4" />
YouTube Video URLs
</h3>
<p className="text-xs sm:text-sm text-muted-foreground">
Add YouTube video URLs to index. Enter a URL and press Enter to add multiple videos.
</p>
</div>
<div className="space-y-2">
<Label htmlFor="youtube-urls" className="text-xs sm:text-sm">
Enter YouTube Video URLs
</Label>
<TagInput
id="youtube-urls"
tags={youtubeTags}
setTags={handleTagsChange}
placeholder="Enter a YouTube URL and press Enter"
onAddTag={handleAddTag}
styleClasses={{
inlineTagsContainer:
"border-input rounded-lg bg-background shadow-sm shadow-black/5 transition-shadow focus-within:border-ring focus-within:outline-none focus-within:ring-[3px] focus-within:ring-ring/20 p-1 gap-1",
input: "w-full min-w-[80px] focus-visible:outline-none shadow-none px-2 h-7 text-xs sm:text-sm",
tag: {
body: "h-7 relative bg-background border border-input hover:bg-background rounded-md font-medium text-xs ps-2 pe-7 flex",
closeButton:
"absolute -inset-y-px -end-px p-0 rounded-e-lg flex size-7 transition-colors outline-0 focus-visible:outline focus-visible:outline-2 focus-visible:outline-ring/70 text-muted-foreground/80 hover:text-foreground",
},
}}
activeTagIndex={activeTagIndex}
setActiveTagIndex={setActiveTagIndex}
/>
<p className="text-[10px] sm:text-xs text-muted-foreground">
Add multiple YouTube URLs by pressing Enter after each one
</p>
</div>
{youtubeTags.length > 0 && (
<div className="p-2 sm:p-3 bg-muted rounded-lg text-xs sm:text-sm space-y-1 sm:space-y-2">
<p className="font-medium">
{youtubeTags.length} video{youtubeTags.length > 1 ? "s" : ""} added
</p>
</div>
)}
<div className="bg-muted/50 rounded-lg p-3 sm:p-4 text-xs sm:text-sm">
<h4 className="font-medium mb-2">Tips for adding YouTube videos:</h4>
<ul className="list-disc pl-5 space-y-1 text-muted-foreground">
<li>Use standard YouTube URLs (youtube.com/watch?v= or youtu.be/)</li>
<li>Make sure videos are publicly accessible</li>
<li>Supported formats: youtube.com/watch?v=VIDEO_ID or youtu.be/VIDEO_ID</li>
<li>Processing may take some time depending on video length</li>
</ul>
</div>
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 p-2 sm:p-3 flex items-center gap-2 [&>svg]:relative [&>svg]:left-0 [&>svg]:top-0 [&>svg+div]:translate-y-0">
<Info className="h-3 w-3 sm:h-4 sm:w-4 shrink-0" />
<AlertDescription className="text-[10px] sm:text-xs !pl-0">
YouTube URLs are used when indexing. You can change this selection when you start indexing.
</AlertDescription>
</Alert>
</div>
);
};

View file

@ -3,6 +3,8 @@
import type { FC } from "react";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { GoogleDriveConfig } from "./components/google-drive-config";
import { WebcrawlerConfig } from "./components/webcrawler-config";
import { YouTubeConfig } from "./components/youtube-config";
export interface ConnectorConfigProps {
connector: SearchSourceConnector;
@ -20,6 +22,10 @@ export function getConnectorConfigComponent(
switch (connectorType) {
case "GOOGLE_DRIVE_CONNECTOR":
return GoogleDriveConfig;
case "WEBCRAWLER_CONNECTOR":
return WebcrawlerConfig;
case "YOUTUBE_CONNECTOR":
return YouTubeConfig;
// OAuth connectors (Gmail, Calendar, Airtable) and others don't need special config UI
default:
return null;

View file

@ -1,6 +1,6 @@
"use client";
import { ArrowLeft, Loader2, Trash2 } from "lucide-react";
import { ArrowLeft, Info, Loader2, Trash2 } from "lucide-react";
import { type FC, useState, useCallback, useRef, useEffect, useMemo } from "react";
import { Button } from "@/components/ui/button";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
@ -111,7 +111,7 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
<button
type="button"
onClick={onBack}
className="flex items-center gap-2 text-sm text-muted-foreground hover:text-foreground mb-6 w-fit"
className="flex items-center gap-2 text-xs sm:text-sm text-muted-foreground hover:text-foreground mb-6 w-fit"
>
<ArrowLeft className="size-4" />
Back to connectors
@ -123,10 +123,10 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
{getConnectorIcon(connector.connector_type, "size-7")}
</div>
<div className="flex-1">
<h2 className="text-2xl font-semibold tracking-tight">
<h2 className="text-xl sm:text-2xl font-semibold tracking-tight">
{connector.name}
</h2>
<p className="text-muted-foreground mt-1">
<p className="text-xs sm:text-base text-muted-foreground mt-1">
Manage your connector settings and sync configuration
</p>
</div>
@ -149,8 +149,8 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
/>
)}
{/* Date range selector - not shown for Google Drive (uses folder selection instead) */}
{connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" && (
{/* Date range selector - not shown for Google Drive (uses folder selection), Webcrawler (uses config), or YouTube (uses URL selection) */}
{connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" && connector.connector_type !== "WEBCRAWLER_CONNECTOR" && connector.connector_type !== "YOUTUBE_CONNECTOR" && (
<DateRangeSelector
startDate={startDate}
endDate={endDate}
@ -169,11 +169,11 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
{/* Info box */}
<div className="rounded-xl border border-border bg-primary/5 p-4 flex items-start gap-3">
<div className="flex h-8 w-8 items-center justify-center rounded-lg bg-primary/10 shrink-0 mt-0.5">
{getConnectorIcon(connector.connector_type, "size-4")}
<Info className="size-4" />
</div>
<div className="text-sm">
<p className="font-medium">Re-indexing runs in the background</p>
<p className="text-muted-foreground mt-1">
<div className="text-xs sm:text-sm">
<p className="font-medium text-xs sm:text-sm">Re-indexing runs in the background</p>
<p className="text-muted-foreground mt-1 text-[10px] sm:text-sm">
You can continue using SurfSense while we sync your data. Check the Active tab to see progress.
</p>
</div>
@ -194,12 +194,13 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
<div className="flex-shrink-0 flex items-center justify-between px-6 sm:px-12 py-6 bg-muted border-t border-border">
{showDisconnectConfirm ? (
<div className="flex items-center gap-3">
<span className="text-sm text-muted-foreground">Are you sure?</span>
<span className="text-xs sm:text-sm text-muted-foreground">Are you sure?</span>
<Button
variant="destructive"
size="sm"
onClick={handleDisconnectConfirm}
disabled={isDisconnecting}
className="text-xs sm:text-sm"
>
{isDisconnecting ? (
<>
@ -215,6 +216,7 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
size="sm"
onClick={handleDisconnectCancel}
disabled={isDisconnecting}
className="text-xs sm:text-sm"
>
Cancel
</Button>
@ -225,12 +227,13 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
size="sm"
onClick={handleDisconnectClick}
disabled={isSaving || isDisconnecting}
className="text-xs sm:text-sm"
>
<Trash2 className="mr-2 h-4 w-4" />
Disconnect
</Button>
)}
<Button onClick={onSave} disabled={isSaving || isDisconnecting}>
<Button onClick={onSave} disabled={isSaving || isDisconnecting} className="text-xs sm:text-sm">
{isSaving ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />

View file

@ -1,6 +1,6 @@
"use client";
import { ArrowLeft, Check, Loader2 } from "lucide-react";
import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
import { type FC, useState, useCallback, useRef, useEffect, useMemo } from "react";
import { Button } from "@/components/ui/button";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
@ -96,7 +96,7 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
<button
type="button"
onClick={onSkip}
className="flex items-center gap-2 text-sm text-muted-foreground hover:text-foreground mb-6 w-fit"
className="flex items-center gap-2 text-xs sm:text-sm text-muted-foreground hover:text-foreground mb-6 w-fit"
>
<ArrowLeft className="size-4" />
Back to connectors
@ -108,10 +108,10 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
<Check className="size-7 text-green-500" />
</div>
<div>
<h2 className="text-2xl font-semibold tracking-tight">
<h2 className="text-xl sm:text-2xl font-semibold tracking-tight">
{config.connectorTitle} Connected!
</h2>
<p className="text-muted-foreground mt-1">
<p className="text-xs sm:text-base text-muted-foreground mt-1">
Configure when to start syncing your data
</p>
</div>
@ -134,8 +134,8 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
/>
)}
{/* Date range selector - not shown for Google Drive (uses folder selection instead) */}
{config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && (
{/* Date range selector - not shown for Google Drive (uses folder selection), Webcrawler (uses config), or YouTube (uses URL selection) */}
{config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && config.connectorType !== "WEBCRAWLER_CONNECTOR" && config.connectorType !== "YOUTUBE_CONNECTOR" && (
<DateRangeSelector
startDate={startDate}
endDate={endDate}
@ -154,11 +154,11 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
{/* Info box */}
<div className="rounded-xl border border-border bg-primary/5 p-4 flex items-start gap-3">
<div className="flex h-8 w-8 items-center justify-center rounded-lg bg-primary/10 shrink-0 mt-0.5">
{getConnectorIcon(config.connectorType, "size-4")}
<Info className="size-4" />
</div>
<div className="text-sm">
<p className="font-medium">Indexing runs in the background</p>
<p className="text-muted-foreground mt-1">
<div className="text-xs sm:text-sm">
<p className="font-medium text-xs sm:text-sm">Indexing runs in the background</p>
<p className="text-muted-foreground mt-1 text-[10px] sm:text-sm">
You can continue using SurfSense while we sync your data. Check the Active tab to see progress.
</p>
</div>
@ -177,10 +177,10 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
{/* Fixed Footer - Action buttons */}
<div className="flex-shrink-0 flex items-center justify-between px-6 sm:px-12 py-6 bg-muted">
<Button variant="ghost" onClick={onSkip} disabled={isStartingIndexing}>
<Button variant="ghost" onClick={onSkip} disabled={isStartingIndexing} className="text-xs sm:text-sm">
Skip for now
</Button>
<Button onClick={onStartIndexing} disabled={isStartingIndexing}>
<Button onClick={onStartIndexing} disabled={isStartingIndexing} className="text-xs sm:text-sm">
{isStartingIndexing ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />

View file

@ -106,6 +106,12 @@ export const OTHER_CONNECTORS = [
description: "Crawl web content",
connectorType: EnumConnectorName.WEBCRAWLER_CONNECTOR,
},
{
id: "youtube-connector",
title: "YouTube",
description: "Index YouTube videos",
connectorType: EnumConnectorName.YOUTUBE_CONNECTOR,
},
{
id: "tavily-api",
title: "Tavily AI",

View file

@ -2,16 +2,17 @@ import { useAtomValue } from "jotai";
import { useRouter, useSearchParams } from "next/navigation";
import { useCallback, useEffect, useState } from "react";
import { toast } from "sonner";
import { deleteConnectorMutationAtom, indexConnectorMutationAtom, updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
import { createConnectorMutationAtom, deleteConnectorMutationAtom, indexConnectorMutationAtom, updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
import { authenticatedFetch } from "@/lib/auth-utils";
import { queryClient } from "@/lib/query-client/client";
import { cacheKeys } from "@/lib/query-client/cache-keys";
import { format } from "date-fns";
import { EnumConnectorName } from "@/contracts/enums/connector";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { searchSourceConnector } from "@/contracts/types/connector.types";
import { OAUTH_CONNECTORS } from "../constants/connector-constants";
import { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants";
import type { IndexingConfigState } from "../constants/connector-constants";
import {
parseConnectorPopupQueryParams,
@ -29,6 +30,7 @@ export const useConnectorDialog = () => {
const { mutateAsync: indexConnector } = useAtomValue(indexConnectorMutationAtom);
const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom);
const { mutateAsync: deleteConnector } = useAtomValue(deleteConnectorMutationAtom);
const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom);
const [isOpen, setIsOpen] = useState(false);
const [activeTab, setActiveTab] = useState("all");
@ -246,19 +248,131 @@ export const useConnectorDialog = () => {
[searchSpaceId]
);
// Handle creating webcrawler connector
const handleCreateWebcrawler = useCallback(async () => {
if (!searchSpaceId) return;
setConnectingId("webcrawler-connector");
try {
const newConnector = await createConnector({
data: {
name: "Web Pages",
connector_type: EnumConnectorName.WEBCRAWLER_CONNECTOR,
config: {},
is_indexable: true,
last_indexed_at: null,
periodic_indexing_enabled: false,
indexing_frequency_minutes: null,
next_scheduled_at: null,
},
queryParams: {
search_space_id: searchSpaceId,
},
});
// Refetch connectors to get the new one
const result = await refetchAllConnectors();
if (result.data) {
const connector = result.data.find(
(c: SearchSourceConnector) => c.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR
);
if (connector) {
const connectorValidation = searchSourceConnector.safeParse(connector);
if (connectorValidation.success) {
const config = validateIndexingConfigState({
connectorType: EnumConnectorName.WEBCRAWLER_CONNECTOR,
connectorId: connector.id,
connectorTitle: "Web Pages",
});
setIndexingConfig(config);
setIndexingConnector(connector);
setIndexingConnectorConfig(connector.config || {});
setIsOpen(true);
const url = new URL(window.location.href);
url.searchParams.set("modal", "connectors");
url.searchParams.set("view", "configure");
window.history.pushState({ modal: true }, "", url.toString());
}
}
}
} catch (error) {
console.error("Error creating webcrawler connector:", error);
toast.error("Failed to create web crawler connector");
} finally {
setConnectingId(null);
}
}, [searchSpaceId, createConnector, refetchAllConnectors]);
// Handle creating YouTube connector
const handleCreateYouTube = useCallback(async () => {
if (!searchSpaceId) return;
setConnectingId("youtube-connector");
try {
const newConnector = await createConnector({
data: {
name: "YouTube",
connector_type: EnumConnectorName.YOUTUBE_CONNECTOR,
config: { youtube_urls: [] },
is_indexable: true,
last_indexed_at: null,
periodic_indexing_enabled: false,
indexing_frequency_minutes: null,
next_scheduled_at: null,
},
queryParams: {
search_space_id: searchSpaceId,
},
});
// Refetch connectors to get the new one
const result = await refetchAllConnectors();
if (result.data) {
const connector = result.data.find(
(c: SearchSourceConnector) => c.connector_type === EnumConnectorName.YOUTUBE_CONNECTOR
);
if (connector) {
const connectorValidation = searchSourceConnector.safeParse(connector);
if (connectorValidation.success) {
const config = validateIndexingConfigState({
connectorType: EnumConnectorName.YOUTUBE_CONNECTOR,
connectorId: connector.id,
connectorTitle: "YouTube",
});
setIndexingConfig(config);
setIndexingConnector(connector);
setIndexingConnectorConfig(connector.config || { youtube_urls: [] });
setIsOpen(true);
const url = new URL(window.location.href);
url.searchParams.set("modal", "connectors");
url.searchParams.set("view", "configure");
window.history.pushState({ modal: true }, "", url.toString());
}
}
}
} catch (error) {
console.error("Error creating YouTube connector:", error);
toast.error("Failed to create YouTube connector");
} finally {
setConnectingId(null);
}
}, [searchSpaceId, createConnector, refetchAllConnectors]);
// Handle starting indexing
const handleStartIndexing = useCallback(async (refreshConnectors: () => void) => {
if (!indexingConfig || !searchSpaceId) return;
// Validate date range
const dateRangeValidation = dateRangeSchema.safeParse({ startDate, endDate });
if (!dateRangeValidation.success) {
const firstIssueMsg =
dateRangeValidation.error.issues && dateRangeValidation.error.issues.length > 0
? dateRangeValidation.error.issues[0].message
: "Invalid date range";
toast.error(firstIssueMsg);
return;
// Validate date range (skip for Google Drive and Webcrawler)
if (indexingConfig.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && indexingConfig.connectorType !== "WEBCRAWLER_CONNECTOR") {
const dateRangeValidation = dateRangeSchema.safeParse({ startDate, endDate });
if (!dateRangeValidation.success) {
const firstIssueMsg =
dateRangeValidation.error.issues && dateRangeValidation.error.issues.length > 0
? dateRangeValidation.error.issues[0].message
: "Invalid date range";
toast.error(firstIssueMsg);
return;
}
}
// Validate frequency minutes if periodic is enabled
@ -313,6 +427,14 @@ export const useConnectorDialog = () => {
setIsStartingIndexing(false);
return;
}
} else if (indexingConfig.connectorType === "WEBCRAWLER_CONNECTOR") {
// Webcrawler doesn't use date ranges, just uses config (API key and URLs)
await indexConnector({
connector_id: indexingConfig.connectorId,
queryParams: {
search_space_id: searchSpaceId,
},
});
} else {
await indexConnector({
connector_id: indexingConfig.connectorId,
@ -372,8 +494,11 @@ export const useConnectorDialog = () => {
(oauthConnector) => oauthConnector.connectorType === connector.connector_type
);
// If not OAuth, redirect to old connector edit page
if (!isOAuthConnector) {
// Check if this is webcrawler (can be managed in popup)
const isWebcrawler = connector.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR;
// If not OAuth and not webcrawler, redirect to old connector edit page
if (!isOAuthConnector && !isWebcrawler) {
router.push(`/dashboard/${searchSpaceId}/connectors/${connector.id}/edit`);
return;
}
@ -405,8 +530,8 @@ export const useConnectorDialog = () => {
const handleSaveConnector = useCallback(async (refreshConnectors: () => void) => {
if (!editingConnector || !searchSpaceId) return;
// Validate date range (skip for Google Drive which uses folder selection)
if (editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR") {
// Validate date range (skip for Google Drive which uses folder selection, and Webcrawler which uses config)
if (editingConnector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" && editingConnector.connector_type !== "WEBCRAWLER_CONNECTOR") {
const dateRangeValidation = dateRangeSchema.safeParse({ startDate, endDate });
if (!dateRangeValidation.success) {
toast.error(dateRangeValidation.error.issues[0]?.message || "Invalid date range");
@ -457,6 +582,15 @@ export const useConnectorDialog = () => {
});
indexingDescription = `Re-indexing started for ${selectedFolders.length} folder(s).`;
}
} else if (editingConnector.connector_type === "WEBCRAWLER_CONNECTOR") {
// Webcrawler uses config (API key and URLs), not date ranges
await indexConnector({
connector_id: editingConnector.id,
queryParams: {
search_space_id: searchSpaceId,
},
});
indexingDescription = "Re-indexing started with updated configuration.";
} else if (startDateStr || endDateStr) {
// Other connectors use date ranges
await indexConnector({
@ -623,6 +757,8 @@ export const useConnectorDialog = () => {
handleTabChange,
handleScroll,
handleConnectOAuth,
handleCreateWebcrawler,
handleCreateYouTube,
handleStartIndexing,
handleSkipIndexing,
handleStartEdit,

View file

@ -13,6 +13,8 @@ interface AllConnectorsTabProps {
connectingId: string | null;
allConnectors: SearchSourceConnector[] | undefined;
onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[0]) => void;
onCreateWebcrawler?: () => void;
onCreateYouTube?: () => void;
onManage?: (connector: SearchSourceConnector) => void;
}
@ -23,6 +25,8 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
connectingId,
allConnectors,
onConnectOAuth,
onCreateWebcrawler,
onCreateYouTube,
onManage,
}) => {
const router = useRouter();
@ -88,11 +92,21 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">
{filteredOther.map((connector) => {
const isConnected = connectedTypes.has(connector.connectorType);
const isConnecting = connectingId === connector.id;
// Find the actual connector object if connected
const actualConnector = isConnected && allConnectors
? allConnectors.find((c: SearchSourceConnector) => c.connector_type === connector.connectorType)
: undefined;
// Special handling for webcrawler and YouTube - create in popup
const isWebcrawler = connector.id === "webcrawler-connector";
const isYouTube = connector.id === "youtube-connector";
const handleConnect = isWebcrawler && onCreateWebcrawler
? onCreateWebcrawler
: isYouTube && onCreateYouTube
? onCreateYouTube
: () => router.push(`/dashboard/${searchSpaceId}/connectors/add/${connector.id}`);
return (
<ConnectorCard
key={connector.id}
@ -101,11 +115,8 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
description={connector.description}
connectorType={connector.connectorType}
isConnected={isConnected}
onConnect={() =>
router.push(
`/dashboard/${searchSpaceId}/connectors/add/${connector.id}`
)
}
isConnecting={isConnecting}
onConnect={handleConnect}
onManage={actualConnector && onManage ? () => onManage(actualConnector) : undefined}
/>
);