apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-01 20:03:30 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: Remove YouTube connector and associated components, streamline source addition process with new YouTube crawler view, and enhance connector management UI for improved user experience.

Browse source
This commit is contained in:
Anish Sarkar 2025-12-31 18:17:03 +05:30
parent de63e77f78
commit 75119bf06b
13 changed files with 405 additions and 331 deletions
Download patch file Download diff file Expand all files Collapse all files

18
surfsense_web/components/assistant-ui/connector-popup.tsx
View file

@ -3,6 +3,7 @@
import { useAtomValue } from "jotai";
import { Cable, Loader2 } from "lucide-react";
import { type FC, useMemo, useEffect } from "react";
import { useSearchParams } from "next/navigation";
import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
import { useLogsSummary } from "@/hooks/use-logs";
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
@ -25,13 +26,18 @@ import { ConnectorDialogHeader } from "./connector-popup/components/connector-di
import { ConnectorEditView } from "./connector-popup/connector-configs/views/connector-edit-view";
import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view";
import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view";
import { YouTubeCrawlerView } from "./connector-popup/views/youtube-crawler-view";
import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
export const ConnectorIndicator: FC = () => {
const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom);
const searchParams = useSearchParams();
const { data: documentTypeCounts, isLoading: documentTypesLoading } =
useAtomValue(documentTypeCountsAtom);
// Check if YouTube view is active
const isYouTubeView = searchParams.get("view") === "youtube";
// Track active indexing tasks
const { summary: logsSummary } = useLogsSummary(
@ -75,6 +81,7 @@ export const ConnectorIndicator: FC = () => {
handleConnectOAuth,
handleConnectNonOAuth,
handleCreateWebcrawler,
handleCreateYouTubeCrawler,
handleSubmitConnectForm,
handleStartIndexing,
handleSkipIndexing,
@ -83,6 +90,7 @@ export const ConnectorIndicator: FC = () => {
handleDisconnectConnector,
handleBackFromEdit,
handleBackFromConnect,
handleBackFromYouTube,
connectorConfig,
setConnectorConfig,
setIndexingConnectorConfig,
@ -190,8 +198,13 @@ export const ConnectorIndicator: FC = () => {
</TooltipIconButton>
<DialogContent className="max-w-3xl w-[95vw] sm:w-full h-[90vh] sm:h-[85vh] flex flex-col p-0 gap-0 overflow-hidden border border-border bg-muted text-foreground [&>button]:right-6 sm:[&>button]:right-12 [&>button]:top-8 sm:[&>button]:top-10 [&>button]:opacity-80 hover:[&>button]:opacity-100 [&>button_svg]:size-5">
{/* Connector Connect View - shown when connecting non-OAuth connectors */}
{connectingConnectorType ? (
{/* YouTube Crawler View - shown when adding YouTube videos */}
{isYouTubeView && searchSpaceId ? (
<YouTubeCrawlerView
searchSpaceId={searchSpaceId}
onBack={handleBackFromYouTube}
/>
) : connectingConnectorType ? (
<ConnectorConnectView
connectorType={connectingConnectorType}
onSubmit={handleSubmitConnectForm}
@ -270,6 +283,7 @@ export const ConnectorIndicator: FC = () => {
onConnectOAuth={handleConnectOAuth}
onConnectNonOAuth={handleConnectNonOAuth}
onCreateWebcrawler={handleCreateWebcrawler}
onCreateYouTubeCrawler={handleCreateYouTubeCrawler}
onManage={handleStartEdit}
/>
</TabsContent>

21
surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx
View file

@ -1,16 +1,18 @@
"use client";
import { IconBrandYoutube } from "@tabler/icons-react";
import { FileText, Loader2 } from "lucide-react";
import { type FC } from "react";
import { Button } from "@/components/ui/button";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { LogActiveTask } from "@/contracts/types/log.types";
import { cn } from "@/lib/utils";
interface ConnectorCardProps {
id: string;
title: string;
description: string;
connectorType: string;
connectorType?: string;
isConnected?: boolean;
isConnecting?: boolean;
documentCount?: number;
@ -88,7 +90,13 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
return (
<div className="group relative flex items-center gap-4 p-4 rounded-xl text-left transition-all duration-200 w-full border border-border bg-slate-400/5 dark:bg-white/5 hover:bg-slate-400/10 dark:hover:bg-white/10">
<div className="flex h-12 w-12 items-center justify-center rounded-lg transition-colors flex-shrink-0 bg-slate-400/5 dark:bg-white/5 border border-slate-400/5 dark:border-white/5">
{getConnectorIcon(connectorType, "size-6")}
{connectorType ? (
getConnectorIcon(connectorType, "size-6")
) : id === "youtube-crawler" ? (
<IconBrandYoutube className="size-6" />
) : (
<FileText className="size-6" />
)}
</div>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2">
@ -101,7 +109,10 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
<Button
size="sm"
variant={isConnected ? "outline" : "default"}
className="h-8 text-[11px] px-3 rounded-lg flex-shrink-0 font-medium"
className={cn(
"h-8 text-[11px] px-3 rounded-lg flex-shrink-0 font-medium",
isConnected && "border-0"
)}
onClick={isConnected ? onManage : onConnect}
disabled={isConnecting || isIndexing}
>
@ -111,8 +122,10 @@ export const ConnectorCard: FC<ConnectorCardProps> = ({
"Syncing..."
) : isConnected ? (
"Manage"
) : (
) : connectorType ? (
"Connect"
) : (
"Add"
)}
</Button>
</div>

22
surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts
View file

@ -32,6 +32,22 @@ export const OAUTH_CONNECTORS = [
},
] as const;
// Content Sources (tools that extract and import content from external sources)
export const CRAWLERS = [
{
id: "youtube-crawler",
title: "YouTube",
description: "Crawl YouTube channels and playlists",
connectorType: null, // Not a connector, handled separately
},
{
id: "webcrawler-connector",
title: "Web Pages",
description: "Crawl web content",
connectorType: EnumConnectorName.WEBCRAWLER_CONNECTOR,
},
] as const;
// Non-OAuth Connectors (redirect to old connector config pages)
export const OTHER_CONNECTORS = [
{
@ -100,12 +116,6 @@ export const OTHER_CONNECTORS = [
description: "Search ES indexes",
connectorType: EnumConnectorName.ELASTICSEARCH_CONNECTOR,
},
{
id: "webcrawler-connector",
title: "Web Pages",
description: "Crawl web content",
connectorType: EnumConnectorName.WEBCRAWLER_CONNECTOR,
},
{
id: "tavily-api",
title: "Tavily AI",

2
surfsense_web/components/assistant-ui/connector-popup/constants/connector-popup.schemas.ts
View file

@ -7,7 +7,7 @@ import { searchSourceConnectorTypeEnum } from "@/contracts/types/connector.types
export const connectorPopupQueryParamsSchema = z.object({
modal: z.enum(["connectors"]).optional(),
tab: z.enum(["all", "active"]).optional(),
view: z.enum(["configure", "edit", "connect"]).optional(),
view: z.enum(["configure", "edit", "connect", "youtube"]).optional(),
connector: z.string().optional(),
connectorId: z.string().optional(),
connectorType: z.string().optional(),

28
surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts
View file

@ -104,6 +104,11 @@ export const useConnectorDialog = () => {
setConnectingConnectorType(params.connectorType);
}
// Handle YouTube view
if (params.view === "youtube") {
// YouTube view is active - no additional state needed
}
if (params.view === "configure" && params.connector && !indexingConfig) {
const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === params.connector);
if (oauthConnector && allConnectors) {
@ -177,6 +182,7 @@ export const useConnectorDialog = () => {
if (connectingConnectorType) {
setConnectingConnectorType(null);
}
// Clear YouTube view when modal is closed (handled by view param check)
}
} catch (error) {
// Invalid query params - log but don't crash
@ -270,6 +276,17 @@ export const useConnectorDialog = () => {
[searchSpaceId]
);
// Handle creating YouTube crawler (not a connector, shows view in popup)
const handleCreateYouTubeCrawler = useCallback(() => {
if (!searchSpaceId) return;
// Update URL to show YouTube view
const url = new URL(window.location.href);
url.searchParams.set("modal", "connectors");
url.searchParams.set("view", "youtube");
window.history.pushState({ modal: true }, "", url.toString());
}, [searchSpaceId]);
// Handle creating webcrawler connector
const handleCreateWebcrawler = useCallback(async () => {
if (!searchSpaceId) return;
@ -525,6 +542,15 @@ export const useConnectorDialog = () => {
router.replace(url.pathname + url.search, { scroll: false });
}, [router]);
// Handle going back from YouTube view
const handleBackFromYouTube = useCallback(() => {
const url = new URL(window.location.href);
url.searchParams.set("modal", "connectors");
url.searchParams.set("tab", "all");
url.searchParams.delete("view");
router.replace(url.pathname + url.search, { scroll: false });
}, [router]);
// Handle starting indexing
const handleStartIndexing = useCallback(async (refreshConnectors: () => void) => {
if (!indexingConfig || !searchSpaceId) return;
@ -951,6 +977,7 @@ export const useConnectorDialog = () => {
handleConnectOAuth,
handleConnectNonOAuth,
handleCreateWebcrawler,
handleCreateYouTubeCrawler,
handleSubmitConnectForm,
handleStartIndexing,
handleSkipIndexing,
@ -959,6 +986,7 @@ export const useConnectorDialog = () => {
handleDisconnectConnector,
handleBackFromEdit,
handleBackFromConnect,
handleBackFromYouTube,
connectorConfig,
setConnectorConfig,
setIndexingConnectorConfig,

2
surfsense_web/components/assistant-ui/connector-popup/index.ts
View file

@ -12,7 +12,7 @@ export { AllConnectorsTab } from "./tabs/all-connectors-tab";
export { ActiveConnectorsTab } from "./tabs/active-connectors-tab";
// Constants and types
export { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "./constants/connector-constants";
export { OAUTH_CONNECTORS, CRAWLERS, OTHER_CONNECTORS } from "./constants/connector-constants";
export type { IndexingConfigState } from "./constants/connector-constants";
// Schemas and validation

2
surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx
View file

@ -117,7 +117,7 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
<Button
variant="outline"
size="sm"
className="h-8 text-[11px] px-3 rounded-lg font-medium"
className="h-8 text-[11px] px-3 rounded-lg font-medium border-0"
onClick={onManage ? () => onManage(connector) : undefined}
disabled={isIndexing}
>

71
surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx
View file

@ -4,7 +4,7 @@ import { useRouter } from "next/navigation";
import { type FC } from "react";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import type { LogActiveTask, LogSummary } from "@/contracts/types/log.types";
import { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "../constants/connector-constants";
import { OAUTH_CONNECTORS, CRAWLERS, OTHER_CONNECTORS } from "../constants/connector-constants";
import { ConnectorCard } from "../components/connector-card";
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
@ -20,6 +20,7 @@ interface AllConnectorsTabProps {
onConnectOAuth: (connector: (typeof OAUTH_CONNECTORS)[number]) => void;
onConnectNonOAuth?: (connectorType: string) => void;
onCreateWebcrawler?: () => void;
onCreateYouTubeCrawler?: () => void;
onManage?: (connector: SearchSourceConnector) => void;
}
@ -35,6 +36,7 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
onConnectOAuth,
onConnectNonOAuth,
onCreateWebcrawler,
onCreateYouTubeCrawler,
onManage,
}) => {
const router = useRouter();
@ -54,6 +56,12 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
c.description.toLowerCase().includes(searchQuery.toLowerCase())
);
const filteredCrawlers = CRAWLERS.filter(
(c) =>
c.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
c.description.toLowerCase().includes(searchQuery.toLowerCase())
);
const filteredOther = OTHER_CONNECTORS.filter(
(c) =>
c.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
@ -104,6 +112,67 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
</section>
)}
{/* Content Sources */}
{filteredCrawlers.length > 0 && (
<section>
<div className="flex items-center gap-2 mb-4">
<h3 className="text-sm font-semibold text-muted-foreground">
Content Sources
</h3>
</div>
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">
{filteredCrawlers.map((crawler) => {
const isYouTube = crawler.id === "youtube-crawler";
const isWebcrawler = crawler.id === "webcrawler-connector";
// For crawlers that are actual connectors, check connection status
const isConnected = crawler.connectorType
? connectedTypes.has(crawler.connectorType)
: false;
const isConnecting = connectingId === crawler.id;
// Find the actual connector object if connected
const actualConnector = isConnected && crawler.connectorType && allConnectors
? allConnectors.find((c: SearchSourceConnector) => c.connector_type === crawler.connectorType)
: undefined;
const documentCount = crawler.connectorType
? getDocumentCountForConnector(crawler.connectorType, documentTypeCounts)
: undefined;
const isIndexing = actualConnector && indexingConnectorIds?.has(actualConnector.id);
const activeTask = actualConnector ? getActiveTaskForConnector(actualConnector.id) : undefined;
const handleConnect = isYouTube && onCreateYouTubeCrawler
? onCreateYouTubeCrawler
: isWebcrawler && onCreateWebcrawler
? onCreateWebcrawler
: crawler.connectorType && onConnectNonOAuth
? () => onConnectNonOAuth(crawler.connectorType!)
: crawler.connectorType
? () => router.push(`/dashboard/${searchSpaceId}/connectors/add/${crawler.id}`)
: () => {}; // Fallback for non-connector crawlers
return (
<ConnectorCard
key={crawler.id}
id={crawler.id}
title={crawler.title}
description={crawler.description}
connectorType={crawler.connectorType || undefined}
isConnected={isConnected}
isConnecting={isConnecting}
documentCount={documentCount}
isIndexing={isIndexing}
activeTask={activeTask}
onConnect={handleConnect}
onManage={actualConnector && onManage ? () => onManage(actualConnector) : undefined}
/>
);
})}
</div>
</section>
)}
{/* More Integrations */}
{filteredOther.length > 0 && (
<section>

249
surfsense_web/components/assistant-ui/connector-popup/views/youtube-crawler-view.tsx Normal file
View file

@ -0,0 +1,249 @@
"use client";
import { IconBrandYoutube } from "@tabler/icons-react";
import { ArrowLeft } from "lucide-react";
import { TagInput, type Tag as TagType } from "emblor";
import { useAtom } from "jotai";
import { Loader2 } from "lucide-react";
import { type FC, useState } from "react";
import { useRouter } from "next/navigation";
import { useTranslations } from "next-intl";
import { toast } from "sonner";
import { createDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
import { Button } from "@/components/ui/button";
import { Label } from "@/components/ui/label";
const youtubeRegex =
/^(https:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})$/;
interface YouTubeCrawlerViewProps {
searchSpaceId: string;
onBack: () => void;
}
export const YouTubeCrawlerView: FC<YouTubeCrawlerViewProps> = ({
searchSpaceId,
onBack,
}) => {
const t = useTranslations("add_youtube");
const router = useRouter();
const [videoTags, setVideoTags] = useState<TagType[]>([]);
const [activeTagIndex, setActiveTagIndex] = useState<number | null>(null);
const [error, setError] = useState<string | null>(null);
// Use the createDocumentMutationAtom
const [createDocumentMutation] = useAtom(createDocumentMutationAtom);
const { mutate: createYouTubeDocument, isPending: isSubmitting } = createDocumentMutation;
const isValidYoutubeUrl = (url: string): boolean => {
return youtubeRegex.test(url);
};
const extractVideoId = (url: string): string | null => {
const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})/);
return match ? match[1] : null;
};
const handleSubmit = async () => {
if (videoTags.length === 0) {
setError(t("error_no_video"));
return;
}
const invalidUrls = videoTags.filter((tag) => !isValidYoutubeUrl(tag.text));
if (invalidUrls.length > 0) {
setError(t("error_invalid_urls", { urls: invalidUrls.map((tag) => tag.text).join(", ") }));
return;
}
setError(null);
toast(t("processing_toast"), {
description: t("processing_toast_desc"),
});
const videoUrls = videoTags.map((tag) => tag.text);
// Use the mutation to create YouTube documents
createYouTubeDocument(
{
document_type: "YOUTUBE_VIDEO",
content: videoUrls,
search_space_id: parseInt(searchSpaceId, 10),
},
{
onSuccess: () => {
toast(t("success_toast"), {
description: t("success_toast_desc"),
});
// Close the popup and navigate to documents
onBack();
router.push(`/dashboard/${searchSpaceId}/documents`);
},
onError: (error: unknown) => {
const errorMessage = error instanceof Error ? error.message : t("error_generic");
setError(errorMessage);
toast(t("error_toast"), {
description: `${t("error_toast_desc")}: ${errorMessage}`,
});
},
}
);
};
const handleAddTag = (text: string) => {
if (!isValidYoutubeUrl(text)) {
toast(t("invalid_url_toast"), {
description: t("invalid_url_toast_desc"),
});
return;
}
if (videoTags.some((tag) => tag.text === text)) {
toast(t("duplicate_url_toast"), {
description: t("duplicate_url_toast_desc"),
});
return;
}
const newTag: TagType = {
id: Date.now().toString(),
text: text,
};
setVideoTags([...videoTags, newTag]);
};
return (
<div className="flex-1 flex flex-col min-h-0 overflow-hidden">
{/* Header */}
<div className="flex-shrink-0 px-6 sm:px-12 pt-8 sm:pt-10">
<button
type="button"
onClick={onBack}
className="flex items-center gap-2 text-xs sm:text-sm text-muted-foreground hover:text-foreground mb-6 w-fit"
>
<ArrowLeft className="size-4" />
Back to connectors
</button>
<div className="flex items-center gap-4 mb-6">
<div className="flex h-14 w-14 items-center justify-center rounded-xl border border-slate-400/30">
<IconBrandYoutube className="h-7 w-7" />
</div>
<div>
<h2 className="text-xl sm:text-2xl font-semibold tracking-tight">
{t("title")}
</h2>
<p className="text-xs sm:text-base text-muted-foreground mt-1">
{t("subtitle")}
</p>
</div>
</div>
</div>
{/* Form Content - Scrollable */}
<div className="flex-1 min-h-0 overflow-y-auto px-6 sm:px-12">
<div className="space-y-4 pb-6">
<div className="space-y-2">
<Label htmlFor="video-input" className="text-sm sm:text-base">
{t("label")}
</Label>
<TagInput
id="video-input"
tags={videoTags}
setTags={setVideoTags}
placeholder={t("placeholder")}
onAddTag={handleAddTag}
styleClasses={{
inlineTagsContainer:
"border border-slate-400/20 rounded-lg bg-muted/50 shadow-sm shadow-black/5 transition-shadow focus-within:border-slate-400/40 focus-within:outline-none focus-within:ring-[3px] focus-within:ring-ring/20 p-1 gap-1",
input: "w-full min-w-[80px] focus-visible:outline-none shadow-none px-2 h-7 text-foreground/90 placeholder:text-muted-foreground bg-transparent",
tag: {
body: "h-7 relative bg-background border border-input hover:bg-background rounded-md font-medium text-xs ps-2 pe-7 flex",
closeButton:
"absolute -inset-y-px -end-px p-0 rounded-e-lg flex size-7 transition-colors outline-0 focus-visible:outline focus-visible:outline-2 focus-visible:outline-ring/70 text-muted-foreground/80 hover:text-foreground",
},
}}
activeTagIndex={activeTagIndex}
setActiveTagIndex={setActiveTagIndex}
/>
<p className="text-xs text-muted-foreground mt-1">{t("hint")}</p>
</div>
{error && (
<div className="text-sm text-red-500 mt-2">
{error}
</div>
)}
<div className="bg-muted/50 rounded-lg p-4 text-sm">
<h4 className="font-medium mb-2">{t("tips_title")}</h4>
<ul className="list-disc pl-5 space-y-1 text-muted-foreground">
<li>{t("tip_1")}</li>
<li>{t("tip_2")}</li>
<li>{t("tip_3")}</li>
<li>{t("tip_4")}</li>
</ul>
</div>
{videoTags.length > 0 && (
<div className="mt-4 space-y-2">
<h4 className="font-medium">{t("preview")}:</h4>
<div className="grid grid-cols-1 gap-3">
{videoTags.map((tag, _index) => {
const videoId = extractVideoId(tag.text);
return videoId ? (
<div
key={tag.id}
className="relative aspect-video rounded-lg overflow-hidden border"
>
<iframe
width="100%"
height="100%"
src={`https://www.youtube.com/embed/${videoId}`}
title="YouTube video player"
allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
allowFullScreen
/>
</div>
) : null;
})}
</div>
</div>
)}
</div>
</div>
{/* Fixed Footer - Action buttons */}
<div className="flex-shrink-0 flex items-center justify-between px-6 sm:px-12 py-6 bg-muted border-t border-border">
<Button
variant="ghost"
onClick={onBack}
disabled={isSubmitting}
className="text-xs sm:text-sm"
>
{t("cancel")}
</Button>
<Button
onClick={handleSubmit}
disabled={isSubmitting || videoTags.length === 0}
className="text-xs sm:text-sm min-w-[140px] disabled:opacity-50 disabled:cursor-not-allowed disabled:pointer-events-none"
>
{isSubmitting ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
{t("processing")}
</>
) : (
<>
<IconBrandYoutube className="mr-2 h-4 w-4" />
{t("submit")}
</>
)}
</Button>
</div>
</div>
);
};