diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx deleted file mode 100644 index 4e39ab39c..000000000 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx +++ /dev/null @@ -1,16 +0,0 @@ -"use client"; - -import { useParams, useRouter } from "next/navigation"; -import { useEffect } from "react"; - -export default function YouTubeRedirect() { - const params = useParams(); - const router = useRouter(); - const search_space_id = params.search_space_id as string; - - useEffect(() => { - router.replace(`/dashboard/${search_space_id}/sources/add?tab=youtube`); - }, [search_space_id, router]); - - return null; -} diff --git a/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx index 7c52354d2..55c6e2ff8 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx @@ -1,40 +1,13 @@ "use client"; -import { IconBrandYoutube } from "@tabler/icons-react"; -import { Cable, Database } from "lucide-react"; +import { Database } from "lucide-react"; import { motion } from "motion/react"; -import { useParams, useRouter, useSearchParams } from "next/navigation"; -import { useEffect, useState } from "react"; +import { useParams } from "next/navigation"; import { ConnectorsTab } from "@/components/sources/ConnectorsTab"; -import { YouTubeTab } from "@/components/sources/YouTubeTab"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { trackSourcesTabViewed } from "@/lib/posthog/events"; export default function AddSourcesPage() { const params = useParams(); - const router = useRouter(); - const searchParams = useSearchParams(); const search_space_id = params.search_space_id as string; - const [activeTab, setActiveTab] = useState("youtube"); - - // Handle tab from query parameter - useEffect(() => { - const tabParam = searchParams.get("tab"); - if (tabParam && ["youtube", "connectors"].includes(tabParam)) { - setActiveTab(tabParam); - } - }, [searchParams]); - - const handleTabChange = (value: string) => { - setActiveTab(value); - // Track tab view - trackSourcesTabViewed(Number(search_space_id), value); - }; - - // Track initial tab view - useEffect(() => { - trackSourcesTabViewed(Number(search_space_id), activeTab); - }, []); return (
@@ -55,30 +28,10 @@ export default function AddSourcesPage() {

- {/* Tabs */} - - - - - YouTube - - - - Connectors - More - - - -
- - - - - - - -
-
+ {/* Connectors */} +
+ +
); diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index ba80f0d7e..9b3d138b3 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -3,6 +3,7 @@ import { useAtomValue } from "jotai"; import { Cable, Loader2 } from "lucide-react"; import { type FC, useMemo, useEffect } from "react"; +import { useSearchParams } from "next/navigation"; import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; import { useLogsSummary } from "@/hooks/use-logs"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; @@ -25,13 +26,18 @@ import { ConnectorDialogHeader } from "./connector-popup/components/connector-di import { ConnectorEditView } from "./connector-popup/connector-configs/views/connector-edit-view"; import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view"; import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view"; +import { YouTubeCrawlerView } from "./connector-popup/views/youtube-crawler-view"; import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; export const ConnectorIndicator: FC = () => { const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); + const searchParams = useSearchParams(); const { data: documentTypeCounts, isLoading: documentTypesLoading } = useAtomValue(documentTypeCountsAtom); + + // Check if YouTube view is active + const isYouTubeView = searchParams.get("view") === "youtube"; // Track active indexing tasks const { summary: logsSummary } = useLogsSummary( @@ -75,6 +81,7 @@ export const ConnectorIndicator: FC = () => { handleConnectOAuth, handleConnectNonOAuth, handleCreateWebcrawler, + handleCreateYouTubeCrawler, handleSubmitConnectForm, handleStartIndexing, handleSkipIndexing, @@ -83,6 +90,7 @@ export const ConnectorIndicator: FC = () => { handleDisconnectConnector, handleBackFromEdit, handleBackFromConnect, + handleBackFromYouTube, connectorConfig, setConnectorConfig, setIndexingConnectorConfig, @@ -190,8 +198,13 @@ export const ConnectorIndicator: FC = () => { - {/* Connector Connect View - shown when connecting non-OAuth connectors */} - {connectingConnectorType ? ( + {/* YouTube Crawler View - shown when adding YouTube videos */} + {isYouTubeView && searchSpaceId ? ( + + ) : connectingConnectorType ? ( { onConnectOAuth={handleConnectOAuth} onConnectNonOAuth={handleConnectNonOAuth} onCreateWebcrawler={handleCreateWebcrawler} + onCreateYouTubeCrawler={handleCreateYouTubeCrawler} onManage={handleStartEdit} /> diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx index 413cc402c..126945fa6 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/connector-card.tsx @@ -1,16 +1,18 @@ "use client"; +import { IconBrandYoutube } from "@tabler/icons-react"; import { FileText, Loader2 } from "lucide-react"; import { type FC } from "react"; import { Button } from "@/components/ui/button"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { LogActiveTask } from "@/contracts/types/log.types"; +import { cn } from "@/lib/utils"; interface ConnectorCardProps { id: string; title: string; description: string; - connectorType: string; + connectorType?: string; isConnected?: boolean; isConnecting?: boolean; documentCount?: number; @@ -88,7 +90,13 @@ export const ConnectorCard: FC = ({ return (
- {getConnectorIcon(connectorType, "size-6")} + {connectorType ? ( + getConnectorIcon(connectorType, "size-6") + ) : id === "youtube-crawler" ? ( + + ) : ( + + )}
@@ -101,7 +109,10 @@ export const ConnectorCard: FC = ({
diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts index 189b5cc5c..60126e135 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts @@ -32,6 +32,22 @@ export const OAUTH_CONNECTORS = [ }, ] as const; +// Content Sources (tools that extract and import content from external sources) +export const CRAWLERS = [ + { + id: "youtube-crawler", + title: "YouTube", + description: "Crawl YouTube channels and playlists", + connectorType: null, // Not a connector, handled separately + }, + { + id: "webcrawler-connector", + title: "Web Pages", + description: "Crawl web content", + connectorType: EnumConnectorName.WEBCRAWLER_CONNECTOR, + }, +] as const; + // Non-OAuth Connectors (redirect to old connector config pages) export const OTHER_CONNECTORS = [ { @@ -100,12 +116,6 @@ export const OTHER_CONNECTORS = [ description: "Search ES indexes", connectorType: EnumConnectorName.ELASTICSEARCH_CONNECTOR, }, - { - id: "webcrawler-connector", - title: "Web Pages", - description: "Crawl web content", - connectorType: EnumConnectorName.WEBCRAWLER_CONNECTOR, - }, { id: "tavily-api", title: "Tavily AI", diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-popup.schemas.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-popup.schemas.ts index 49b398733..89d8553b6 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-popup.schemas.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-popup.schemas.ts @@ -7,7 +7,7 @@ import { searchSourceConnectorTypeEnum } from "@/contracts/types/connector.types export const connectorPopupQueryParamsSchema = z.object({ modal: z.enum(["connectors"]).optional(), tab: z.enum(["all", "active"]).optional(), - view: z.enum(["configure", "edit", "connect"]).optional(), + view: z.enum(["configure", "edit", "connect", "youtube"]).optional(), connector: z.string().optional(), connectorId: z.string().optional(), connectorType: z.string().optional(), diff --git a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts index 685d7ad65..a24ada2e3 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts @@ -104,6 +104,11 @@ export const useConnectorDialog = () => { setConnectingConnectorType(params.connectorType); } + // Handle YouTube view + if (params.view === "youtube") { + // YouTube view is active - no additional state needed + } + if (params.view === "configure" && params.connector && !indexingConfig) { const oauthConnector = OAUTH_CONNECTORS.find(c => c.id === params.connector); if (oauthConnector && allConnectors) { @@ -177,6 +182,7 @@ export const useConnectorDialog = () => { if (connectingConnectorType) { setConnectingConnectorType(null); } + // Clear YouTube view when modal is closed (handled by view param check) } } catch (error) { // Invalid query params - log but don't crash @@ -270,6 +276,17 @@ export const useConnectorDialog = () => { [searchSpaceId] ); + // Handle creating YouTube crawler (not a connector, shows view in popup) + const handleCreateYouTubeCrawler = useCallback(() => { + if (!searchSpaceId) return; + + // Update URL to show YouTube view + const url = new URL(window.location.href); + url.searchParams.set("modal", "connectors"); + url.searchParams.set("view", "youtube"); + window.history.pushState({ modal: true }, "", url.toString()); + }, [searchSpaceId]); + // Handle creating webcrawler connector const handleCreateWebcrawler = useCallback(async () => { if (!searchSpaceId) return; @@ -525,6 +542,15 @@ export const useConnectorDialog = () => { router.replace(url.pathname + url.search, { scroll: false }); }, [router]); + // Handle going back from YouTube view + const handleBackFromYouTube = useCallback(() => { + const url = new URL(window.location.href); + url.searchParams.set("modal", "connectors"); + url.searchParams.set("tab", "all"); + url.searchParams.delete("view"); + router.replace(url.pathname + url.search, { scroll: false }); + }, [router]); + // Handle starting indexing const handleStartIndexing = useCallback(async (refreshConnectors: () => void) => { if (!indexingConfig || !searchSpaceId) return; @@ -951,6 +977,7 @@ export const useConnectorDialog = () => { handleConnectOAuth, handleConnectNonOAuth, handleCreateWebcrawler, + handleCreateYouTubeCrawler, handleSubmitConnectForm, handleStartIndexing, handleSkipIndexing, @@ -959,6 +986,7 @@ export const useConnectorDialog = () => { handleDisconnectConnector, handleBackFromEdit, handleBackFromConnect, + handleBackFromYouTube, connectorConfig, setConnectorConfig, setIndexingConnectorConfig, diff --git a/surfsense_web/components/assistant-ui/connector-popup/index.ts b/surfsense_web/components/assistant-ui/connector-popup/index.ts index 1c5ebc471..7d7b737fd 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/index.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/index.ts @@ -12,7 +12,7 @@ export { AllConnectorsTab } from "./tabs/all-connectors-tab"; export { ActiveConnectorsTab } from "./tabs/active-connectors-tab"; // Constants and types -export { OAUTH_CONNECTORS, OTHER_CONNECTORS } from "./constants/connector-constants"; +export { OAUTH_CONNECTORS, CRAWLERS, OTHER_CONNECTORS } from "./constants/connector-constants"; export type { IndexingConfigState } from "./constants/connector-constants"; // Schemas and validation diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx index 330658fbb..b033b5f20 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx @@ -117,7 +117,7 @@ export const ActiveConnectorsTab: FC = ({ + +
+
+ +
+
+

+ {t("title")} +

+

+ {t("subtitle")} +

+
+
+
+ + {/* Form Content - Scrollable */} +
+
+
+ + +

{t("hint")}

+
+ + {error && ( +
+ {error} +
+ )} + +
+

{t("tips_title")}

+
    +
  • {t("tip_1")}
  • +
  • {t("tip_2")}
  • +
  • {t("tip_3")}
  • +
  • {t("tip_4")}
  • +
+
+ + {videoTags.length > 0 && ( +
+

{t("preview")}:

+
+ {videoTags.map((tag, _index) => { + const videoId = extractVideoId(tag.text); + return videoId ? ( +
+