feat(youtube): integrate YouTube video processing connector

- Added support for processing YouTube videos, including transcript extraction and document creation. - Implemented a new background task for adding YouTube video documents. - Enhanced the connector service to search for YouTube videos and return relevant results. - Updated frontend components to include YouTube video options in the dashboard and connector sources. - Added necessary dependencies for YouTube transcript API.
2026-04-25 00:36:31 +02:00 · 2025-04-11 15:05:17 -07:00 · 2025-04-11 15:05:17 -07:00 · b43272a115
commit b43272a115
parent 753f40dfea
13 changed files with 608 additions and 18 deletions
--- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
@ -94,7 +94,7 @@ import rehypeSanitize from "rehype-sanitize";
 import remarkGfm from "remark-gfm";
 import { DocumentViewer } from "@/components/document-viewer";
 import { JsonMetadataViewer } from "@/components/json-metadata-viewer";
-import { IconBrandNotion, IconBrandSlack } from "@tabler/icons-react";
+import { IconBrandNotion, IconBrandSlack, IconBrandYoutube } from "@tabler/icons-react";

 // Define animation variants for reuse
 const fadeInScale = {
@ -114,7 +114,7 @@ const fadeInScale = {
 type Document = {
    id: number;
    title: string;
-    document_type: "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE";
+    document_type: "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE" | "YOUTUBE_VIDEO";
    document_metadata: any;
    content: string;
    created_at: string;
@ -141,6 +141,7 @@ const documentTypeIcons = {
    SLACK_CONNECTOR: IconBrandSlack,
    NOTION_CONNECTOR: IconBrandNotion,
    FILE: File,
+    YOUTUBE_VIDEO: IconBrandYoutube,
 } as const;

 const columns: ColumnDef<Document>[] = [
--- a/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx
@ -0,0 +1,302 @@
+"use client";
+
+import { useState } from 'react';
+import { useParams, useRouter } from 'next/navigation';
+import { Tag, TagInput } from "emblor";
+import { Button } from "@/components/ui/button";
+import { Label } from "@/components/ui/label";
+import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } from "@/components/ui/card";
+import { toast } from "sonner";
+import { Youtube, Loader2 } from "lucide-react";
+import { motion } from "framer-motion";
+
+// YouTube video ID validation regex
+const youtubeRegex = /^(https:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})$/;
+
+export default function YouTubeVideoAdder() {
+  const params = useParams();
+  const router = useRouter();
+  const search_space_id = params.search_space_id as string;
+  
+  const [videoTags, setVideoTags] = useState<Tag[]>([]);
+  const [activeTagIndex, setActiveTagIndex] = useState<number | null>(null);
+  const [isSubmitting, setIsSubmitting] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+
+  // Function to validate a YouTube URL
+  const isValidYoutubeUrl = (url: string): boolean => {
+    return youtubeRegex.test(url);
+  };
+
+  // Function to extract video ID from URL
+  const extractVideoId = (url: string): string | null => {
+    const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})/);
+    return match ? match[1] : null;
+  };
+
+  // Function to handle video URL submission
+  const handleSubmit = async () => {
+    // Validate that we have at least one video URL
+    if (videoTags.length === 0) {
+      setError("Please add at least one YouTube video URL");
+      return;
+    }
+
+    // Validate all URLs
+    const invalidUrls = videoTags.filter(tag => !isValidYoutubeUrl(tag.text));
+    if (invalidUrls.length > 0) {
+      setError(`Invalid YouTube URLs detected: ${invalidUrls.map(tag => tag.text).join(', ')}`);
+      return;
+    }
+
+    setError(null);
+    setIsSubmitting(true);
+
+    try {
+      toast("YouTube Video Processing", {
+        description: "Starting YouTube video processing...",
+      });
+
+      // Extract URLs from tags
+      const videoUrls = videoTags.map(tag => tag.text);
+
+      // Make API call to backend
+      const response = await fetch(`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/`, {
+        method: "POST",
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${localStorage.getItem("surfsense_bearer_token")}`
+        },
+        body: JSON.stringify({
+          "document_type": "YOUTUBE_VIDEO",
+          "content": videoUrls,
+          "search_space_id": parseInt(search_space_id)
+        }),
+      });
+
+      if (!response.ok) {
+        throw new Error("Failed to process YouTube videos");
+      }
+
+      await response.json();
+
+      toast("Processing Successful", {
+        description: "YouTube videos have been submitted for processing",
+      });
+
+      // Redirect to documents page
+      router.push(`/dashboard/${search_space_id}/documents`);
+    } catch (error: any) {
+      setError(error.message || "An error occurred while processing YouTube videos");
+      toast("Processing Error", {
+        description: `Error processing YouTube videos: ${error.message}`,
+      });
+    } finally {
+      setIsSubmitting(false);
+    }
+  };
+
+  // Function to add a new video URL tag
+  const handleAddTag = (text: string) => {
+    // Basic URL validation
+    if (!isValidYoutubeUrl(text)) {
+      toast("Invalid YouTube URL", {
+        description: "Please enter a valid YouTube video URL",
+      });
+      return;
+    }
+
+    // Check for duplicates
+    if (videoTags.some(tag => tag.text === text)) {
+      toast("Duplicate URL", {
+        description: "This YouTube video has already been added",
+      });
+      return;
+    }
+
+    // Add the new tag
+    const newTag: Tag = {
+      id: Date.now().toString(),
+      text: text,
+    };
+
+    setVideoTags([...videoTags, newTag]);
+  };
+
+  // Animation variants
+  const containerVariants = {
+    hidden: { opacity: 0 },
+    visible: { 
+      opacity: 1,
+      transition: { 
+        staggerChildren: 0.1
+      }
+    }
+  };
+  
+  const itemVariants = {
+    hidden: { y: 20, opacity: 0 },
+    visible: { 
+      y: 0, 
+      opacity: 1,
+      transition: {
+        type: "spring",
+        stiffness: 300,
+        damping: 24
+      }
+    }
+  };
+
+  return (
+    <div className="container mx-auto py-8">
+      <motion.div
+        initial="hidden"
+        animate="visible"
+        variants={containerVariants}
+      >
+        <Card className="max-w-2xl mx-auto">
+          <motion.div variants={itemVariants}>
+            <CardHeader>
+              <CardTitle className="flex items-center gap-2">
+                <Youtube className="h-5 w-5" />
+                Add YouTube Videos
+              </CardTitle>
+              <CardDescription>
+                Enter YouTube video URLs to add to your document collection
+              </CardDescription>
+            </CardHeader>
+          </motion.div>
+          
+          <motion.div variants={itemVariants}>
+            <CardContent>
+              <div className="space-y-4">
+                <div className="space-y-2">
+                  <Label htmlFor="video-input">Enter YouTube Video URLs</Label>
+                  <TagInput
+                    id="video-input"
+                    tags={videoTags}
+                    setTags={setVideoTags}
+                    placeholder="Enter a YouTube URL and press Enter"
+                    onAddTag={handleAddTag}
+                    styleClasses={{
+                      inlineTagsContainer:
+                        "border-input rounded-lg bg-background shadow-sm shadow-black/5 transition-shadow focus-within:border-ring focus-within:outline-none focus-within:ring-[3px] focus-within:ring-ring/20 p-1 gap-1",
+                      input: "w-full min-w-[80px] focus-visible:outline-none shadow-none px-2 h-7",
+                      tag: {
+                        body: "h-7 relative bg-background border border-input hover:bg-background rounded-md font-medium text-xs ps-2 pe-7 flex",
+                        closeButton:
+                          "absolute -inset-y-px -end-px p-0 rounded-e-lg flex size-7 transition-colors outline-0 focus-visible:outline focus-visible:outline-2 focus-visible:outline-ring/70 text-muted-foreground/80 hover:text-foreground",
+                      },
+                    }}
+                    activeTagIndex={activeTagIndex}
+                    setActiveTagIndex={setActiveTagIndex}
+                  />
+                  <p className="text-xs text-muted-foreground mt-1">
+                    Add multiple YouTube URLs by pressing Enter after each one
+                  </p>
+                </div>
+
+                {error && (
+                  <motion.div 
+                    className="text-sm text-red-500 mt-2"
+                    initial={{ opacity: 0, scale: 0.9 }}
+                    animate={{ opacity: 1, scale: 1 }}
+                    transition={{ type: "spring", stiffness: 500, damping: 30 }}
+                  >
+                    {error}
+                  </motion.div>
+                )}
+
+                <motion.div 
+                  variants={itemVariants}
+                  className="bg-muted/50 rounded-lg p-4 text-sm"
+                >
+                  <h4 className="font-medium mb-2">Tips for adding YouTube videos:</h4>
+                  <ul className="list-disc pl-5 space-y-1 text-muted-foreground">
+                    <li>Use standard YouTube URLs (youtube.com/watch?v= or youtu.be/)</li>
+                    <li>Make sure videos are publicly accessible</li>
+                    <li>Supported formats: youtube.com/watch?v=VIDEO_ID or youtu.be/VIDEO_ID</li>
+                    <li>Processing may take some time depending on video length</li>
+                  </ul>
+                </motion.div>
+
+                {videoTags.length > 0 && (
+                  <motion.div
+                    variants={itemVariants}
+                    className="mt-4 space-y-2"
+                  >
+                    <h4 className="font-medium">Preview:</h4>
+                    <div className="grid grid-cols-1 gap-3">
+                      {videoTags.map((tag, index) => {
+                        const videoId = extractVideoId(tag.text);
+                        return videoId ? (
+                          <motion.div
+                            key={tag.id}
+                            initial={{ opacity: 0, y: 10 }}
+                            animate={{ opacity: 1, y: 0 }}
+                            transition={{ delay: index * 0.1 }}
+                            className="relative aspect-video rounded-lg overflow-hidden border"
+                          >
+                            <iframe
+                              width="100%"
+                              height="100%"
+                              src={`https://www.youtube.com/embed/${videoId}`}
+                              title="YouTube video player"
+                              allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+                              allowFullScreen
+                            ></iframe>
+                          </motion.div>
+                        ) : null;
+                      })}
+                    </div>
+                  </motion.div>
+                )}
+              </div>
+            </CardContent>
+          </motion.div>
+          
+          <motion.div variants={itemVariants}>
+            <CardFooter className="flex justify-between">
+              <Button 
+                variant="outline" 
+                onClick={() => router.push(`/dashboard/${search_space_id}/documents`)}
+              >
+                Cancel
+              </Button>
+              <Button 
+                onClick={handleSubmit} 
+                disabled={isSubmitting || videoTags.length === 0}
+                className="relative overflow-hidden"
+              >
+                {isSubmitting ? (
+                  <>
+                    <Loader2 className="mr-2 h-4 w-4 animate-spin" />
+                    Processing...
+                  </>
+                ) : (
+                  <>
+                    <motion.span
+                      initial={{ x: -5, opacity: 0 }}
+                      animate={{ x: 0, opacity: 1 }}
+                      transition={{ delay: 0.2 }}
+                      className="mr-2"
+                    >
+                      <Youtube className="h-4 w-4" />
+                    </motion.span>
+                    Submit YouTube Videos
+                  </>
+                )}
+                <motion.div
+                  className="absolute inset-0 bg-primary/10"
+                  initial={{ x: "-100%" }}
+                  animate={isSubmitting ? { x: "0%" } : { x: "-100%" }}
+                  transition={{ duration: 0.5, ease: "easeInOut" }}
+                />
+              </Button>
+            </CardFooter>
+          </motion.div>
+        </Card>
+      </motion.div>
+    </div>
+  );
+}
--- a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx
@ -48,6 +48,10 @@ export default function DashboardLayout({
          title: "Add Webpages",
          url: `/dashboard/${search_space_id}/documents/webpage`,
        },
+        {
+          title: "Add Youtube Videos",
+          url: `/dashboard/${search_space_id}/documents/youtube`,
+        },
        {
          title: "Manage Documents",
          url: `/dashboard/${search_space_id}/documents`,
--- a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx
@ -239,12 +239,10 @@ const SourcesDialogContent = ({

 const ChatPage = () => {
  const [token, setToken] = React.useState<string | null>(null);
-  const [showAnswer, setShowAnswer] = useState(true);
  const [activeTab, setActiveTab] = useState("");
  const [dialogOpen, setDialogOpen] = useState(false);
  const [sourcesPage, setSourcesPage] = useState(1);
  const [expandedSources, setExpandedSources] = useState(false);
-  const [isLoadingMore, setIsLoadingMore] = useState(false);
  const [canScrollLeft, setCanScrollLeft] = useState(false);
  const [canScrollRight, setCanScrollRight] = useState(true);
  const [sourceFilter, setSourceFilter] = useState("");
@ -258,7 +256,6 @@ const ChatPage = () => {
  const terminalMessagesRef = useRef<HTMLDivElement>(null);
  const { connectorSourceItems, isLoading: isLoadingConnectors } = useSearchSourceConnectors();

-  const SOURCES_PER_PAGE = 5;
  const INITIAL_SOURCES_DISPLAY = 3;

  const { search_space_id, chat_id } = useParams();
@ -836,7 +833,7 @@ const ChatPage = () => {
                        {connectorSources.map(connector => (
                          <TabsContent key={connector.id} value={connector.type} className="mt-0">
                            <div className="space-y-3">
-                              {getMainViewSources(connector).map((source: any) => (
+                              {getMainViewSources(connector)?.map((source: any) => (
                                <Card key={source.id} className="p-3 hover:bg-gray-50 dark:hover:bg-gray-800 cursor-pointer">
                                  <div className="flex items-start gap-3">
                                    <div className="flex-shrink-0 w-6 h-6 flex items-center justify-center">
@ -874,7 +871,7 @@ const ChatPage = () => {
                                      setSourcesPage={setSourcesPage}
                                      setSourceFilter={setSourceFilter}
                                      setExpandedSources={setExpandedSources}
-                                      isLoadingMore={isLoadingMore}
+                                      isLoadingMore={false}
                                    />
                                  </DialogContent>
                                </Dialog>
@ -887,7 +884,7 @@ const ChatPage = () => {

                    {/* Answer Section */}
                    <div className="mb-6">
-                      {showAnswer && (
+                      {
                        <div className="prose dark:prose-invert max-w-none">
                          {message.annotations && (() => {
                            // Get all ANSWER annotations
@ -913,7 +910,7 @@ const ChatPage = () => {
                            return <MarkdownViewer content={message.content} getCitationSource={getCitationSource} />;
                          })()}
                        </div>
-                      )}
+                      }
                    </div>
                    {/* Scroll to bottom button */}
                    <div className="fixed bottom-8 right-8">
--- a/surfsense_web/components/chat/ConnectorComponents.tsx
+++ b/surfsense_web/components/chat/ConnectorComponents.tsx
@ -4,15 +4,14 @@ import {
  Plus,
  Search,
  Globe,
-  BookOpen,
  Sparkles,
  Microscope,
  Telescope,
  File,
  Link,
-  Slack,
-  Webhook
+  Webhook,
 } from 'lucide-react';
+import { IconBrandNotion, IconBrandSlack, IconBrandYoutube } from "@tabler/icons-react";
 import { Button } from '@/components/ui/button';
 import { Connector, ResearchMode } from './types';

@ -21,6 +20,8 @@ export const getConnectorIcon = (connectorType: string) => {
  const iconProps = { className: "h-4 w-4" };
  
  switch(connectorType) {
+    case 'YOUTUBE_VIDEO':
+      return <IconBrandYoutube {...iconProps} />;
    case 'CRAWLED_URL':
      return <Globe {...iconProps} />;
    case 'FILE':
@ -31,9 +32,9 @@ export const getConnectorIcon = (connectorType: string) => {
    case 'TAVILY_API':
      return <Link {...iconProps} />;
    case 'SLACK_CONNECTOR':
-      return <Slack {...iconProps} />;
+      return <IconBrandSlack {...iconProps} />;
    case 'NOTION_CONNECTOR':
-      return <BookOpen {...iconProps} />;
+      return <IconBrandNotion {...iconProps} />;
    case 'DEEP':
      return <Sparkles {...iconProps} />;
    case 'DEEPER':
--- a/surfsense_web/components/chat/connector-sources.ts
+++ b/surfsense_web/components/chat/connector-sources.ts
@ -15,4 +15,9 @@ export const connectorSourcesMenu = [
      name: "Extension",
      type: "EXTENSION",
    },
+    {
+      id: 4,
+      name: "Youtube Video",
+      type: "YOUTUBE_VIDEO",
+    }
  ];
--- a/surfsense_web/hooks/useSearchSourceConnectors.ts
+++ b/surfsense_web/hooks/useSearchSourceConnectors.ts
@ -43,6 +43,12 @@ export const useSearchSourceConnectors = () => {
      name: "Extension",
      type: "EXTENSION",
      sources: [],
+    },
+    {
+      id: 4,
+      name: "Youtube Video",
+      type: "YOUTUBE_VIDEO",
+      sources: [],
    }
  ]);

@ -108,6 +114,12 @@ export const useSearchSourceConnectors = () => {
        name: "Extension",
        type: "EXTENSION",
        sources: [],
+      },
+      {
+        id: 4,
+        name: "Youtube Video",
+        type: "YOUTUBE_VIDEO",
+        sources: [],
      }
    ];