From b43272a1155db7ffc487a2557214fba016f20eb3 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Fri, 11 Apr 2025 15:05:17 -0700 Subject: [PATCH] feat(youtube): integrate YouTube video processing connector - Added support for processing YouTube videos, including transcript extraction and document creation. - Implemented a new background task for adding YouTube video documents. - Enhanced the connector service to search for YouTube videos and return relevant results. - Updated frontend components to include YouTube video options in the dashboard and connector sources. - Added necessary dependencies for YouTube transcript API. --- .../app/routes/documents_routes.py | 5 +- .../app/tasks/background_tasks.py | 139 ++++++++ .../tasks/stream_connector_search_results.py | 26 ++ .../app/utils/connector_service.py | 81 ++++- surfsense_backend/pyproject.toml | 1 + surfsense_backend/uv.lock | 24 ++ .../documents/(manage)/page.tsx | 5 +- .../documents/youtube/page.tsx | 302 ++++++++++++++++++ .../dashboard/[search_space_id]/layout.tsx | 4 + .../researcher/[chat_id]/page.tsx | 11 +- .../components/chat/ConnectorComponents.tsx | 11 +- .../components/chat/connector-sources.ts | 5 + .../hooks/useSearchSourceConnectors.ts | 12 + 13 files changed, 608 insertions(+), 18 deletions(-) create mode 100644 surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index 76e550bc7..2298005bc 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -6,7 +6,7 @@ from app.db import get_async_session, User, SearchSpace, Document, DocumentType from app.schemas import DocumentsCreate, DocumentUpdate, DocumentRead from app.users import current_active_user from app.utils.check_ownership import check_ownership -from app.tasks.background_tasks import add_extension_received_document, add_received_file_document, add_crawled_url_document +from app.tasks.background_tasks import add_extension_received_document, add_received_file_document, add_crawled_url_document, add_youtube_video_document # Force asyncio to use standard event loop before unstructured imports import asyncio try: @@ -368,8 +368,7 @@ async def process_youtube_video_with_new_session( async with async_session_maker() as session: try: - # TODO: Implement YouTube video processing - print("Processing YouTube video with new session") + await add_youtube_video_document(session, url, search_space_id) except Exception as e: import logging logging.error(f"Error processing YouTube video: {str(e)}") diff --git a/surfsense_backend/app/tasks/background_tasks.py b/surfsense_backend/app/tasks/background_tasks.py index bb0c3db2c..b2f6f8c81 100644 --- a/surfsense_backend/app/tasks/background_tasks.py +++ b/surfsense_backend/app/tasks/background_tasks.py @@ -244,3 +244,142 @@ async def add_received_file_document( except Exception as e: await session.rollback() raise RuntimeError(f"Failed to process file document: {str(e)}") + + +async def add_youtube_video_document( + session: AsyncSession, + url: str, + search_space_id: int +): + """ + Process a YouTube video URL, extract transcripts, and add as document. + """ + try: + from youtube_transcript_api import YouTubeTranscriptApi + + # Extract video ID from URL + def get_youtube_video_id(url: str): + from urllib.parse import urlparse, parse_qs + + parsed_url = urlparse(url) + hostname = parsed_url.hostname + + if hostname == "youtu.be": + return parsed_url.path[1:] + if hostname in ("www.youtube.com", "youtube.com"): + if parsed_url.path == "/watch": + query_params = parse_qs(parsed_url.query) + return query_params.get("v", [None])[0] + if parsed_url.path.startswith("/embed/"): + return parsed_url.path.split("/")[2] + if parsed_url.path.startswith("/v/"): + return parsed_url.path.split("/")[2] + return None + + # Get video ID + video_id = get_youtube_video_id(url) + if not video_id: + raise ValueError(f"Could not extract video ID from URL: {url}") + + # Get video metadata + import json + from urllib.parse import urlencode + from urllib.request import urlopen + + params = {"format": "json", "url": f"https://www.youtube.com/watch?v={video_id}"} + oembed_url = "https://www.youtube.com/oembed" + query_string = urlencode(params) + full_url = oembed_url + "?" + query_string + + with urlopen(full_url) as response: + response_text = response.read() + video_data = json.loads(response_text.decode()) + + # Get video transcript + try: + captions = YouTubeTranscriptApi.get_transcript(video_id) + # Include complete caption information with timestamps + transcript_segments = [] + for line in captions: + start_time = line.get("start", 0) + duration = line.get("duration", 0) + text = line.get("text", "") + timestamp = f"[{start_time:.2f}s-{start_time + duration:.2f}s]" + transcript_segments.append(f"{timestamp} {text}") + transcript_text = "\n".join(transcript_segments) + except Exception as e: + transcript_text = f"No captions available for this video. Error: {str(e)}" + + # Format document metadata in a more maintainable way + metadata_sections = [ + ("METADATA", [ + f"TITLE: {video_data.get('title', 'YouTube Video')}", + f"URL: {url}", + f"VIDEO_ID: {video_id}", + f"AUTHOR: {video_data.get('author_name', 'Unknown')}", + f"THUMBNAIL: {video_data.get('thumbnail_url', '')}" + ]), + ("CONTENT", [ + "FORMAT: transcript", + "TEXT_START", + transcript_text, + "TEXT_END" + ]) + ] + + # Build the document string more efficiently + document_parts = [] + document_parts.append("") + + for section_title, section_content in metadata_sections: + document_parts.append(f"<{section_title}>") + document_parts.extend(section_content) + document_parts.append(f"") + + document_parts.append("") + combined_document_string = '\n'.join(document_parts) + + # Generate summary + summary_chain = SUMMARY_PROMPT_TEMPLATE | config.long_context_llm_instance + summary_result = await summary_chain.ainvoke({"document": combined_document_string}) + summary_content = summary_result.content + summary_embedding = config.embedding_model_instance.embed(summary_content) + + # Process chunks + chunks = [ + Chunk(content=chunk.text, embedding=chunk.embedding) + for chunk in config.chunker_instance.chunk(transcript_text) + ] + + # Create document + from app.db import Document, DocumentType + + document = Document( + title=video_data.get("title", "YouTube Video"), + document_type=DocumentType.YOUTUBE_VIDEO, + document_metadata={ + "url": url, + "video_id": video_id, + "video_title": video_data.get("title", "YouTube Video"), + "author": video_data.get("author_name", "Unknown"), + "thumbnail": video_data.get("thumbnail_url", "") + }, + content=summary_content, + embedding=summary_embedding, + chunks=chunks, + search_space_id=search_space_id + ) + + session.add(document) + await session.commit() + await session.refresh(document) + + return document + except SQLAlchemyError as db_error: + await session.rollback() + raise db_error + except Exception as e: + await session.rollback() + import logging + logging.error(f"Failed to process YouTube video: {str(e)}") + raise diff --git a/surfsense_backend/app/tasks/stream_connector_search_results.py b/surfsense_backend/app/tasks/stream_connector_search_results.py index 99b039489..5c563dcbb 100644 --- a/surfsense_backend/app/tasks/stream_connector_search_results.py +++ b/surfsense_backend/app/tasks/stream_connector_search_results.py @@ -59,6 +59,32 @@ async def stream_connector_search_results( # Process each selected connector for connector in selected_connectors: + if connector == "YOUTUBE_VIDEO": + # Send terminal message about starting search + yield streaming_service.add_terminal_message("Starting to search for youtube videos...") + + # Search for YouTube videos using reformulated query + result_object, youtube_chunks = await connector_service.search_youtube( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=TOP_K + ) + + # Send terminal message about search results + yield streaming_service.add_terminal_message( + f"Found {len(result_object['sources'])} relevant YouTube videos", + "success" + ) + + # Update sources + all_sources.append(result_object) + yield streaming_service.update_sources(all_sources) + + # Add documents to collection + all_raw_documents.extend(youtube_chunks) + + # Extension Docs if connector == "EXTENSION": # Send terminal message about starting search diff --git a/surfsense_backend/app/utils/connector_service.py b/surfsense_backend/app/utils/connector_service.py index 60dd1cf6d..9e676e59d 100644 --- a/surfsense_backend/app/utils/connector_service.py +++ b/surfsense_backend/app/utils/connector_service.py @@ -479,4 +479,83 @@ class ConnectorService: "sources": sources_list, } - return result_object, extension_chunks \ No newline at end of file + return result_object, extension_chunks + + async def search_youtube(self, user_query: str, user_id: int, search_space_id: int, top_k: int = 20) -> tuple: + """ + Search for YouTube videos and return both the source information and langchain documents + + Args: + user_query: The user's query + user_id: The user's ID + search_space_id: The search space ID to search in + top_k: Maximum number of results to return + + Returns: + tuple: (sources_info, langchain_documents) + """ + youtube_chunks = await self.retriever.hybrid_search( + query_text=user_query, + top_k=top_k, + user_id=user_id, + search_space_id=search_space_id, + document_type="YOUTUBE_VIDEO" + ) + + # Map youtube_chunks to the required format + mapped_sources = {} + for i, chunk in enumerate(youtube_chunks): + # Fix for UI + youtube_chunks[i]['document']['id'] = self.source_id_counter + + # Extract document metadata + document = chunk.get('document', {}) + metadata = document.get('metadata', {}) + + # Extract YouTube-specific metadata + video_title = metadata.get('video_title', 'Untitled Video') + video_id = metadata.get('video_id', '') + channel_name = metadata.get('channel_name', '') + published_date = metadata.get('published_date', '') + + # Create a more descriptive title for YouTube videos + title = video_title + if channel_name: + title += f" - {channel_name}" + + # Create a more descriptive description for YouTube videos + description = metadata.get('description', chunk.get('content', '')[:100]) + if len(description) == 100: + description += "..." + + # For URL, construct a URL to the YouTube video + url = f"https://www.youtube.com/watch?v={video_id}" if video_id else "" + + source = { + "id": self.source_id_counter, + "title": title, + "description": description, + "url": url, + "video_id": video_id, # Additional field for YouTube videos + "channel_name": channel_name # Additional field for YouTube videos + } + + self.source_id_counter += 1 + + # Use video_id as a unique identifier for tracking unique sources + source_key = video_id or f"youtube_{i}" + if source_key and source_key not in mapped_sources: + mapped_sources[source_key] = source + + # Convert to list of sources + sources_list = list(mapped_sources.values()) + + # Create result object + result_object = { + "id": 6, # Assign a unique ID for the YouTube connector + "name": "YouTube Videos", + "type": "YOUTUBE_VIDEO", + "sources": sources_list, + } + + return result_object, youtube_chunks \ No newline at end of file diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index c71bfa275..2d1e00a63 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -26,4 +26,5 @@ dependencies = [ "unstructured[all-docs]>=0.16.25", "uvicorn[standard]>=0.34.0", "validators>=0.34.0", + "youtube-transcript-api>=1.0.3", ] diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index cb2829666..e64ff3958 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -580,6 +580,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686 }, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604 }, +] + [[package]] name = "deprecated" version = "1.2.18" @@ -3240,6 +3249,7 @@ dependencies = [ { name = "unstructured-client" }, { name = "uvicorn", extra = ["standard"] }, { name = "validators" }, + { name = "youtube-transcript-api" }, ] [package.metadata] @@ -3265,6 +3275,7 @@ requires-dist = [ { name = "unstructured-client", specifier = ">=0.30.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, { name = "validators", specifier = ">=0.34.0" }, + { name = "youtube-transcript-api", specifier = ">=1.0.3" }, ] [[package]] @@ -3919,6 +3930,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f5/4b/a06e0ec3d155924f77835ed2d167ebd3b211a7b0853da1cf8d8414d784ef/yarl-1.18.3-py3-none-any.whl", hash = "sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b", size = 45109 }, ] +[[package]] +name = "youtube-transcript-api" +version = "1.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "defusedxml" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/32/f60d87a99c05a53604c58f20f670c7ea6262b55e0bbeb836ffe4550b248b/youtube_transcript_api-1.0.3.tar.gz", hash = "sha256:902baf90e7840a42e1e148335e09fe5575dbff64c81414957aea7038e8a4db46", size = 2153252 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/44/40c03bb0f8bddfb9d2beff2ed31641f52d96c287ba881d20e0c074784ac2/youtube_transcript_api-1.0.3-py3-none-any.whl", hash = "sha256:d1874e57de65cf14c9d7d09b2b37c814d6287fa0e770d4922c4cd32a5b3f6c47", size = 2169911 }, +] + [[package]] name = "zipp" version = "3.21.0" diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index fd2f25bc2..66f8b0810 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -94,7 +94,7 @@ import rehypeSanitize from "rehype-sanitize"; import remarkGfm from "remark-gfm"; import { DocumentViewer } from "@/components/document-viewer"; import { JsonMetadataViewer } from "@/components/json-metadata-viewer"; -import { IconBrandNotion, IconBrandSlack } from "@tabler/icons-react"; +import { IconBrandNotion, IconBrandSlack, IconBrandYoutube } from "@tabler/icons-react"; // Define animation variants for reuse const fadeInScale = { @@ -114,7 +114,7 @@ const fadeInScale = { type Document = { id: number; title: string; - document_type: "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE"; + document_type: "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE" | "YOUTUBE_VIDEO"; document_metadata: any; content: string; created_at: string; @@ -141,6 +141,7 @@ const documentTypeIcons = { SLACK_CONNECTOR: IconBrandSlack, NOTION_CONNECTOR: IconBrandNotion, FILE: File, + YOUTUBE_VIDEO: IconBrandYoutube, } as const; const columns: ColumnDef[] = [ diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx new file mode 100644 index 000000000..540d26182 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/youtube/page.tsx @@ -0,0 +1,302 @@ +"use client"; + +import { useState } from 'react'; +import { useParams, useRouter } from 'next/navigation'; +import { Tag, TagInput } from "emblor"; +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } from "@/components/ui/card"; +import { toast } from "sonner"; +import { Youtube, Loader2 } from "lucide-react"; +import { motion } from "framer-motion"; + +// YouTube video ID validation regex +const youtubeRegex = /^(https:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})$/; + +export default function YouTubeVideoAdder() { + const params = useParams(); + const router = useRouter(); + const search_space_id = params.search_space_id as string; + + const [videoTags, setVideoTags] = useState([]); + const [activeTagIndex, setActiveTagIndex] = useState(null); + const [isSubmitting, setIsSubmitting] = useState(false); + const [error, setError] = useState(null); + + // Function to validate a YouTube URL + const isValidYoutubeUrl = (url: string): boolean => { + return youtubeRegex.test(url); + }; + + // Function to extract video ID from URL + const extractVideoId = (url: string): string | null => { + const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})/); + return match ? match[1] : null; + }; + + // Function to handle video URL submission + const handleSubmit = async () => { + // Validate that we have at least one video URL + if (videoTags.length === 0) { + setError("Please add at least one YouTube video URL"); + return; + } + + // Validate all URLs + const invalidUrls = videoTags.filter(tag => !isValidYoutubeUrl(tag.text)); + if (invalidUrls.length > 0) { + setError(`Invalid YouTube URLs detected: ${invalidUrls.map(tag => tag.text).join(', ')}`); + return; + } + + setError(null); + setIsSubmitting(true); + + try { + toast("YouTube Video Processing", { + description: "Starting YouTube video processing...", + }); + + // Extract URLs from tags + const videoUrls = videoTags.map(tag => tag.text); + + // Make API call to backend + const response = await fetch(`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/`, { + method: "POST", + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${localStorage.getItem("surfsense_bearer_token")}` + }, + body: JSON.stringify({ + "document_type": "YOUTUBE_VIDEO", + "content": videoUrls, + "search_space_id": parseInt(search_space_id) + }), + }); + + if (!response.ok) { + throw new Error("Failed to process YouTube videos"); + } + + await response.json(); + + toast("Processing Successful", { + description: "YouTube videos have been submitted for processing", + }); + + // Redirect to documents page + router.push(`/dashboard/${search_space_id}/documents`); + } catch (error: any) { + setError(error.message || "An error occurred while processing YouTube videos"); + toast("Processing Error", { + description: `Error processing YouTube videos: ${error.message}`, + }); + } finally { + setIsSubmitting(false); + } + }; + + // Function to add a new video URL tag + const handleAddTag = (text: string) => { + // Basic URL validation + if (!isValidYoutubeUrl(text)) { + toast("Invalid YouTube URL", { + description: "Please enter a valid YouTube video URL", + }); + return; + } + + // Check for duplicates + if (videoTags.some(tag => tag.text === text)) { + toast("Duplicate URL", { + description: "This YouTube video has already been added", + }); + return; + } + + // Add the new tag + const newTag: Tag = { + id: Date.now().toString(), + text: text, + }; + + setVideoTags([...videoTags, newTag]); + }; + + // Animation variants + const containerVariants = { + hidden: { opacity: 0 }, + visible: { + opacity: 1, + transition: { + staggerChildren: 0.1 + } + } + }; + + const itemVariants = { + hidden: { y: 20, opacity: 0 }, + visible: { + y: 0, + opacity: 1, + transition: { + type: "spring", + stiffness: 300, + damping: 24 + } + } + }; + + return ( +
+ + + + + + + Add YouTube Videos + + + Enter YouTube video URLs to add to your document collection + + + + + + +
+
+ + +

+ Add multiple YouTube URLs by pressing Enter after each one +

+
+ + {error && ( + + {error} + + )} + + +

Tips for adding YouTube videos:

+
    +
  • Use standard YouTube URLs (youtube.com/watch?v= or youtu.be/)
    • +
  • Make sure videos are publicly accessible
    • +
  • Supported formats: youtube.com/watch?v=VIDEO_ID or youtu.be/VIDEO_ID
    • +
  • Processing may take some time depending on video length
    • +
+ + + {videoTags.length > 0 && ( + +

Preview:

+
+ {videoTags.map((tag, index) => { + const videoId = extractVideoId(tag.text); + return videoId ? ( + + + + ) : null; + })} +
+ + )} +
+ + + + + + + + + + + +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx index 6eeb720d9..7449e10b5 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx @@ -48,6 +48,10 @@ export default function DashboardLayout({ title: "Add Webpages", url: `/dashboard/${search_space_id}/documents/webpage`, }, + { + title: "Add Youtube Videos", + url: `/dashboard/${search_space_id}/documents/youtube`, + }, { title: "Manage Documents", url: `/dashboard/${search_space_id}/documents`, diff --git a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx index 6d74488da..8156f6d2a 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx @@ -239,12 +239,10 @@ const SourcesDialogContent = ({ const ChatPage = () => { const [token, setToken] = React.useState(null); - const [showAnswer, setShowAnswer] = useState(true); const [activeTab, setActiveTab] = useState(""); const [dialogOpen, setDialogOpen] = useState(false); const [sourcesPage, setSourcesPage] = useState(1); const [expandedSources, setExpandedSources] = useState(false); - const [isLoadingMore, setIsLoadingMore] = useState(false); const [canScrollLeft, setCanScrollLeft] = useState(false); const [canScrollRight, setCanScrollRight] = useState(true); const [sourceFilter, setSourceFilter] = useState(""); @@ -258,7 +256,6 @@ const ChatPage = () => { const terminalMessagesRef = useRef(null); const { connectorSourceItems, isLoading: isLoadingConnectors } = useSearchSourceConnectors(); - const SOURCES_PER_PAGE = 5; const INITIAL_SOURCES_DISPLAY = 3; const { search_space_id, chat_id } = useParams(); @@ -836,7 +833,7 @@ const ChatPage = () => { {connectorSources.map(connector => (
- {getMainViewSources(connector).map((source: any) => ( + {getMainViewSources(connector)?.map((source: any) => (
@@ -874,7 +871,7 @@ const ChatPage = () => { setSourcesPage={setSourcesPage} setSourceFilter={setSourceFilter} setExpandedSources={setExpandedSources} - isLoadingMore={isLoadingMore} + isLoadingMore={false} /> @@ -887,7 +884,7 @@ const ChatPage = () => { {/* Answer Section */}
- {showAnswer && ( + {
{message.annotations && (() => { // Get all ANSWER annotations @@ -913,7 +910,7 @@ const ChatPage = () => { return ; })()}
- )} + }
{/* Scroll to bottom button */}
diff --git a/surfsense_web/components/chat/ConnectorComponents.tsx b/surfsense_web/components/chat/ConnectorComponents.tsx index b6dad8216..56aaa21b1 100644 --- a/surfsense_web/components/chat/ConnectorComponents.tsx +++ b/surfsense_web/components/chat/ConnectorComponents.tsx @@ -4,15 +4,14 @@ import { Plus, Search, Globe, - BookOpen, Sparkles, Microscope, Telescope, File, Link, - Slack, - Webhook + Webhook, } from 'lucide-react'; +import { IconBrandNotion, IconBrandSlack, IconBrandYoutube } from "@tabler/icons-react"; import { Button } from '@/components/ui/button'; import { Connector, ResearchMode } from './types'; @@ -21,6 +20,8 @@ export const getConnectorIcon = (connectorType: string) => { const iconProps = { className: "h-4 w-4" }; switch(connectorType) { + case 'YOUTUBE_VIDEO': + return ; case 'CRAWLED_URL': return ; case 'FILE': @@ -31,9 +32,9 @@ export const getConnectorIcon = (connectorType: string) => { case 'TAVILY_API': return ; case 'SLACK_CONNECTOR': - return ; + return ; case 'NOTION_CONNECTOR': - return ; + return ; case 'DEEP': return ; case 'DEEPER': diff --git a/surfsense_web/components/chat/connector-sources.ts b/surfsense_web/components/chat/connector-sources.ts index 6f0985730..58a4bd1a4 100644 --- a/surfsense_web/components/chat/connector-sources.ts +++ b/surfsense_web/components/chat/connector-sources.ts @@ -15,4 +15,9 @@ export const connectorSourcesMenu = [ name: "Extension", type: "EXTENSION", }, + { + id: 4, + name: "Youtube Video", + type: "YOUTUBE_VIDEO", + } ]; \ No newline at end of file diff --git a/surfsense_web/hooks/useSearchSourceConnectors.ts b/surfsense_web/hooks/useSearchSourceConnectors.ts index 499bea5b7..2bb6bc1f6 100644 --- a/surfsense_web/hooks/useSearchSourceConnectors.ts +++ b/surfsense_web/hooks/useSearchSourceConnectors.ts @@ -43,6 +43,12 @@ export const useSearchSourceConnectors = () => { name: "Extension", type: "EXTENSION", sources: [], + }, + { + id: 4, + name: "Youtube Video", + type: "YOUTUBE_VIDEO", + sources: [], } ]); @@ -108,6 +114,12 @@ export const useSearchSourceConnectors = () => { name: "Extension", type: "EXTENSION", sources: [], + }, + { + id: 4, + name: "Youtube Video", + type: "YOUTUBE_VIDEO", + sources: [], } ];