diff --git a/.dockerignore b/.dockerignore index ad6805174..70d7fb07e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -75,7 +75,6 @@ surfsense_backend/lib64/ # Logs **/*.log -**/logs/ # Temporary files **/tmp/ diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py index 8fd5f3b71..6c8deb409 100644 --- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py +++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py @@ -50,6 +50,9 @@ def create_surfsense_deep_agent( - display_image: Display images in chat - scrape_webpage: Extract content from webpages + The agent also includes TodoListMiddleware by default (via create_deep_agent) which provides: + - write_todos: Create and update planning/todo lists for complex tasks + The system prompt can be configured via agent_config: - Custom system instructions (or use defaults) - Citation toggle (enable/disable citation requirements) @@ -138,6 +141,7 @@ def create_surfsense_deep_agent( system_prompt = build_surfsense_system_prompt() # Create the deep agent with system prompt and checkpointer + # Note: TodoListMiddleware (write_todos) is included by default in create_deep_agent agent = create_deep_agent( model=llm, tools=tools, diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/agents/new_chat/system_prompt.py index 61a8fbdd6..695d62bfb 100644 --- a/surfsense_backend/app/agents/new_chat/system_prompt.py +++ b/surfsense_backend/app/agents/new_chat/system_prompt.py @@ -64,18 +64,23 @@ You have access to the following tools: - The preview card will automatically be displayed in the chat. 4. display_image: Display an image in the chat with metadata. - - Use this tool when you want to show an image from a URL to the user. + - Use this tool ONLY when you have a valid public HTTP/HTTPS image URL to show. - This displays the image with an optional title, description, and source attribution. - - Common use cases: - * Showing an image from a URL mentioned in the conversation - * Displaying a diagram, chart, or illustration you're referencing - * Showing visual examples when explaining concepts - - IMPORTANT: Do NOT use this tool for user-uploaded image attachments! - * User attachments are already visible in the chat UI - the user can see them - * This tool requires a valid HTTP/HTTPS URL, not a local file path - * When a user uploads an image, just analyze it and respond - don't try to display it again + - Valid use cases: + * Showing an image from a URL the user explicitly mentioned in their message + * Displaying images found in scraped webpage content (from scrape_webpage tool) + * Showing a publicly accessible diagram or chart from a known URL + + CRITICAL - NEVER USE THIS TOOL FOR USER-UPLOADED ATTACHMENTS: + When a user uploads/attaches an image file to their message: + * The image is ALREADY VISIBLE in the chat UI as a thumbnail on their message + * You do NOT have a URL for their uploaded image - only extracted text/description + * Calling display_image will FAIL and show "Image not available" error + * Simply analyze the image content and respond with your analysis - DO NOT try to display it + * The user can already see their own uploaded image - they don't need you to show it again + - Args: - - src: The URL of the image to display (must be a valid HTTP/HTTPS image URL, not a local path) + - src: The URL of the image (MUST be a valid public HTTP/HTTPS URL that you know exists) - alt: Alternative text describing the image (for accessibility) - title: Optional title to display below the image - description: Optional description providing context about the image @@ -104,6 +109,20 @@ You have access to the following tools: * This makes your response more visual and engaging. * Prioritize showing: diagrams, charts, infographics, key illustrations, or images that help explain the content. * Don't show every image - just the most relevant 1-3 images that enhance understanding. + +6. write_todos: Create and update a planning/todo list to break down complex tasks. + - IMPORTANT: Use this tool when the user asks you to create a plan, break down a task, or explain something in structured steps. + - This tool creates a visual plan with progress tracking that the user can see in the UI. + - When to use: + * User asks to "create a plan" or "break down" a task + * User asks for "steps" to do something + * User asks you to "explain" something in sections + * Any multi-step task that would benefit from structured planning + - Args: + - todos: List of todo items, each with: + * content: Description of the task (required) + * status: "pending", "in_progress", or "completed" (required) + - The tool automatically adds IDs and formats the output for the UI. - User: "Fetch all my notes and what's in them?" @@ -134,8 +153,15 @@ You have access to the following tools: - User: "Show me this image: https://example.com/image.png" - Call: `display_image(src="https://example.com/image.png", alt="User shared image")` -- User: "Can you display a diagram of a neural network?" - - Call: `display_image(src="https://example.com/neural-network.png", alt="Neural network diagram", title="Neural Network Architecture", description="A visual representation of a neural network with input, hidden, and output layers")` +- User uploads an image file and asks: "What is this image about?" + - DO NOT call display_image! The user's uploaded image is already visible in the chat. + - Simply analyze the image content (which you receive as extracted text/description) and respond. + - WRONG: `display_image(src="...", ...)` - This will fail with "Image not available" + - CORRECT: Just provide your analysis directly: "Based on the image you shared, this appears to be..." + +- User uploads a screenshot and asks: "Can you explain what's in this image?" + - DO NOT call display_image! Just analyze and respond directly. + - The user can already see their screenshot - they don't need you to display it again. - User: "Read this article and summarize it for me: https://example.com/blog/ai-trends" - Call: `scrape_webpage(url="https://example.com/blog/ai-trends")` @@ -154,6 +180,34 @@ You have access to the following tools: - Then, if the content contains useful diagrams/images like `![Neural Network Diagram](https://example.com/nn-diagram.png)`: - Call: `display_image(src="https://example.com/nn-diagram.png", alt="Neural Network Diagram", title="Neural Network Architecture")` - Then provide your explanation, referencing the displayed image + +- User: "Create a plan for building a user authentication system" + - Call: `write_todos(todos=[{"content": "Design database schema for users and sessions", "status": "in_progress"}, {"content": "Implement registration and login endpoints", "status": "pending"}, {"content": "Add password reset functionality", "status": "pending"}])` + - Then explain each step in detail as you work through them + +- User: "Break down how to build a REST API into steps" + - Call: `write_todos(todos=[{"content": "Design API endpoints and data models", "status": "in_progress"}, {"content": "Set up server framework and routing", "status": "pending"}, {"content": "Implement CRUD operations", "status": "pending"}, {"content": "Add authentication and error handling", "status": "pending"}])` + - Then provide detailed explanations for each step + +- User: "Help me plan my trip to Japan" + - Call: `write_todos(todos=[{"content": "Research best time to visit and book flights", "status": "in_progress"}, {"content": "Plan itinerary for cities to visit", "status": "pending"}, {"content": "Book accommodations", "status": "pending"}, {"content": "Prepare travel documents and currency", "status": "pending"}])` + - Then provide travel preparation guidance + +- User: "Break down how to learn guitar" + - Call: `write_todos(todos=[{"content": "Learn basic chords and finger positioning", "status": "in_progress"}, {"content": "Practice strumming patterns", "status": "pending"}, {"content": "Learn to read tabs and sheet music", "status": "pending"}, {"content": "Master simple songs", "status": "pending"}])` + - Then provide learning milestones and tips + +- User: "Plan my workout routine for the week" + - Call: `write_todos(todos=[{"content": "Monday: Upper body strength training", "status": "in_progress"}, {"content": "Tuesday: Cardio and core workout", "status": "pending"}, {"content": "Wednesday: Rest or light stretching", "status": "pending"}, {"content": "Thursday: Lower body strength training", "status": "pending"}, {"content": "Friday: Full body HIIT session", "status": "pending"}])` + - Then provide exercise details and tips + +- User: "Help me organize my home renovation project" + - Call: `write_todos(todos=[{"content": "Define scope and create budget", "status": "in_progress"}, {"content": "Research and hire contractors", "status": "pending"}, {"content": "Obtain necessary permits", "status": "pending"}, {"content": "Order materials and fixtures", "status": "pending"}, {"content": "Execute renovation phases", "status": "pending"}])` + - Then provide detailed renovation guidance + +- User: "What steps should I take to start a podcast?" + - Call: `write_todos(todos=[{"content": "Define podcast concept and target audience", "status": "in_progress"}, {"content": "Set up recording equipment and software", "status": "pending"}, {"content": "Plan episode structure and content", "status": "pending"}, {"content": "Record and edit first episodes", "status": "pending"}, {"content": "Choose hosting platform and publish", "status": "pending"}])` + - Then provide podcast launch guidance """ diff --git a/surfsense_backend/app/agents/new_chat/tools/link_preview.py b/surfsense_backend/app/agents/new_chat/tools/link_preview.py index 17e89345e..3e2070a14 100644 --- a/surfsense_backend/app/agents/new_chat/tools/link_preview.py +++ b/surfsense_backend/app/agents/new_chat/tools/link_preview.py @@ -14,8 +14,8 @@ from urllib.parse import urlparse import httpx import trafilatura from fake_useragent import UserAgent -from langchain_community.document_loaders import AsyncChromiumLoader from langchain_core.tools import tool +from playwright.async_api import async_playwright logger = logging.getLogger(__name__) @@ -170,7 +170,7 @@ def _make_absolute_url(image_url: str, base_url: str) -> str: async def fetch_with_chromium(url: str) -> dict[str, Any] | None: """ - Fetch page content using headless Chromium browser. + Fetch page content using headless Chromium browser via Playwright. Used as a fallback when simple HTTP requests are blocked (403, etc.). Args: @@ -186,18 +186,17 @@ async def fetch_with_chromium(url: str) -> dict[str, Any] | None: ua = UserAgent() user_agent = ua.random - # Use AsyncChromiumLoader to fetch the page - crawl_loader = AsyncChromiumLoader( - urls=[url], headless=True, user_agent=user_agent - ) - documents = await crawl_loader.aload() + # Use Playwright to fetch the page + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + context = await browser.new_context(user_agent=user_agent) + page = await context.new_page() - if not documents: - logger.warning(f"[link_preview] Chromium returned no documents for {url}") - return None - - doc = documents[0] - raw_html = doc.page_content + try: + await page.goto(url, wait_until="domcontentloaded", timeout=30000) + raw_html = await page.content() + finally: + await browser.close() if not raw_html or len(raw_html.strip()) == 0: logger.warning(f"[link_preview] Chromium returned empty content for {url}") @@ -280,15 +279,18 @@ def create_link_preview_tool(): url = f"https://{url}" try: + # Generate a random User-Agent to avoid bot detection + ua = UserAgent() + user_agent = ua.random + # Use a browser-like User-Agent to fetch Open Graph metadata. - # This is the same approach used by Slack, Discord, Twitter, etc. for link previews. # We're only fetching publicly available metadata (title, description, thumbnail) # that websites intentionally expose via OG tags for link preview purposes. async with httpx.AsyncClient( timeout=10.0, follow_redirects=True, headers={ - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "User-Agent": user_agent, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9", "Accept-Encoding": "gzip, deflate, br", diff --git a/surfsense_backend/app/agents/new_chat/tools/registry.py b/surfsense_backend/app/agents/new_chat/tools/registry.py index 3b0c2ddac..bc305aecc 100644 --- a/surfsense_backend/app/agents/new_chat/tools/registry.py +++ b/surfsense_backend/app/agents/new_chat/tools/registry.py @@ -125,6 +125,7 @@ BUILTIN_TOOLS: list[ToolDefinition] = [ ), requires=[], # firecrawl_api_key is optional ), + # Note: write_todos is now provided by TodoListMiddleware from deepagents # ========================================================================= # ADD YOUR CUSTOM TOOLS BELOW # ========================================================================= diff --git a/surfsense_backend/app/connectors/webcrawler_connector.py b/surfsense_backend/app/connectors/webcrawler_connector.py index 3fc61f0b5..5d6ea98c8 100644 --- a/surfsense_backend/app/connectors/webcrawler_connector.py +++ b/surfsense_backend/app/connectors/webcrawler_connector.py @@ -1,7 +1,7 @@ """ WebCrawler Connector Module -A module for crawling web pages and extracting content using Firecrawl or AsyncChromiumLoader. +A module for crawling web pages and extracting content using Firecrawl or Playwright. Provides a unified interface for web scraping. """ @@ -12,7 +12,7 @@ import trafilatura import validators from fake_useragent import UserAgent from firecrawl import AsyncFirecrawlApp -from langchain_community.document_loaders import AsyncChromiumLoader +from playwright.async_api import async_playwright logger = logging.getLogger(__name__) @@ -25,7 +25,9 @@ class WebCrawlerConnector: Initialize the WebCrawlerConnector class. Args: - firecrawl_api_key: Firecrawl API key (optional, will use AsyncChromiumLoader if not provided) + firecrawl_api_key: Firecrawl API key (optional). If provided, Firecrawl will be tried first + and Chromium will be used as fallback if Firecrawl fails. If not provided, + Chromium will be used directly. """ self.firecrawl_api_key = firecrawl_api_key self.use_firecrawl = bool(firecrawl_api_key) @@ -46,6 +48,9 @@ class WebCrawlerConnector: """ Crawl a single URL and extract its content. + If Firecrawl API key is provided, tries Firecrawl first and falls back to Chromium + if Firecrawl fails. If no Firecrawl API key is provided, uses Chromium directly. + Args: url: URL to crawl formats: List of formats to extract (e.g., ["markdown", "html"]) - only for Firecrawl @@ -56,19 +61,37 @@ class WebCrawlerConnector: - content: Extracted content (markdown or HTML) - metadata: Page metadata (title, description, etc.) - source: Original URL - - crawler_type: Type of crawler used + - crawler_type: Type of crawler used ("firecrawl" or "chromium") """ try: # Validate URL if not validators.url(url): return None, f"Invalid URL: {url}" + # Try Firecrawl first if API key is provided if self.use_firecrawl: - result = await self._crawl_with_firecrawl(url, formats) + try: + logger.info(f"[webcrawler] Using Firecrawl for: {url}") + result = await self._crawl_with_firecrawl(url, formats) + return result, None + except Exception as firecrawl_error: + # Firecrawl failed, fallback to Chromium + logger.warning( + f"[webcrawler] Firecrawl failed, falling back to Chromium+Trafilatura for: {url}" + ) + try: + result = await self._crawl_with_chromium(url) + return result, None + except Exception as chromium_error: + return ( + None, + f"Both Firecrawl and Chromium failed. Firecrawl error: {firecrawl_error!s}, Chromium error: {chromium_error!s}", + ) else: + # No Firecrawl API key, use Chromium directly + logger.info(f"[webcrawler] Using Chromium+Trafilatura for: {url}") result = await self._crawl_with_chromium(url) - - return result, None + return result, None except Exception as e: return None, f"Error crawling URL {url}: {e!s}" @@ -126,7 +149,7 @@ class WebCrawlerConnector: async def _crawl_with_chromium(self, url: str) -> dict[str, Any]: """ - Crawl URL using AsyncChromiumLoader with Trafilatura for content extraction. + Crawl URL using Playwright with Trafilatura for content extraction. Falls back to raw HTML if Trafilatura extraction fails. Args: @@ -142,30 +165,30 @@ class WebCrawlerConnector: ua = UserAgent() user_agent = ua.random - # Pass User-Agent to AsyncChromiumLoader - crawl_loader = AsyncChromiumLoader( - urls=[url], headless=True, user_agent=user_agent - ) - documents = await crawl_loader.aload() + # Use Playwright to fetch the page + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + context = await browser.new_context(user_agent=user_agent) + page = await context.new_page() - if not documents: + try: + await page.goto(url, wait_until="domcontentloaded", timeout=30000) + raw_html = await page.content() + page_title = await page.title() + finally: + await browser.close() + + if not raw_html: raise ValueError(f"Failed to load content from {url}") - doc = documents[0] - raw_html = doc.page_content - - # Extract basic metadata from the document - base_metadata = doc.metadata if doc.metadata else {} + # Extract basic metadata from the page + base_metadata = {"title": page_title} if page_title else {} # Try to extract main content using Trafilatura extracted_content = None trafilatura_metadata = None try: - logger.info( - f"Attempting to extract main content from {url} using Trafilatura" - ) - # Extract main content as markdown extracted_content = trafilatura.extract( raw_html, @@ -179,23 +202,10 @@ class WebCrawlerConnector: # Extract metadata using Trafilatura trafilatura_metadata = trafilatura.extract_metadata(raw_html) - if extracted_content and len(extracted_content.strip()) > 0: - logger.info( - f"Successfully extracted main content from {url} using Trafilatura " - f"({len(extracted_content)} chars vs {len(raw_html)} chars raw HTML)" - ) - else: - logger.warning( - f"Trafilatura extraction returned empty content for {url}, " - "falling back to raw HTML" - ) + if not extracted_content or len(extracted_content.strip()) == 0: extracted_content = None - except Exception as e: - logger.warning( - f"Trafilatura extraction failed for {url}: {e}. " - "Falling back to raw HTML" - ) + except Exception: extracted_content = None # Build metadata, preferring Trafilatura metadata when available diff --git a/surfsense_backend/app/routes/logs_routes.py b/surfsense_backend/app/routes/logs_routes.py index 98fd9141e..e7e00280e 100644 --- a/surfsense_backend/app/routes/logs_routes.py +++ b/surfsense_backend/app/routes/logs_routes.py @@ -319,6 +319,9 @@ async def get_logs_summary( if log.log_metadata else "Unknown" ) + document_id = ( + log.log_metadata.get("document_id") if log.log_metadata else None + ) summary["active_tasks"].append( { "id": log.id, @@ -326,6 +329,7 @@ async def get_logs_summary( "message": log.message, "started_at": log.created_at, "source": log.source, + "document_id": document_id, } ) diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py index aff6fa32b..11024e513 100644 --- a/surfsense_backend/app/tasks/chat/stream_new_chat.py +++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py @@ -69,6 +69,30 @@ def format_mentioned_documents_as_context(documents: list[Document]) -> str: return "\n".join(context_parts) +def extract_todos_from_deepagents(command_output) -> dict: + """ + Extract todos from deepagents' TodoListMiddleware Command output. + + deepagents returns a Command object with: + - Command.update['todos'] = [{'content': '...', 'status': '...'}] + + Returns the todos directly (no transformation needed - UI matches deepagents format). + """ + todos_data = [] + if hasattr(command_output, "update"): + # It's a Command object from deepagents + update = command_output.update + todos_data = update.get("todos", []) + elif isinstance(command_output, dict): + # Already a dict - check if it has todos directly or in update + if "todos" in command_output: + todos_data = command_output.get("todos", []) + elif "update" in command_output and isinstance(command_output["update"], dict): + todos_data = command_output["update"].get("todos", []) + + return {"todos": todos_data} + + async def stream_new_chat( user_query: str, search_space_id: int, @@ -146,6 +170,16 @@ async def stream_new_chat( # Create connector service connector_service = ConnectorService(session, search_space_id=search_space_id) + # Get Firecrawl API key from webcrawler connector if configured + from app.db import SearchSourceConnectorType + + firecrawl_api_key = None + webcrawler_connector = await connector_service.get_connector_by_type( + SearchSourceConnectorType.WEBCRAWLER_CONNECTOR, search_space_id + ) + if webcrawler_connector and webcrawler_connector.config: + firecrawl_api_key = webcrawler_connector.config.get("FIRECRAWL_API_KEY") + # Get the PostgreSQL checkpointer for persistent conversation memory checkpointer = await get_checkpointer() @@ -157,6 +191,7 @@ async def stream_new_chat( connector_service=connector_service, checkpointer=checkpointer, agent_config=agent_config, # Pass prompt configuration + firecrawl_api_key=firecrawl_api_key, # Pass Firecrawl API key if configured ) # Build input with message history from frontend @@ -211,7 +246,8 @@ async def stream_new_chat( config = { "configurable": { "thread_id": str(chat_id), - } + }, + "recursion_limit": 80, # Increase from default 25 to allow more tool iterations } # Start the message stream @@ -233,6 +269,8 @@ async def stream_new_chat( completed_step_ids: set[str] = set() # Track if we just finished a tool (text flows silently after tools) just_finished_tool: bool = False + # Track write_todos calls to show "Creating plan" vs "Updating plan" + write_todos_call_count: int = 0 def next_thinking_step_id() -> str: nonlocal thinking_step_counter @@ -441,6 +479,60 @@ async def stream_new_chat( status="in_progress", items=last_active_step_items, ) + elif tool_name == "write_todos": + # Track write_todos calls for better messaging + write_todos_call_count += 1 + todos = ( + tool_input.get("todos", []) + if isinstance(tool_input, dict) + else [] + ) + todo_count = len(todos) if isinstance(todos, list) else 0 + + if write_todos_call_count == 1: + # First call - creating the plan + last_active_step_title = "Creating plan" + last_active_step_items = [f"Defining {todo_count} tasks..."] + else: + # Subsequent calls - updating the plan + # Try to provide context about what's being updated + in_progress_count = ( + sum( + 1 + for t in todos + if isinstance(t, dict) + and t.get("status") == "in_progress" + ) + if isinstance(todos, list) + else 0 + ) + completed_count = ( + sum( + 1 + for t in todos + if isinstance(t, dict) + and t.get("status") == "completed" + ) + if isinstance(todos, list) + else 0 + ) + + last_active_step_title = "Updating progress" + last_active_step_items = ( + [ + f"Progress: {completed_count}/{todo_count} completed", + f"In progress: {in_progress_count} tasks", + ] + if completed_count > 0 + else [f"Working on {todo_count} tasks"] + ) + + yield streaming_service.format_thinking_step( + step_id=tool_step_id, + title=last_active_step_title, + status="in_progress", + items=last_active_step_items, + ) elif tool_name == "generate_podcast": podcast_title = ( tool_input.get("podcast_title", "SurfSense Podcast") @@ -465,6 +557,15 @@ async def stream_new_chat( status="in_progress", items=last_active_step_items, ) + # elif tool_name == "ls": + # last_active_step_title = "Exploring files" + # last_active_step_items = [] + # yield streaming_service.format_thinking_step( + # step_id=tool_step_id, + # title="Exploring files", + # status="in_progress", + # items=None, + # ) else: last_active_step_title = f"Using {tool_name.replace('_', ' ')}" last_active_step_items = [] @@ -546,9 +647,11 @@ async def stream_new_chat( tool_name = event.get("name", "unknown_tool") raw_output = event.get("data", {}).get("output", "") - # Extract content from ToolMessage if needed - # LangGraph may return a ToolMessage object instead of raw dict - if hasattr(raw_output, "content"): + # Handle deepagents' write_todos Command object specially + if tool_name == "write_todos" and hasattr(raw_output, "update"): + # deepagents returns a Command object - extract todos directly + tool_output = extract_todos_from_deepagents(raw_output) + elif hasattr(raw_output, "content"): # It's a ToolMessage object - extract the content content = raw_output.content # If content is a string that looks like JSON, try to parse it @@ -707,6 +810,104 @@ async def stream_new_chat( status="completed", items=completed_items, ) + elif tool_name == "write_todos": + # Build completion items for planning/updating + if isinstance(tool_output, dict): + todos = tool_output.get("todos", []) + todo_count = len(todos) if isinstance(todos, list) else 0 + completed_count = ( + sum( + 1 + for t in todos + if isinstance(t, dict) + and t.get("status") == "completed" + ) + if isinstance(todos, list) + else 0 + ) + in_progress_count = ( + sum( + 1 + for t in todos + if isinstance(t, dict) + and t.get("status") == "in_progress" + ) + if isinstance(todos, list) + else 0 + ) + + # Use context-aware completion message + if last_active_step_title == "Creating plan": + completed_items = [f"Created {todo_count} tasks"] + else: + # Updating progress - show stats + completed_items = [ + f"Progress: {completed_count}/{todo_count} completed", + ] + if in_progress_count > 0: + # Find the currently in-progress task name + in_progress_task = next( + ( + t.get("content", "")[:40] + for t in todos + if isinstance(t, dict) + and t.get("status") == "in_progress" + ), + None, + ) + if in_progress_task: + completed_items.append( + f"Current: {in_progress_task}..." + ) + else: + completed_items = ["Plan updated"] + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title=last_active_step_title, + status="completed", + items=completed_items, + ) + elif tool_name == "ls": + # Build completion items showing file names found + if isinstance(tool_output, dict): + result = tool_output.get("result", "") + elif isinstance(tool_output, str): + result = tool_output + else: + result = str(tool_output) if tool_output else "" + + # Parse file paths and extract just the file names + file_names = [] + if result: + # The ls tool returns paths, extract just the file/folder names + for line in result.strip().split("\n"): + line = line.strip() + if line: + # Get just the filename from the path + name = line.rstrip("/").split("/")[-1] + if name and len(name) <= 40: + file_names.append(name) + elif name: + file_names.append(name[:37] + "...") + + # Build display items - wrap file names in brackets for icon rendering + if file_names: + if len(file_names) <= 5: + # Wrap each file name in brackets for styled tile rendering + completed_items = [f"[{name}]" for name in file_names] + else: + # Show first few with brackets and count + completed_items = [f"[{name}]" for name in file_names[:4]] + completed_items.append(f"(+{len(file_names) - 4} more)") + else: + completed_items = ["No files found"] + + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title="Exploring files", + status="completed", + items=completed_items, + ) else: yield streaming_service.format_thinking_step( step_id=original_step_id, @@ -843,6 +1044,27 @@ async def stream_new_chat( yield streaming_service.format_terminal_info( "Knowledge base search completed", "success" ) + elif tool_name == "write_todos": + # Stream the full write_todos result so frontend can render the Plan component + yield streaming_service.format_tool_output_available( + tool_call_id, + tool_output + if isinstance(tool_output, dict) + else {"result": tool_output}, + ) + # Send terminal message with plan info + if isinstance(tool_output, dict): + todos = tool_output.get("todos", []) + todo_count = len(todos) if isinstance(todos, list) else 0 + yield streaming_service.format_terminal_info( + f"Plan created ({todo_count} tasks)", + "success", + ) + else: + yield streaming_service.format_terminal_info( + "Plan created", + "success", + ) else: # Default handling for other tools yield streaming_service.format_tool_output_available( diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 5548314cd..2cbdd85f1 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -7,6 +7,8 @@ requires-python = ">=3.12" dependencies = [ "alembic>=1.13.0", "asyncpg>=0.30.0", + "datasets>=2.21.0", + "pyarrow>=15.0.0,<19.0.0", "discord-py>=2.5.2", "docling>=2.15.0", "fastapi>=0.115.8", diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index 208509993..d4af0123d 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -1148,6 +1148,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686 }, ] +[[package]] +name = "datasets" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, + { name = "filelock" }, + { name = "fsspec", extra = ["http"] }, + { name = "huggingface-hub" }, + { name = "multiprocess" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/9d/348ed92110ba5f9b70b51ca1078d4809767a835aa2b7ce7e74ad2b98323d/datasets-4.0.0.tar.gz", hash = "sha256:9657e7140a9050db13443ba21cb5de185af8af944479b00e7ff1e00a61c8dbf1", size = 569566 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/62/eb8157afb21bd229c864521c1ab4fa8e9b4f1b06bafdd8c4668a7a31b5dd/datasets-4.0.0-py3-none-any.whl", hash = "sha256:7ef95e62025fd122882dbce6cb904c8cd3fbc829de6669a5eb939c77d50e203d", size = 494825 }, +] + [[package]] name = "dateparser" version = "1.2.2" @@ -1213,11 +1237,11 @@ wheels = [ [[package]] name = "dill" -version = "0.4.0" +version = "0.3.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976 } +sdist = { url = "https://files.pythonhosted.org/packages/17/4d/ac7ffa80c69ea1df30a8aa11b3578692a5118e7cd1aa157e3ef73b092d15/dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca", size = 184847 } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668 }, + { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252 }, ] [[package]] @@ -1875,11 +1899,16 @@ wheels = [ [[package]] name = "fsspec" -version = "2025.5.1" +version = "2025.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/f7/27f15d41f0ed38e8fcc488584b57e902b331da7f7c6dcda53721b15838fc/fsspec-2025.5.1.tar.gz", hash = "sha256:2e55e47a540b91843b755e83ded97c6e897fa0942b11490113f09e9c443c2475", size = 303033 } +sdist = { url = "https://files.pythonhosted.org/packages/34/f4/5721faf47b8c499e776bc34c6a8fc17efdf7fdef0b00f398128bc5dcb4ac/fsspec-2025.3.0.tar.gz", hash = "sha256:a935fd1ea872591f2b5148907d103488fc523295e6c64b835cfad8c3eca44972", size = 298491 } wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/61/78c7b3851add1481b048b5fdc29067397a1784e2910592bc81bb3f608635/fsspec-2025.5.1-py3-none-any.whl", hash = "sha256:24d3a2e663d5fc735ab256263c4075f374a174c3410c0b25e5bd1970bceaa462", size = 199052 }, + { url = "https://files.pythonhosted.org/packages/56/53/eb690efa8513166adef3e0669afd31e95ffde69fb3c52ec2ac7223ed6018/fsspec-2025.3.0-py3-none-any.whl", hash = "sha256:efb87af3efa9103f94ca91a7f8cb7a4df91af9f74fc106c9c7ea0efd7277c1b3", size = 193615 }, +] + +[package.optional-dependencies] +http = [ + { name = "aiohttp" }, ] [[package]] @@ -3711,19 +3740,18 @@ wheels = [ [[package]] name = "multiprocess" -version = "0.70.18" +version = "0.70.16" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dill" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/fd/2ae3826f5be24c6ed87266bc4e59c46ea5b059a103f3d7e7eb76a52aeecb/multiprocess-0.70.18.tar.gz", hash = "sha256:f9597128e6b3e67b23956da07cf3d2e5cba79e2f4e0fba8d7903636663ec6d0d", size = 1798503 } +sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/d8/0cba6cf51a1a31f20471fbc823a716170c73012ddc4fb85d706630ed6e8f/multiprocess-0.70.18-py310-none-any.whl", hash = "sha256:60c194974c31784019c1f459d984e8f33ee48f10fcf42c309ba97b30d9bd53ea", size = 134948 }, - { url = "https://files.pythonhosted.org/packages/4b/88/9039f2fed1012ef584751d4ceff9ab4a51e5ae264898f0b7cbf44340a859/multiprocess-0.70.18-py311-none-any.whl", hash = "sha256:5aa6eef98e691281b3ad923be2832bf1c55dd2c859acd73e5ec53a66aae06a1d", size = 144462 }, - { url = "https://files.pythonhosted.org/packages/bf/b6/5f922792be93b82ec6b5f270bbb1ef031fd0622847070bbcf9da816502cc/multiprocess-0.70.18-py312-none-any.whl", hash = "sha256:9b78f8e5024b573730bfb654783a13800c2c0f2dfc0c25e70b40d184d64adaa2", size = 150287 }, - { url = "https://files.pythonhosted.org/packages/ee/25/7d7e78e750bc1aecfaf0efbf826c69a791d2eeaf29cf20cba93ff4cced78/multiprocess-0.70.18-py313-none-any.whl", hash = "sha256:871743755f43ef57d7910a38433cfe41319e72be1bbd90b79c7a5ac523eb9334", size = 151917 }, - { url = "https://files.pythonhosted.org/packages/3b/c3/ca84c19bd14cdfc21c388fdcebf08b86a7a470ebc9f5c3c084fc2dbc50f7/multiprocess-0.70.18-py38-none-any.whl", hash = "sha256:dbf705e52a154fe5e90fb17b38f02556169557c2dd8bb084f2e06c2784d8279b", size = 132636 }, - { url = "https://files.pythonhosted.org/packages/6c/28/dd72947e59a6a8c856448a5e74da6201cb5502ddff644fbc790e4bd40b9a/multiprocess-0.70.18-py39-none-any.whl", hash = "sha256:e78ca805a72b1b810c690b6b4cc32579eba34f403094bbbae962b7b5bf9dfcb8", size = 133478 }, + { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824 }, + { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519 }, + { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741 }, + { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628 }, + { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351 }, ] [[package]] @@ -4931,6 +4959,34 @@ bcrypt = [ { name = "bcrypt" }, ] +[[package]] +name = "pyarrow" +version = "18.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7f/7b/640785a9062bb00314caa8a387abce547d2a420cf09bd6c715fe659ccffb/pyarrow-18.1.0.tar.gz", hash = "sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73", size = 1118671 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/50/12829e7111b932581e51dda51d5cb39207a056c30fe31ef43f14c63c4d7e/pyarrow-18.1.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d", size = 29514620 }, + { url = "https://files.pythonhosted.org/packages/d1/41/468c944eab157702e96abab3d07b48b8424927d4933541ab43788bb6964d/pyarrow-18.1.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee", size = 30856494 }, + { url = "https://files.pythonhosted.org/packages/68/f9/29fb659b390312a7345aeb858a9d9c157552a8852522f2c8bad437c29c0a/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992", size = 39203624 }, + { url = "https://files.pythonhosted.org/packages/6e/f6/19360dae44200e35753c5c2889dc478154cd78e61b1f738514c9f131734d/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54", size = 40139341 }, + { url = "https://files.pythonhosted.org/packages/bb/e6/9b3afbbcf10cc724312e824af94a2e993d8ace22994d823f5c35324cebf5/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33", size = 38618629 }, + { url = "https://files.pythonhosted.org/packages/3a/2e/3b99f8a3d9e0ccae0e961978a0d0089b25fb46ebbcfb5ebae3cca179a5b3/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30", size = 40078661 }, + { url = "https://files.pythonhosted.org/packages/76/52/f8da04195000099d394012b8d42c503d7041b79f778d854f410e5f05049a/pyarrow-18.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99", size = 25092330 }, + { url = "https://files.pythonhosted.org/packages/cb/87/aa4d249732edef6ad88899399047d7e49311a55749d3c373007d034ee471/pyarrow-18.1.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b", size = 29497406 }, + { url = "https://files.pythonhosted.org/packages/3c/c7/ed6adb46d93a3177540e228b5ca30d99fc8ea3b13bdb88b6f8b6467e2cb7/pyarrow-18.1.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2", size = 30835095 }, + { url = "https://files.pythonhosted.org/packages/41/d7/ed85001edfb96200ff606943cff71d64f91926ab42828676c0fc0db98963/pyarrow-18.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191", size = 39194527 }, + { url = "https://files.pythonhosted.org/packages/59/16/35e28eab126342fa391593415d79477e89582de411bb95232f28b131a769/pyarrow-18.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa", size = 40131443 }, + { url = "https://files.pythonhosted.org/packages/0c/95/e855880614c8da20f4cd74fa85d7268c725cf0013dc754048593a38896a0/pyarrow-18.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c", size = 38608750 }, + { url = "https://files.pythonhosted.org/packages/54/9d/f253554b1457d4fdb3831b7bd5f8f00f1795585a606eabf6fec0a58a9c38/pyarrow-18.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c", size = 40066690 }, + { url = "https://files.pythonhosted.org/packages/2f/58/8912a2563e6b8273e8aa7b605a345bba5a06204549826f6493065575ebc0/pyarrow-18.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181", size = 25081054 }, + { url = "https://files.pythonhosted.org/packages/82/f9/d06ddc06cab1ada0c2f2fd205ac8c25c2701182de1b9c4bf7a0a44844431/pyarrow-18.1.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc", size = 29525542 }, + { url = "https://files.pythonhosted.org/packages/ab/94/8917e3b961810587ecbdaa417f8ebac0abb25105ae667b7aa11c05876976/pyarrow-18.1.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386", size = 30829412 }, + { url = "https://files.pythonhosted.org/packages/5e/e3/3b16c3190f3d71d3b10f6758d2d5f7779ef008c4fd367cedab3ed178a9f7/pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324", size = 39119106 }, + { url = "https://files.pythonhosted.org/packages/1d/d6/5d704b0d25c3c79532f8c0639f253ec2803b897100f64bcb3f53ced236e5/pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8", size = 40090940 }, + { url = "https://files.pythonhosted.org/packages/37/29/366bc7e588220d74ec00e497ac6710c2833c9176f0372fe0286929b2d64c/pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9", size = 38548177 }, + { url = "https://files.pythonhosted.org/packages/c8/11/fabf6ecabb1fe5b7d96889228ca2a9158c4c3bb732e3b8ee3f7f6d40b703/pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba", size = 40043567 }, +] + [[package]] name = "pyasn1" version = "0.6.1" @@ -6353,7 +6409,7 @@ wheels = [ [[package]] name = "surf-new-backend" -version = "0.0.8" +version = "0.0.9" source = { virtual = "." } dependencies = [ { name = "alembic" }, @@ -6361,6 +6417,7 @@ dependencies = [ { name = "boto3" }, { name = "celery", extra = ["redis"] }, { name = "chonkie", extra = ["all"] }, + { name = "datasets" }, { name = "deepagents" }, { name = "discord-py" }, { name = "docling" }, @@ -6391,6 +6448,7 @@ dependencies = [ { name = "pgvector" }, { name = "playwright" }, { name = "psycopg", extra = ["binary", "pool"] }, + { name = "pyarrow" }, { name = "pypdf" }, { name = "python-ffmpeg" }, { name = "redis" }, @@ -6421,6 +6479,7 @@ requires-dist = [ { name = "boto3", specifier = ">=1.35.0" }, { name = "celery", extras = ["redis"], specifier = ">=5.5.3" }, { name = "chonkie", extras = ["all"], specifier = ">=1.5.0" }, + { name = "datasets", specifier = ">=2.21.0" }, { name = "deepagents", specifier = ">=0.3.0" }, { name = "discord-py", specifier = ">=2.5.2" }, { name = "docling", specifier = ">=2.15.0" }, @@ -6451,6 +6510,7 @@ requires-dist = [ { name = "pgvector", specifier = ">=0.3.6" }, { name = "playwright", specifier = ">=1.50.0" }, { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.3.2" }, + { name = "pyarrow", specifier = ">=15.0.0,<19.0.0" }, { name = "pypdf", specifier = ">=5.1.0" }, { name = "python-ffmpeg", specifier = ">=2.0.12" }, { name = "redis", specifier = ">=5.2.1" }, diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index bbbfd61e0..5854cb706 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -1,6 +1,7 @@ "use client"; import { format } from "date-fns"; +import { useAtomValue } from "jotai"; import { Calendar as CalendarIcon, Clock, @@ -18,6 +19,12 @@ import { useParams, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useEffect, useState } from "react"; import { toast } from "sonner"; +import { + deleteConnectorMutationAtom, + indexConnectorMutationAtom, + updateConnectorMutationAtom, +} from "@/atoms/connectors/connector-mutation.atoms"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { AlertDialog, AlertDialogAction, @@ -62,7 +69,6 @@ import { import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; import { cn } from "@/lib/utils"; import { authenticatedFetch } from "@/lib/auth-utils"; import { GoogleDriveFolderTree } from "@/components/connectors/google-drive-folder-tree"; @@ -94,8 +100,12 @@ export default function ConnectorsPage() { const searchSpaceId = params.search_space_id as string; const today = new Date(); - const { connectors, isLoading, error, deleteConnector, indexConnector, updateConnector } = - useSearchSourceConnectors(false, parseInt(searchSpaceId)); + const { data: connectors = [], isLoading, error } = useAtomValue(connectorsAtom); + + const { mutateAsync: deleteConnector } = useAtomValue(deleteConnectorMutationAtom); + const { mutateAsync: indexConnector } = useAtomValue(indexConnectorMutationAtom); + const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom); + const [connectorToDelete, setConnectorToDelete] = useState(null); const [indexingConnectorId, setIndexingConnectorId] = useState(null); const [datePickerOpen, setDatePickerOpen] = useState(false); @@ -133,11 +143,9 @@ export default function ConnectorsPage() { if (connectorToDelete === null) return; try { - await deleteConnector(connectorToDelete); - toast.success(t("delete_success")); + await deleteConnector({ id: connectorToDelete }); } catch (error) { console.error("Error deleting connector:", error); - toast.error(t("delete_failed")); } finally { setConnectorToDelete(null); } @@ -228,7 +236,14 @@ export default function ConnectorsPage() { const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined; - await indexConnector(selectedConnectorForIndexing, searchSpaceId, startDateStr, endDateStr); + await indexConnector({ + connector_id: selectedConnectorForIndexing, + queryParams: { + search_space_id: searchSpaceId, + start_date: startDateStr, + end_date: endDateStr, + }, + }); toast.success(t("indexing_started")); } catch (error) { console.error("Error indexing connector content:", error); @@ -245,7 +260,12 @@ export default function ConnectorsPage() { const handleQuickIndexConnector = async (connectorId: number) => { setIndexingConnectorId(connectorId); try { - await indexConnector(connectorId, searchSpaceId); + await indexConnector({ + connector_id: connectorId, + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success(t("indexing_started")); } catch (error) { console.error("Error indexing connector content:", error); @@ -305,9 +325,12 @@ export default function ConnectorsPage() { } } - await updateConnector(selectedConnectorForPeriodic, { - periodic_indexing_enabled: periodicEnabled, - indexing_frequency_minutes: frequency, + await updateConnector({ + id: selectedConnectorForPeriodic, + data: { + periodic_indexing_enabled: periodicEnabled, + indexing_frequency_minutes: frequency, + }, }); toast.success( diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx index 1346c1cd0..252a8e1d2 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,8 @@ import { useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; @@ -21,10 +24,8 @@ import { FormMessage, } from "@/components/ui/form"; import { Input } from "@/components/ui/input"; -import { - type SearchSourceConnector, - useSearchSourceConnectors, -} from "@/hooks/use-search-source-connectors"; +import type { EnumConnectorName } from "@/contracts/enums/connector"; +import type { SearchSourceConnector } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const apiConnectorFormSchema = z.object({ @@ -85,7 +86,8 @@ export default function EditConnectorPage() { const searchSpaceId = params.search_space_id as string; const connectorId = parseInt(params.connector_id as string, 10); - const { connectors, updateConnector } = useSearchSourceConnectors(false, parseInt(searchSpaceId)); + const { data: connectors = [] } = useAtomValue(connectorsAtom); + const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom); const [connector, setConnector] = useState(null); const [isLoading, setIsLoading] = useState(true); const [isSubmitting, setIsSubmitting] = useState(false); @@ -99,14 +101,12 @@ export default function EditConnectorPage() { }, }); - // Find connector in the list useEffect(() => { const currentConnector = connectors.find((c) => c.id === connectorId); if (currentConnector) { setConnector(currentConnector); - // Check if connector type is supported const apiKeyField = getApiKeyFieldName(currentConnector.connector_type); if (apiKeyField) { form.reset({ @@ -114,14 +114,12 @@ export default function EditConnectorPage() { api_key: currentConnector.config[apiKeyField] || "", }); } else { - // Redirect if not a supported connector type toast.error("This connector type is not supported for editing"); router.push(`/dashboard/${searchSpaceId}/connectors`); } setIsLoading(false); } else if (!isLoading && connectors.length > 0) { - // If connectors are loaded but this one isn't found toast.error("Connector not found"); router.push(`/dashboard/${searchSpaceId}/connectors`); } @@ -135,18 +133,20 @@ export default function EditConnectorPage() { try { const apiKeyField = getApiKeyFieldName(connector.connector_type); - // Only update the API key if a new one was provided const updatedConfig = { ...connector.config }; if (values.api_key) { updatedConfig[apiKeyField] = values.api_key; } - await updateConnector(connectorId, { - name: values.name, - connector_type: connector.connector_type, - config: updatedConfig, - is_indexable: connector.is_indexable, - last_indexed_at: connector.last_indexed_at, + await updateConnector({ + id: connectorId, + data: { + name: values.name, + connector_type: connector.connector_type as EnumConnectorName, + config: updatedConfig, + is_indexable: connector.is_indexable, + last_indexed_at: connector.last_indexed_at, + }, }); toast.success("Connector updated successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx index cc4330203..639a9cb95 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx @@ -1,11 +1,13 @@ "use client"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, ExternalLink, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import Link from "next/link"; import { useParams, useRouter } from "next/navigation"; import { useEffect, useState } from "react"; import { toast } from "sonner"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { Button } from "@/components/ui/button"; import { Card, @@ -18,10 +20,7 @@ import { import { EnumConnectorName } from "@/contracts/enums/connector"; // import { IconBrandAirtable } from "@tabler/icons-react"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { - type SearchSourceConnector, - useSearchSourceConnectors, -} from "@/hooks/use-search-source-connectors"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { authenticatedFetch } from "@/lib/auth-utils"; export default function AirtableConnectorPage() { @@ -31,11 +30,12 @@ export default function AirtableConnectorPage() { const [isConnecting, setIsConnecting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId)); + const { refetch: fetchConnectors } = useAtomValue(connectorsAtom); useEffect(() => { - fetchConnectors(parseInt(searchSpaceId)).then((data) => { - const connector = data.find( + fetchConnectors().then((data) => { + const connectors = data.data || []; + const connector = connectors.find( (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.AIRTABLE_CONNECTOR ); if (connector) { diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/baidu-search-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/baidu-search-api/page.tsx index 3e9f4898e..204925d26 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/baidu-search-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/baidu-search-api/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { @@ -38,7 +40,6 @@ import { import { Switch } from "@/components/ui/switch"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const baiduSearchApiFormSchema = z.object({ @@ -61,7 +62,7 @@ export default function BaiduSearchApiPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -95,8 +96,8 @@ export default function BaiduSearchApiPage() { config.BAIDU_ENABLE_DEEP_SEARCH = values.enable_deep_search; } - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.BAIDU_SEARCH_API, config, @@ -106,8 +107,10 @@ export default function BaiduSearchApiPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Baidu Search connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/bookstack-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/bookstack-connector/page.tsx index 576fa4ebc..3fa634238 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/bookstack-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/bookstack-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; @@ -24,7 +26,6 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const bookstackConnectorFormSchema = z.object({ @@ -50,7 +51,7 @@ export default function BookStackConnectorPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -67,8 +68,8 @@ export default function BookStackConnectorPage() { const onSubmit = async (values: BookStackConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.BOOKSTACK_CONNECTOR, config: { @@ -82,8 +83,10 @@ export default function BookStackConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("BookStack connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx index 2d5f6954c..319d333b4 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, ExternalLink, Eye, EyeOff } from "lucide-react"; import Link from "next/link"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { @@ -22,7 +24,6 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const clickupConnectorFormSchema = z.object({ @@ -41,7 +42,7 @@ export default function ClickUpConnectorPage() { const router = useRouter(); const params = useParams(); const searchSpaceId = params.search_space_id as string; - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); const [isLoading, setIsLoading] = useState(false); const [showApiToken, setShowApiToken] = useState(false); @@ -59,20 +60,23 @@ export default function ClickUpConnectorPage() { setIsLoading(true); try { - const connectorData = { - name: values.name, - connector_type: EnumConnectorName.CLICKUP_CONNECTOR, - is_indexable: true, - config: { - CLICKUP_API_TOKEN: values.api_token, + await createConnector({ + data: { + name: values.name, + connector_type: EnumConnectorName.CLICKUP_CONNECTOR, + is_indexable: true, + config: { + CLICKUP_API_TOKEN: values.api_token, + }, + last_indexed_at: null, + periodic_indexing_enabled: false, + indexing_frequency_minutes: null, + next_scheduled_at: null, }, - last_indexed_at: null, - periodic_indexing_enabled: false, - indexing_frequency_minutes: null, - next_scheduled_at: null, - }; - - await createConnector(connectorData, parseInt(searchSpaceId)); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("ClickUp connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx index c625f8900..7fcd03062 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; @@ -24,7 +26,6 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const confluenceConnectorFormSchema = z.object({ @@ -60,7 +61,7 @@ export default function ConfluenceConnectorPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -77,8 +78,8 @@ export default function ConfluenceConnectorPage() { const onSubmit = async (values: ConfluenceConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.CONFLUENCE_CONNECTOR, config: { @@ -92,8 +93,10 @@ export default function ConfluenceConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Confluence connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx index 1daa6bcd0..94ef849e2 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Accordion, AccordionContent, @@ -37,7 +39,6 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const discordConnectorFormSchema = z.object({ @@ -58,7 +59,7 @@ export default function DiscordConnectorPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -73,8 +74,8 @@ export default function DiscordConnectorPage() { const onSubmit = async (values: DiscordConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.DISCORD_CONNECTOR, config: { @@ -86,8 +87,10 @@ export default function DiscordConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Discord connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx index e417995ed..15e03f3aa 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx @@ -2,6 +2,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import * as RadioGroup from "@radix-ui/react-radio-group"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter, useSearchParams } from "next/navigation"; @@ -9,7 +10,7 @@ import { useId, useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; - +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Accordion, AccordionContent, @@ -40,10 +41,8 @@ import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Separator } from "@/components/ui/separator"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; - import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const elasticsearchConnectorFormSchema = z @@ -91,7 +90,7 @@ export default function ElasticsearchConnectorPage() { const authBasicId = useId(); const authApiKeyId = useId(); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -173,19 +172,21 @@ export default function ElasticsearchConnectorPage() { config.ELASTICSEARCH_MAX_DOCUMENTS = values.max_documents; } - const connectorPayload = { - name: values.name, - connector_type: EnumConnectorName.ELASTICSEARCH_CONNECTOR, - is_indexable: true, - last_indexed_at: null, - periodic_indexing_enabled: false, - indexing_frequency_minutes: null, - next_scheduled_at: null, - config, - }; - - // Use existing hook method - await createConnector(connectorPayload, searchSpaceIdNum); + await createConnector({ + data: { + name: values.name, + connector_type: EnumConnectorName.ELASTICSEARCH_CONNECTOR, + is_indexable: true, + last_indexed_at: null, + periodic_indexing_enabled: false, + indexing_frequency_minutes: null, + next_scheduled_at: null, + config, + }, + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Elasticsearch connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx index 833d716a8..a6c0c147b 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, CircleAlert, Github, Info, ListChecks, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Accordion, AccordionContent, @@ -38,8 +40,6 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -// Assuming useSearchSourceConnectors hook exists and works similarly -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; import { authenticatedFetch, redirectToLogin } from "@/lib/auth-utils"; // Define the form schema with Zod for GitHub PAT entry step @@ -85,7 +85,7 @@ export default function GithubConnectorPage() { const [connectorName, setConnectorName] = useState("GitHub Connector"); const [validatedPat, setValidatedPat] = useState(""); // Store the validated PAT - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form for PAT entry const form = useForm({ @@ -141,8 +141,8 @@ export default function GithubConnectorPage() { setIsCreatingConnector(true); try { - await createConnector( - { + await createConnector({ + data: { name: connectorName, // Use the stored name connector_type: EnumConnectorName.GITHUB_CONNECTOR, config: { @@ -155,8 +155,10 @@ export default function GithubConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("GitHub connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx index 8179fbabc..d208b1659 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, ExternalLink, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import Link from "next/link"; @@ -9,6 +10,7 @@ import { useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import { z } from "zod"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { Button } from "@/components/ui/button"; import { Card, @@ -20,10 +22,7 @@ import { } from "@/components/ui/card"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { - type SearchSourceConnector, - useSearchSourceConnectors, -} from "@/hooks/use-search-source-connectors"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { authenticatedFetch } from "@/lib/auth-utils"; export default function GoogleCalendarConnectorPage() { @@ -33,11 +32,12 @@ export default function GoogleCalendarConnectorPage() { const [isConnecting, setIsConnecting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId)); + const { refetch: fetchConnectors } = useAtomValue(connectorsAtom); useEffect(() => { - fetchConnectors(parseInt(searchSpaceId)).then((data) => { - const connector = data.find( + fetchConnectors().then((data) => { + const connectors = data.data || []; + const connector = connectors.find( (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx index 8659d937c..5ca8874d1 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, ExternalLink, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import Link from "next/link"; @@ -9,6 +10,7 @@ import { useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import { z } from "zod"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { Button } from "@/components/ui/button"; import { Card, @@ -20,10 +22,7 @@ import { } from "@/components/ui/card"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { - type SearchSourceConnector, - useSearchSourceConnectors, -} from "@/hooks/use-search-source-connectors"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { authenticatedFetch } from "@/lib/auth-utils"; export default function GoogleGmailConnectorPage() { @@ -33,11 +32,12 @@ export default function GoogleGmailConnectorPage() { const [isConnecting, setIsConnecting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId)); + const { refetch: fetchConnectors } = useAtomValue(connectorsAtom); useEffect(() => { - fetchConnectors(parseInt(searchSpaceId)).then((data) => { - const connector = data.find( + fetchConnectors().then((data) => { + const connectors = data.data || []; + const connector = connectors.find( (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.GOOGLE_GMAIL_CONNECTOR ); if (connector) { diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx index 6f4e31114..a0c22582b 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Accordion, AccordionContent, @@ -37,7 +39,6 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const jiraConnectorFormSchema = z.object({ @@ -73,7 +74,7 @@ export default function JiraConnectorPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -90,8 +91,8 @@ export default function JiraConnectorPage() { const onSubmit = async (values: JiraConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.JIRA_CONNECTOR, config: { @@ -105,8 +106,10 @@ export default function JiraConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Jira connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx index 13df9a910..ba747bf6e 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Accordion, AccordionContent, @@ -37,7 +39,6 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const linearConnectorFormSchema = z.object({ @@ -62,7 +63,7 @@ export default function LinearConnectorPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -77,8 +78,8 @@ export default function LinearConnectorPage() { const onSubmit = async (values: LinearConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.LINEAR_CONNECTOR, config: { @@ -90,8 +91,10 @@ export default function LinearConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Linear connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx index c20c2b576..16c0700d7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { @@ -30,7 +32,6 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const linkupApiFormSchema = z.object({ @@ -50,7 +51,7 @@ export default function LinkupApiPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -65,8 +66,8 @@ export default function LinkupApiPage() { const onSubmit = async (values: LinkupApiFormValues) => { setIsSubmitting(true); try { - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.LINKUP_API, config: { @@ -78,8 +79,10 @@ export default function LinkupApiPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Linkup API connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx index 7d4b82b68..0b78cf40c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Key, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import Link from "next/link"; @@ -9,6 +10,8 @@ import { useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { Button } from "@/components/ui/button"; import { Card, @@ -30,10 +33,7 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { - type SearchSourceConnector, - useSearchSourceConnectors, -} from "@/hooks/use-search-source-connectors"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; // Define the form schema with Zod const lumaConnectorFormSchema = z.object({ @@ -55,10 +55,8 @@ export default function LumaConnectorPage() { const [isSubmitting, setIsSubmitting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors, createConnector } = useSearchSourceConnectors( - true, - parseInt(searchSpaceId) - ); + const { data: connectors } = useAtomValue(connectorsAtom); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -69,29 +67,26 @@ export default function LumaConnectorPage() { }, }); + const { refetch: fetchConnectors } = useAtomValue(connectorsAtom); + useEffect(() => { - fetchConnectors(parseInt(searchSpaceId)) - .then((data) => { - if (data && Array.isArray(data)) { - const connector = data.find( - (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.LUMA_CONNECTOR - ); - if (connector) { - setDoesConnectorExist(true); - } - } - }) - .catch((error) => { - console.error("Error fetching connectors:", error); - }); - }, [fetchConnectors, searchSpaceId]); + fetchConnectors().then((data) => { + const connectors = data.data || []; + const connector = connectors.find( + (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.LUMA_CONNECTOR + ); + if (connector) { + setDoesConnectorExist(true); + } + }); + }, []); // Handle form submission const onSubmit = async (values: LumaConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.LUMA_CONNECTOR, config: { @@ -103,8 +98,10 @@ export default function LumaConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Luma connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx index 310c31811..160808e21 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Accordion, AccordionContent, @@ -37,7 +39,6 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const notionConnectorFormSchema = z.object({ @@ -57,7 +58,7 @@ export default function NotionConnectorPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -72,8 +73,8 @@ export default function NotionConnectorPage() { const onSubmit = async (values: NotionConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.NOTION_CONNECTOR, config: { @@ -85,8 +86,10 @@ export default function NotionConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Notion connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx index 9645a3657..aec72bbc7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { @@ -31,7 +33,6 @@ import { Input } from "@/components/ui/input"; import { Switch } from "@/components/ui/switch"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; const searxngFormSchema = z.object({ name: z.string().min(3, { @@ -67,7 +68,7 @@ export default function SearxngConnectorPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); const form = useForm({ resolver: zodResolver(searxngFormSchema), @@ -115,8 +116,8 @@ export default function SearxngConnectorPage() { config.SEARXNG_VERIFY_SSL = false; } - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.SEARXNG_API, config, @@ -126,8 +127,10 @@ export default function SearxngConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("SearxNG connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx index 314ed8442..f3cf2ca6c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Accordion, AccordionContent, @@ -37,7 +39,6 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const slackConnectorFormSchema = z.object({ @@ -57,7 +58,7 @@ export default function SlackConnectorPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -72,8 +73,8 @@ export default function SlackConnectorPage() { const onSubmit = async (values: SlackConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.SLACK_CONNECTOR, config: { @@ -85,8 +86,10 @@ export default function SlackConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Slack connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx index 40b98ca5c..a9ddd5a41 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; @@ -8,6 +9,7 @@ import { useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { @@ -30,7 +32,6 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const tavilyApiFormSchema = z.object({ @@ -50,7 +51,7 @@ export default function TavilyApiPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -65,8 +66,8 @@ export default function TavilyApiPage() { const onSubmit = async (values: TavilyApiFormValues) => { setIsSubmitting(true); try { - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.TAVILY_API, config: { @@ -78,8 +79,10 @@ export default function TavilyApiPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Tavily API connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/webcrawler-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/webcrawler-connector/page.tsx index 8edc34728..0b7e811d0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/webcrawler-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/webcrawler-connector/page.tsx @@ -1,6 +1,7 @@ "use client"; import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { ArrowLeft, Check, Globe, Loader2 } from "lucide-react"; import { motion } from "motion/react"; import Link from "next/link"; @@ -9,6 +10,8 @@ import { useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import * as z from "zod"; +import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { Button } from "@/components/ui/button"; import { Card, @@ -31,10 +34,7 @@ import { Input } from "@/components/ui/input"; import { Textarea } from "@/components/ui/textarea"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { - type SearchSourceConnector, - useSearchSourceConnectors, -} from "@/hooks/use-search-source-connectors"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; // Define the form schema with Zod const webcrawlerConnectorFormSchema = z.object({ @@ -55,10 +55,8 @@ export default function WebcrawlerConnectorPage() { const [isSubmitting, setIsSubmitting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors, createConnector } = useSearchSourceConnectors( - true, - parseInt(searchSpaceId) - ); + const { refetch: fetchConnectors } = useAtomValue(connectorsAtom); + const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); // Initialize the form const form = useForm({ @@ -71,22 +69,16 @@ export default function WebcrawlerConnectorPage() { }); useEffect(() => { - fetchConnectors(parseInt(searchSpaceId)) - .then((data) => { - if (data && Array.isArray(data)) { - const connector = data.find( - (c: SearchSourceConnector) => - c.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR - ); - if (connector) { - setDoesConnectorExist(true); - } - } - }) - .catch((error) => { - console.error("Error fetching connectors:", error); - }); - }, [fetchConnectors, searchSpaceId]); + fetchConnectors().then((data) => { + const connectors = data.data || []; + const connector = connectors.find( + (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR + ); + if (connector) { + setDoesConnectorExist(true); + } + }); + }, []); // Handle form submission const onSubmit = async (values: WebcrawlerConnectorFormValues) => { @@ -104,8 +96,8 @@ export default function WebcrawlerConnectorPage() { config.INITIAL_URLS = values.initial_urls; } - await createConnector( - { + await createConnector({ + data: { name: values.name, connector_type: EnumConnectorName.WEBCRAWLER_CONNECTOR, config: config, @@ -115,8 +107,10 @@ export default function WebcrawlerConnectorPage() { indexing_frequency_minutes: null, next_scheduled_at: null, }, - parseInt(searchSpaceId) - ); + queryParams: { + search_space_id: searchSpaceId, + }, + }); toast.success("Webcrawler connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/ProcessingIndicator.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/ProcessingIndicator.tsx new file mode 100644 index 000000000..bd53bab18 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/ProcessingIndicator.tsx @@ -0,0 +1,43 @@ +"use client"; + +import { Loader2 } from "lucide-react"; +import { AnimatePresence, motion } from "motion/react"; +import { useTranslations } from "next-intl"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; + +interface ProcessingIndicatorProps { + activeTasksCount: number; +} + +export function ProcessingIndicator({ activeTasksCount }: ProcessingIndicatorProps) { + const t = useTranslations("documents"); + + if (activeTasksCount === 0) return null; + + return ( + + + +
+
+ +
+
+ + {t("processing_documents")} + + + {t("active_tasks_count", { count: activeTasksCount })} + +
+
+
+
+
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 065267b2e..edfda7dbf 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -2,19 +2,23 @@ import { useQuery } from "@tanstack/react-query"; import { useAtomValue } from "jotai"; +import { RefreshCw } from "lucide-react"; import { motion } from "motion/react"; import { useParams } from "next/navigation"; import { useTranslations } from "next-intl"; -import { useCallback, useEffect, useId, useMemo, useState } from "react"; +import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms"; import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms"; +import { Button } from "@/components/ui/button"; import type { DocumentTypeEnum } from "@/contracts/types/document.types"; +import { useLogsSummary } from "@/hooks/use-logs"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import { cacheKeys } from "@/lib/query-client/cache-keys"; import { DocumentsFilters } from "./components/DocumentsFilters"; import { DocumentsTableShell, type SortKey } from "./components/DocumentsTableShell"; import { PaginationControls } from "./components/PaginationControls"; +import { ProcessingIndicator } from "./components/ProcessingIndicator"; import type { ColumnVisibility } from "./components/types"; function useDebounced(value: T, delay = 250) { @@ -127,7 +131,22 @@ export default function DocumentsTable() { } else { await refetchDocuments(); } - }, [debouncedSearch, refetchSearch, refetchDocuments]); + toast.success(t("refresh_success") || "Documents refreshed"); + }, [debouncedSearch, refetchSearch, refetchDocuments, t]); + + // Set up polling for active tasks + const { summary } = useLogsSummary(searchSpaceId, 24, { refetchInterval: 5000 }); + const activeTasksCount = summary?.active_tasks.length || 0; + const prevActiveTasksCount = useRef(activeTasksCount); + + // Auto-refresh when a task finishes + useEffect(() => { + if (prevActiveTasksCount.current > activeTasksCount) { + // A task has finished! + refreshCurrentView(); + } + prevActiveTasksCount.current = activeTasksCount; + }, [activeTasksCount, refreshCurrentView]); // Create a delete function for single document deletion const deleteDocument = useCallback( @@ -189,8 +208,26 @@ export default function DocumentsTable() { initial={{ opacity: 0, y: 20 }} animate={{ opacity: 1, y: 0 }} transition={{ duration: 0.3 }} - className="w-full px-6 py-4 min-h-[calc(100vh-64px)]" + className="w-full px-6 py-4 space-y-6 min-h-[calc(100vh-64px)]" > + +
+

{t("title")}

+

{t("subtitle")}

+
+ +
+ + + { + try { + await createLogMutation(data); + return true; + } catch (error) { + console.error("Failed to create log:", error); + return false; + } + }, + [createLogMutation] + ); + + const updateLog = useCallback( + async (logId: number, data: UpdateLogRequest) => { + try { + await updateLogMutation({ logId, data }); + return true; + } catch (error) { + console.error("Failed to update log:", error); + return false; + } + }, + [updateLogMutation] + ); + + const deleteLog = useCallback( + async (id: number) => { + try { + await deleteLogMutation({ id }); + return true; + } catch (error) { + console.error("Failed to delete log:", error); + return false; + } + }, + [deleteLogMutation] + ); + + const { logs, loading: logsLoading, error: logsError, refreshLogs } = useLogs(searchSpaceId); const { summary, loading: summaryLoading, @@ -408,7 +452,7 @@ export default function LogsManagePage() { return; } - const deletePromises = selectedRows.map((row) => deleteLog(row.original.id)); + const deletePromises = selectedRows.map((row) => deleteLog(row.original.id)); // Already passes { id } via wrapper try { const results = await Promise.all(deletePromises); @@ -437,7 +481,7 @@ export default function LogsManagePage() { Promise.resolve(false)), - refreshLogs: refreshLogs || (() => Promise.resolve()), + refreshLogs: () => refreshLogs().then(() => void 0), }} > ; + +/** + * Extract persisted attachments from message content (type-safe with Zod) + */ +function extractPersistedAttachments(content: unknown): PersistedAttachment[] { + if (!Array.isArray(content)) return []; + + for (const part of content) { + const result = AttachmentsPartSchema.safeParse(part); + if (result.success) { + return result.data.items; + } + } + + return []; +} + /** * Convert backend message to assistant-ui ThreadMessageLike format * Filters out 'thinking-steps' part as it's handled separately via messageThinkingSteps + * Restores attachments for user messages from persisted data */ function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike { let content: ThreadMessageLike["content"]; @@ -105,8 +148,12 @@ function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike { const filteredContent = msg.content.filter((part: unknown) => { if (typeof part !== "object" || part === null || !("type" in part)) return true; const partType = (part as { type: string }).type; - // Filter out thinking-steps and mentioned-documents - return partType !== "thinking-steps" && partType !== "mentioned-documents"; + // Filter out thinking-steps, mentioned-documents, and attachments + return ( + partType !== "thinking-steps" && + partType !== "mentioned-documents" && + partType !== "attachments" + ); }); content = filteredContent.length > 0 @@ -116,11 +163,31 @@ function convertToThreadMessage(msg: MessageRecord): ThreadMessageLike { content = [{ type: "text", text: String(msg.content) }]; } + // Restore attachments for user messages + let attachments: ThreadMessageLike["attachments"]; + if (msg.role === "user") { + const persistedAttachments = extractPersistedAttachments(msg.content); + if (persistedAttachments.length > 0) { + attachments = persistedAttachments.map((att) => ({ + id: att.id, + name: att.name, + type: att.type as "document" | "image" | "file", + contentType: att.contentType || "application/octet-stream", + status: { type: "complete" as const }, + content: [], + // Custom fields for our ChatAttachment interface + imageDataUrl: att.imageDataUrl, + extractedContent: att.extractedContent, + })); + } + } + return { id: `msg-${msg.id}`, role: msg.role, content, createdAt: new Date(msg.created_at), + attachments, }; } @@ -132,6 +199,7 @@ const TOOLS_WITH_UI = new Set([ "link_preview", "display_image", "scrape_webpage", + "write_todos", ]); /** @@ -146,6 +214,7 @@ interface ThinkingStepData { export default function NewChatPage() { const params = useParams(); + const queryClient = useQueryClient(); const [isInitializing, setIsInitializing] = useState(true); const [threadId, setThreadId] = useState(null); const [messages, setMessages] = useState([]); @@ -163,6 +232,7 @@ export default function NewChatPage() { const setMentionedDocumentIds = useSetAtom(mentionedDocumentIdsAtom); const setMentionedDocuments = useSetAtom(mentionedDocumentsAtom); const setMessageDocumentsMap = useSetAtom(messageDocumentsMapAtom); + const hydratePlanState = useSetAtom(hydratePlanStateAtom); // Create the attachment adapter for file processing const attachmentAdapter = useMemo(() => createAttachmentAdapter(), []); @@ -198,6 +268,7 @@ export default function NewChatPage() { setMentionedDocumentIds([]); setMentionedDocuments([]); setMessageDocumentsMap({}); + clearPlanOwnerRegistry(); // Reset plan ownership for new chat try { if (urlChatId > 0) { @@ -219,6 +290,11 @@ export default function NewChatPage() { if (steps.length > 0) { restoredThinkingSteps.set(`msg-${msg.id}`, steps); } + // Hydrate write_todos plan state from persisted tool calls + const writeTodosCalls = extractWriteTodosFromContent(msg.content); + for (const todoData of writeTodosCalls) { + hydratePlanState(todoData); + } } if (msg.role === "user") { const docs = extractMentionedDocuments(msg.content); @@ -247,7 +323,13 @@ export default function NewChatPage() { } finally { setIsInitializing(false); } - }, [urlChatId, setMessageDocumentsMap, setMentionedDocumentIds, setMentionedDocuments]); + }, [ + urlChatId, + setMessageDocumentsMap, + setMentionedDocumentIds, + setMentionedDocuments, + hydratePlanState, + ]); // Initialize on mount useEffect(() => { @@ -306,6 +388,7 @@ export default function NewChatPage() { // Lazy thread creation: create thread on first message if it doesn't exist let currentThreadId = threadId; + let isNewThread = false; if (!currentThreadId) { try { const newThread = await createThread(searchSpaceId, "New Chat"); @@ -315,6 +398,7 @@ export default function NewChatPage() { // Track chat creation trackChatCreated(searchSpaceId, currentThreadId); + isNewThread = true; // Update URL silently using browser API (not router.replace) to avoid // interrupting the ongoing fetch/streaming with React navigation window.history.replaceState( @@ -361,25 +445,50 @@ export default function NewChatPage() { })); } - // Persist user message with mentioned documents (don't await, fire and forget) - const persistContent = - mentionedDocuments.length > 0 - ? [ - ...message.content, - { - type: "mentioned-documents", - documents: mentionedDocuments.map((doc) => ({ - id: doc.id, - title: doc.title, - document_type: doc.document_type, - })), - }, - ] - : message.content; + // Persist user message with mentioned documents and attachments (don't await, fire and forget) + const persistContent: unknown[] = [...message.content]; + + // Add mentioned documents for persistence + if (mentionedDocuments.length > 0) { + persistContent.push({ + type: "mentioned-documents", + documents: mentionedDocuments.map((doc) => ({ + id: doc.id, + title: doc.title, + document_type: doc.document_type, + })), + }); + } + + // Add attachments for persistence (so they survive page reload) + if (message.attachments && message.attachments.length > 0) { + persistContent.push({ + type: "attachments", + items: message.attachments.map((att) => ({ + id: att.id, + name: att.name, + type: att.type, + contentType: (att as { contentType?: string }).contentType, + // Include imageDataUrl for images so they can be displayed after reload + imageDataUrl: (att as { imageDataUrl?: string }).imageDataUrl, + // Include extractedContent for context (already extracted, no re-processing needed) + extractedContent: (att as { extractedContent?: string }).extractedContent, + })), + }); + } + appendMessage(currentThreadId, { role: "user", content: persistContent, - }).catch((err) => console.error("Failed to persist user message:", err)); + }) + .then(() => { + // For new threads, the backend updates the title from the first user message + // Invalidate threads query so sidebar shows the updated title in real-time + if (isNewThread) { + queryClient.invalidateQueries({ queryKey: ["threads", String(searchSpaceId)] }); + } + }) + .catch((err) => console.error("Failed to persist user message:", err)); // Start streaming response setIsRunning(true); @@ -676,7 +785,19 @@ export default function NewChatPage() { } } catch (error) { if (error instanceof Error && error.name === "AbortError") { - // Request was cancelled + // Request was cancelled by user - persist partial response if any content was received + const hasContent = contentParts.some( + (part) => + (part.type === "text" && part.text.length > 0) || + (part.type === "tool-call" && TOOLS_WITH_UI.has(part.toolName)) + ); + if (hasContent && currentThreadId) { + const partialContent = buildContentForPersistence(); + appendMessage(currentThreadId, { + role: "assistant", + content: partialContent, + }).catch((err) => console.error("Failed to persist partial assistant message:", err)); + } return; } console.error("[NewChatPage] Chat error:", error); @@ -720,6 +841,7 @@ export default function NewChatPage() { setMentionedDocumentIds, setMentionedDocuments, setMessageDocumentsMap, + queryClient, ] ); @@ -789,6 +911,7 @@ export default function NewChatPage() { +
{ - router.push(`/dashboard/${search_space_id}/chats`); - }, []); + router.push(`/dashboard/${search_space_id}/new-chat`); + }, [router, search_space_id]); return <>; } diff --git a/surfsense_web/app/layout.tsx b/surfsense_web/app/layout.tsx index abd64761e..d952ad1a7 100644 --- a/surfsense_web/app/layout.tsx +++ b/surfsense_web/app/layout.tsx @@ -1,6 +1,5 @@ import type { Metadata } from "next"; import "./globals.css"; -import { GoogleAnalytics } from "@next/third-parties/google"; import { RootProvider } from "fumadocs-ui/provider/next"; import { Roboto } from "next/font/google"; import { I18nProvider } from "@/components/providers/I18nProvider"; @@ -92,7 +91,6 @@ export default function RootLayout({ // Locale state is managed by LocaleContext and persisted in localStorage return ( - diff --git a/surfsense_web/atoms/chat/plan-state.atom.ts b/surfsense_web/atoms/chat/plan-state.atom.ts new file mode 100644 index 000000000..2436dd300 --- /dev/null +++ b/surfsense_web/atoms/chat/plan-state.atom.ts @@ -0,0 +1,224 @@ +/** + * Plan State Atom + * + * Tracks the latest state of each plan by title. + * When write_todos is called multiple times with the same title, + * only the FIRST component renders (stays fixed in position), + * subsequent calls just update the shared state. + */ + +import { atom } from "jotai"; + +export interface PlanTodo { + id: string; + content: string; + status: "pending" | "in_progress" | "completed" | "cancelled"; +} + +export interface PlanState { + id: string; + title: string; + todos: PlanTodo[]; + lastUpdated: number; + /** The toolCallId of the first component that rendered this plan */ + ownerToolCallId: string; +} + +/** + * SYNCHRONOUS ownership registry - prevents race conditions + * Only ONE plan allowed per conversation - first plan wins + */ +let firstPlanOwner: { toolCallId: string; title: string } | null = null; + +/** + * Register as owner of a plan SYNCHRONOUSLY + * ONE PLAN PER CONVERSATION: Only the first write_todos call renders. + * All subsequent calls update the state but don't render their own card. + */ +export function registerPlanOwner(title: string, toolCallId: string): boolean { + if (!firstPlanOwner) { + // First plan in this conversation - claim ownership + firstPlanOwner = { toolCallId, title }; + return true; + } + + // Check if we're the owner + return firstPlanOwner.toolCallId === toolCallId; +} + +/** + * Get the canonical title for a plan + * Returns the first plan's title if one exists, otherwise the provided title + */ +export function getCanonicalPlanTitle(title: string): string { + return firstPlanOwner?.title || title; +} + +/** + * Check if a plan already exists in this conversation + */ +export function hasPlan(): boolean { + return firstPlanOwner !== null; +} + +/** + * Get the first plan's info + */ +export function getFirstPlanInfo(): { toolCallId: string; title: string } | null { + return firstPlanOwner; +} + +/** + * Check if a toolCallId is the owner of the plan SYNCHRONOUSLY + */ +export function isPlanOwner(toolCallId: string): boolean { + return !firstPlanOwner || firstPlanOwner.toolCallId === toolCallId; +} + +/** + * Clear ownership registry (call when starting a new chat) + */ +export function clearPlanOwnerRegistry(): void { + firstPlanOwner = null; +} + +/** + * Map of plan title -> latest plan state + * Using title as key since it stays constant across updates + */ +export const planStatesAtom = atom>(new Map()); + +/** + * Input type for updating plan state + */ +export interface UpdatePlanInput { + id: string; + title: string; + todos: PlanTodo[]; + toolCallId: string; +} + +/** + * Helper atom to update a plan state + */ +export const updatePlanStateAtom = atom(null, (get, set, plan: UpdatePlanInput) => { + const states = new Map(get(planStatesAtom)); + + // Register ownership synchronously if not already done + registerPlanOwner(plan.title, plan.toolCallId); + + // Get the actual owner from the first plan + const ownerToolCallId = firstPlanOwner?.toolCallId || plan.toolCallId; + + // Always use the canonical (first) title for the plan key + const canonicalTitle = getCanonicalPlanTitle(plan.title); + + states.set(canonicalTitle, { + id: plan.id, + title: canonicalTitle, + todos: plan.todos, + lastUpdated: Date.now(), + ownerToolCallId, + }); + set(planStatesAtom, states); +}); + +/** + * Helper atom to get the latest plan state by title + */ +export const getPlanStateAtom = atom((get) => { + const states = get(planStatesAtom); + return (title: string) => states.get(title); +}); + +/** + * Helper atom to clear all plan states (useful when starting a new chat) + */ +export const clearPlanStatesAtom = atom(null, (get, set) => { + clearPlanOwnerRegistry(); + set(planStatesAtom, new Map()); +}); + +/** + * Hydrate plan state from persisted message content + * Call this when loading messages from the database to restore plan state + */ +export interface HydratePlanInput { + toolCallId: string; + result: { + id?: string; + title?: string; + todos?: Array<{ + id?: string; + content: string; + status: "pending" | "in_progress" | "completed" | "cancelled"; + }>; + }; +} + +export const hydratePlanStateAtom = atom(null, (get, set, plan: HydratePlanInput) => { + if (!plan.result?.todos || plan.result.todos.length === 0) return; + + const states = new Map(get(planStatesAtom)); + const title = plan.result.title || "Plan"; + + // Register this as the owner if no plan exists yet + registerPlanOwner(title, plan.toolCallId); + + // Get the canonical title + const canonicalTitle = getCanonicalPlanTitle(title); + const ownerToolCallId = firstPlanOwner?.toolCallId || plan.toolCallId; + + // Only set if this is newer or doesn't exist + const existing = states.get(canonicalTitle); + if (!existing) { + states.set(canonicalTitle, { + id: plan.result.id || `plan-${Date.now()}`, + title: canonicalTitle, + todos: plan.result.todos.map((t, i) => ({ + id: t.id || `todo-${i}`, + content: t.content, + status: t.status, + })), + lastUpdated: Date.now(), + ownerToolCallId, + }); + set(planStatesAtom, states); + } +}); + +/** + * Extract write_todos tool call data from message content + * Returns an array of { toolCallId, result } for each write_todos call found + */ +export function extractWriteTodosFromContent(content: unknown): HydratePlanInput[] { + if (!Array.isArray(content)) return []; + + const results: HydratePlanInput[] = []; + + for (const part of content) { + if ( + typeof part === "object" && + part !== null && + "type" in part && + (part as { type: string }).type === "tool-call" && + "toolName" in part && + (part as { toolName: string }).toolName === "write_todos" && + "toolCallId" in part && + "result" in part + ) { + const toolCall = part as { + toolCallId: string; + result: HydratePlanInput["result"]; + }; + if (toolCall.result) { + results.push({ + toolCallId: toolCall.toolCallId, + result: toolCall.result, + }); + } + } + } + + return results; +} diff --git a/surfsense_web/atoms/connectors/connector-mutation.atoms.ts b/surfsense_web/atoms/connectors/connector-mutation.atoms.ts new file mode 100644 index 000000000..70b5b0322 --- /dev/null +++ b/surfsense_web/atoms/connectors/connector-mutation.atoms.ts @@ -0,0 +1,100 @@ +import { atomWithMutation } from "jotai-tanstack-query"; +import { toast } from "sonner"; +import type { + CreateConnectorRequest, + DeleteConnectorRequest, + GetConnectorsResponse, + IndexConnectorRequest, + IndexConnectorResponse, + UpdateConnectorRequest, +} from "@/contracts/types/connector.types"; +import { connectorsApiService } from "@/lib/apis/connectors-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queryClient } from "@/lib/query-client/client"; +import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms"; + +export const createConnectorMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: cacheKeys.connectors.all(searchSpaceId!), + enabled: !!searchSpaceId, + mutationFn: async (request: CreateConnectorRequest) => { + return connectorsApiService.createConnector(request); + }, + + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: cacheKeys.connectors.all(searchSpaceId!), + }); + }, + }; +}); + +export const updateConnectorMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: cacheKeys.connectors.all(searchSpaceId!), + enabled: !!searchSpaceId, + mutationFn: async (request: UpdateConnectorRequest) => { + return connectorsApiService.updateConnector(request); + }, + + onSuccess: (_, request: UpdateConnectorRequest) => { + queryClient.invalidateQueries({ + queryKey: cacheKeys.connectors.all(searchSpaceId!), + }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.connectors.byId(String(request.id)), + }); + }, + }; +}); + +export const deleteConnectorMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: cacheKeys.connectors.all(searchSpaceId!), + enabled: !!searchSpaceId, + mutationFn: async (request: DeleteConnectorRequest) => { + return connectorsApiService.deleteConnector(request); + }, + + onSuccess: (_, request: DeleteConnectorRequest) => { + queryClient.setQueryData( + cacheKeys.connectors.all(searchSpaceId!), + (oldData: GetConnectorsResponse | undefined) => { + if (!oldData) return oldData; + return oldData.filter((connector) => connector.id !== request.id); + } + ); + queryClient.invalidateQueries({ + queryKey: cacheKeys.connectors.byId(String(request.id)), + }); + }, + }; +}); + +export const indexConnectorMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: cacheKeys.connectors.index(), + enabled: !!searchSpaceId, + mutationFn: async (request: IndexConnectorRequest) => { + return connectorsApiService.indexConnector(request); + }, + + onSuccess: (response: IndexConnectorResponse) => { + toast.success(response.message); + queryClient.invalidateQueries({ + queryKey: cacheKeys.connectors.all(searchSpaceId!), + }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.connectors.byId(String(response.connector_id)), + }); + }, + }; +}); diff --git a/surfsense_web/atoms/connectors/connector-query.atoms.ts b/surfsense_web/atoms/connectors/connector-query.atoms.ts new file mode 100644 index 000000000..24777c8c6 --- /dev/null +++ b/surfsense_web/atoms/connectors/connector-query.atoms.ts @@ -0,0 +1,21 @@ +import { atomWithQuery } from "jotai-tanstack-query"; +import { connectorsApiService } from "@/lib/apis/connectors-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms"; + +export const connectorsAtom = atomWithQuery((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + queryKey: cacheKeys.connectors.all(searchSpaceId!), + enabled: !!searchSpaceId, + staleTime: 5 * 60 * 1000, // 5 minutes + queryFn: async () => { + return connectorsApiService.getConnectors({ + queryParams: { + search_space_id: searchSpaceId!, + }, + }); + }, + }; +}); diff --git a/surfsense_web/atoms/connectors/ui.atoms.ts b/surfsense_web/atoms/connectors/ui.atoms.ts new file mode 100644 index 000000000..e085ad34e --- /dev/null +++ b/surfsense_web/atoms/connectors/ui.atoms.ts @@ -0,0 +1,7 @@ +import { atom } from "jotai"; +import type { GetConnectorsRequest } from "@/contracts/types/connector.types"; + +export const globalConnectorsQueryParamsAtom = atom({ + skip: 0, + limit: 10, +}); diff --git a/surfsense_web/atoms/logs/log-mutation.atoms.ts b/surfsense_web/atoms/logs/log-mutation.atoms.ts new file mode 100644 index 000000000..e17b42fb6 --- /dev/null +++ b/surfsense_web/atoms/logs/log-mutation.atoms.ts @@ -0,0 +1,68 @@ +import { atomWithMutation } from "jotai-tanstack-query"; +import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; +import type { + CreateLogRequest, + DeleteLogRequest, + UpdateLogRequest, +} from "@/contracts/types/log.types"; +import { logsApiService } from "@/lib/apis/logs-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queryClient } from "@/lib/query-client/client"; + +/** + * Create Log Mutation + */ +export const createLogMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + return { + mutationKey: cacheKeys.logs.list(searchSpaceId ?? undefined), + enabled: !!searchSpaceId, + mutationFn: async (request: CreateLogRequest) => logsApiService.createLog(request), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: cacheKeys.logs.list(searchSpaceId ?? undefined) }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.logs.summary(searchSpaceId ?? undefined), + }); + }, + }; +}); + +/** + * Update Log Mutation + */ +export const updateLogMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + return { + mutationKey: cacheKeys.logs.list(searchSpaceId ?? undefined), + enabled: !!searchSpaceId, + mutationFn: async ({ logId, data }: { logId: number; data: UpdateLogRequest }) => + logsApiService.updateLog(logId, data), + onSuccess: (_data, variables) => { + queryClient.invalidateQueries({ queryKey: cacheKeys.logs.detail(variables.logId) }); + queryClient.invalidateQueries({ queryKey: cacheKeys.logs.list(searchSpaceId ?? undefined) }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.logs.summary(searchSpaceId ?? undefined), + }); + }, + }; +}); + +/** + * Delete Log Mutation + */ +export const deleteLogMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + return { + mutationKey: cacheKeys.logs.list(searchSpaceId ?? undefined), + enabled: !!searchSpaceId, + mutationFn: async (request: DeleteLogRequest) => logsApiService.deleteLog(request), + onSuccess: (_data, request) => { + queryClient.invalidateQueries({ queryKey: cacheKeys.logs.list(searchSpaceId ?? undefined) }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.logs.summary(searchSpaceId ?? undefined), + }); + if (request?.id) + queryClient.invalidateQueries({ queryKey: cacheKeys.logs.detail(request.id) }); + }, + }; +}); diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 90d4e62a3..9ec7386c3 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -15,8 +15,6 @@ import { AlertCircle, ArrowDownIcon, ArrowUpIcon, - Brain, - CheckCircle2, CheckIcon, ChevronLeftIcon, ChevronRightIcon, @@ -28,8 +26,6 @@ import { Plug2, Plus, RefreshCwIcon, - Search, - Sparkles, SquareIcon, } from "lucide-react"; import Link from "next/link"; @@ -75,13 +71,8 @@ import { DocumentMentionPicker, type DocumentMentionPickerRef, } from "@/components/new-chat/document-mention-picker"; -import { - ChainOfThought, - ChainOfThoughtContent, - ChainOfThoughtItem, - ChainOfThoughtStep, - ChainOfThoughtTrigger, -} from "@/components/prompt-kit/chain-of-thought"; +import { ChainOfThoughtItem } from "@/components/prompt-kit/chain-of-thought"; +import { TextShimmerLoader } from "@/components/prompt-kit/loader"; import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; import { Button } from "@/components/ui/button"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; @@ -103,124 +94,146 @@ interface ThreadProps { const ThinkingStepsContext = createContext>(new Map()); /** - * Get icon based on step status and title - */ -function getStepIcon(status: "pending" | "in_progress" | "completed", title: string) { - const titleLower = title.toLowerCase(); - - if (status === "in_progress") { - return ; - } - - if (status === "completed") { - return ; - } - - if (titleLower.includes("search") || titleLower.includes("knowledge")) { - return ; - } - - if (titleLower.includes("analy") || titleLower.includes("understand")) { - return ; - } - - return ; -} - -/** - * Chain of thought display component with smart expand/collapse behavior + * Chain of thought display component - single collapsible dropdown design */ const ThinkingStepsDisplay: FC<{ steps: ThinkingStep[]; isThreadRunning?: boolean }> = ({ steps, isThreadRunning = true, }) => { - // Track which steps the user has manually toggled (overrides auto behavior) - const [manualOverrides, setManualOverrides] = useState>({}); - // Track previous step statuses to detect changes - const prevStatusesRef = useRef>({}); + const [isOpen, setIsOpen] = useState(true); - // Derive effective status: if thread stopped and step is in_progress, treat as completed - const getEffectiveStatus = (step: ThinkingStep): "pending" | "in_progress" | "completed" => { - if (step.status === "in_progress" && !isThreadRunning) { - return "completed"; // Thread was stopped, so mark as completed - } - return step.status; - }; - - // Clear manual overrides when a step's status changes - useEffect(() => { - const currentStatuses: Record = {}; - steps.forEach((step) => { - currentStatuses[step.id] = step.status; - // If status changed, clear any manual override for this step - if (prevStatusesRef.current[step.id] && prevStatusesRef.current[step.id] !== step.status) { - setManualOverrides((prev) => { - const next = { ...prev }; - delete next[step.id]; - return next; - }); + // Derive effective status for each step + const getEffectiveStatus = useCallback( + (step: ThinkingStep): "pending" | "in_progress" | "completed" => { + if (step.status === "in_progress" && !isThreadRunning) { + return "completed"; } - }); - prevStatusesRef.current = currentStatuses; - }, [steps]); + return step.status; + }, + [isThreadRunning] + ); + + // Calculate summary info + const completedSteps = steps.filter((s) => getEffectiveStatus(s) === "completed").length; + const inProgressStep = steps.find((s) => getEffectiveStatus(s) === "in_progress"); + const allCompleted = completedSteps === steps.length && steps.length > 0 && !isThreadRunning; + const isProcessing = isThreadRunning && !allCompleted; + + // Auto-collapse when all tasks are completed + useEffect(() => { + if (allCompleted) { + setIsOpen(false); + } + }, [allCompleted]); if (steps.length === 0) return null; - const getStepOpenState = (step: ThinkingStep): boolean => { - const effectiveStatus = getEffectiveStatus(step); - // If user has manually toggled, respect that - if (manualOverrides[step.id] !== undefined) { - return manualOverrides[step.id]; + // Generate header text + const getHeaderText = () => { + if (allCompleted) { + return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`; } - // Auto behavior: open if in progress - if (effectiveStatus === "in_progress") { - return true; + if (inProgressStep) { + return inProgressStep.title; } - // Default: collapsed (all steps collapse when processing is done) - return false; - }; - - const handleToggle = (stepId: string, currentOpen: boolean) => { - setManualOverrides((prev) => ({ - ...prev, - [stepId]: !currentOpen, - })); + if (isProcessing) { + return `Processing ${completedSteps}/${steps.length} steps`; + } + return `Reviewed ${completedSteps} ${completedSteps === 1 ? "step" : "steps"}`; }; return (
- - {steps.map((step) => { - const effectiveStatus = getEffectiveStatus(step); - const icon = getStepIcon(effectiveStatus, step.title); - const isOpen = getStepOpenState(step); - return ( - handleToggle(step.id, isOpen)} - > - - {step.title} - - {step.items && step.items.length > 0 && ( - - {step.items.map((item, idx) => ( - {item} - ))} - - )} - - ); - })} - +
+ {/* Main collapsible header */} + + + {/* Collapsible content with CSS grid animation */} +
+
+
+ {steps.map((step, index) => { + const effectiveStatus = getEffectiveStatus(step); + const isLast = index === steps.length - 1; + + return ( +
+ {/* Dot and line column */} +
+ {/* Vertical connection line - extends to next dot */} + {!isLast && ( +
+ )} + {/* Step dot - on top of line */} +
+ {effectiveStatus === "in_progress" ? ( + + ) : ( + + )} +
+
+ + {/* Step content */} +
+ {/* Step title */} +
+ {effectiveStatus === "in_progress" ? ( + + ) : ( + step.title + )} +
+ + {/* Step items (sub-content) */} + {step.items && step.items.length > 0 && ( +
+ {step.items.map((item, idx) => ( + + {item} + + ))} +
+ )} +
+
+ ); + })} +
+
+
+
); }; @@ -286,7 +299,7 @@ export const Thread: FC = ({ messageThinkingSteps = new Map(), head > {/* Optional sticky header for model selector etc. */} {header &&
{header}
} @@ -428,13 +441,6 @@ const Composer: FC = () => { } }, [isThreadEmpty]); - // Reset auto-focus flag when thread becomes non-empty (user sent a message) - useEffect(() => { - if (!isThreadEmpty) { - hasAutoFocusedRef.current = false; - } - }, [isThreadEmpty]); - // Sync mentioned document IDs to atom for use in chat request useEffect(() => { setMentionedDocumentIds(mentionedDocuments.map((doc) => doc.id)); @@ -561,7 +567,7 @@ const Composer: FC = () => {
{ ) : ( <> - {totalSourceCount > 0 ? ( + {totalSourceCount > 0 && ( {totalSourceCount > 99 ? "99+" : totalSourceCount} - ) : ( - - - )} )} @@ -917,7 +919,7 @@ const AssistantMessageInner: FC = () => {
-
+
diff --git a/surfsense_web/components/prompt-kit/loader.tsx b/surfsense_web/components/prompt-kit/loader.tsx new file mode 100644 index 000000000..435a6a136 --- /dev/null +++ b/surfsense_web/components/prompt-kit/loader.tsx @@ -0,0 +1,66 @@ +"use client"; + +import { cn } from "@/lib/utils"; + +export interface LoaderProps { + variant?: "text-shimmer"; + size?: "sm" | "md" | "lg"; + text?: string; + className?: string; +} + +const textSizes = { + sm: "text-xs", + md: "text-sm", + lg: "text-base", +} as const; + +/** + * TextShimmerLoader - A text loader with a shimmer gradient animation + * Used for in-progress states in write_todos and chain-of-thought + */ +export function TextShimmerLoader({ + text = "Thinking", + className, + size = "md", +}: { + text?: string; + className?: string; + size?: "sm" | "md" | "lg"; +}) { + return ( + <> + + + {text} + + + ); +} + +/** + * Loader component - currently only supports text-shimmer variant + * Can be extended with more variants if needed in the future + */ +export function Loader({ variant = "text-shimmer", size = "md", text, className }: LoaderProps) { + switch (variant) { + case "text-shimmer": + default: + return ; + } +} diff --git a/surfsense_web/components/sidebar/AppSidebarProvider.tsx b/surfsense_web/components/sidebar/AppSidebarProvider.tsx index 5e7f08c4d..f5146c427 100644 --- a/surfsense_web/components/sidebar/AppSidebarProvider.tsx +++ b/surfsense_web/components/sidebar/AppSidebarProvider.tsx @@ -3,7 +3,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useAtomValue, useSetAtom } from "jotai"; import { Trash2 } from "lucide-react"; -import { useRouter } from "next/navigation"; +import { useParams, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useCallback, useMemo, useState } from "react"; import { hasUnsavedEditorChangesAtom, pendingEditorNavigationAtom } from "@/atoms/editor/ui.atoms"; @@ -50,7 +50,13 @@ export function AppSidebarProvider({ const t = useTranslations("dashboard"); const tCommon = useTranslations("common"); const router = useRouter(); + const params = useParams(); const queryClient = useQueryClient(); + + // Get current chat ID from URL params + const currentChatId = params?.chat_id + ? Number(Array.isArray(params.chat_id) ? params.chat_id[0] : params.chat_id) + : null; const [isDeletingThread, setIsDeletingThread] = useState(false); // Editor state for handling unsaved changes @@ -61,7 +67,6 @@ export function AppSidebarProvider({ const { data: threadsData, error: threadError, - isLoading: isLoadingThreads, refetch: refetchThreads, } = useQuery({ queryKey: ["threads", searchSpaceId], @@ -73,7 +78,6 @@ export function AppSidebarProvider({ data: searchSpace, isLoading: isLoadingSearchSpace, error: searchSpaceError, - refetch: fetchSearchSpace, } = useQuery({ queryKey: cacheKeys.searchSpaces.detail(searchSpaceId), queryFn: () => searchSpacesApiService.getSearchSpace({ id: Number(searchSpaceId) }), @@ -83,12 +87,7 @@ export function AppSidebarProvider({ const { data: user } = useAtomValue(currentUserAtom); // Fetch notes - const { - data: notesData, - error: notesError, - isLoading: isLoadingNotes, - refetch: refetchNotes, - } = useQuery({ + const { data: notesData, refetch: refetchNotes } = useQuery({ queryKey: ["notes", searchSpaceId], queryFn: () => notesApiService.getNotes({ @@ -108,11 +107,6 @@ export function AppSidebarProvider({ } | null>(null); const [isDeletingNote, setIsDeletingNote] = useState(false); - // Retry function - const retryFetch = useCallback(() => { - fetchSearchSpace(); - }, [fetchSearchSpace]); - // Transform threads to the format expected by AppSidebar const recentChats = useMemo(() => { if (!threadsData?.threads) return []; @@ -149,6 +143,10 @@ export function AppSidebarProvider({ await deleteThread(threadToDelete.id); // Invalidate threads query to refresh the list queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); + // Only navigate to new-chat if the deleted chat is currently open + if (currentChatId === threadToDelete.id) { + router.push(`/dashboard/${searchSpaceId}/new-chat`); + } } catch (error) { console.error("Error deleting thread:", error); } finally { @@ -156,7 +154,7 @@ export function AppSidebarProvider({ setShowDeleteDialog(false); setThreadToDelete(null); } - }, [threadToDelete, queryClient, searchSpaceId]); + }, [threadToDelete, queryClient, searchSpaceId, router, currentChatId]); // Handle delete note with confirmation const handleDeleteNote = useCallback(async () => { diff --git a/surfsense_web/components/sidebar/all-chats-sidebar.tsx b/surfsense_web/components/sidebar/all-chats-sidebar.tsx index 9076715a3..ef55142fa 100644 --- a/surfsense_web/components/sidebar/all-chats-sidebar.tsx +++ b/surfsense_web/components/sidebar/all-chats-sidebar.tsx @@ -13,7 +13,7 @@ import { X, } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; -import { useRouter } from "next/navigation"; +import { useParams, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useCallback, useEffect, useState } from "react"; import { createPortal } from "react-dom"; @@ -47,7 +47,15 @@ interface AllChatsSidebarProps { export function AllChatsSidebar({ open, onOpenChange, searchSpaceId }: AllChatsSidebarProps) { const t = useTranslations("sidebar"); const router = useRouter(); + const params = useParams(); const queryClient = useQueryClient(); + + // Get the current chat ID from URL to check if user is deleting the currently open chat + const currentChatId = Array.isArray(params.chat_id) + ? Number(params.chat_id[0]) + : params.chat_id + ? Number(params.chat_id) + : null; const [deletingThreadId, setDeletingThreadId] = useState(null); const [archivingThreadId, setArchivingThreadId] = useState(null); const [searchQuery, setSearchQuery] = useState(""); @@ -126,6 +134,15 @@ export function AllChatsSidebar({ open, onOpenChange, searchSpaceId }: AllChatsS queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); + + // If the deleted chat is currently open, close sidebar first then redirect + if (currentChatId === threadId) { + onOpenChange(false); + // Wait for sidebar close animation to complete before navigating + setTimeout(() => { + router.push(`/dashboard/${searchSpaceId}/new-chat`); + }, 250); + } } catch (error) { console.error("Error deleting thread:", error); toast.error(t("error_deleting_chat") || "Failed to delete chat"); @@ -133,7 +150,7 @@ export function AllChatsSidebar({ open, onOpenChange, searchSpaceId }: AllChatsS setDeletingThreadId(null); } }, - [queryClient, searchSpaceId, t] + [queryClient, searchSpaceId, t, currentChatId, router, onOpenChange] ); // Handle thread archive/unarchive @@ -293,6 +310,7 @@ export function AllChatsSidebar({ open, onOpenChange, searchSpaceId }: AllChatsS const isDeleting = deletingThreadId === thread.id; const isArchiving = archivingThreadId === thread.id; const isBusy = isDeleting || isArchiving; + const isActive = currentChatId === thread.id; return (
diff --git a/surfsense_web/components/sidebar/all-notes-sidebar.tsx b/surfsense_web/components/sidebar/all-notes-sidebar.tsx index d66a01780..ff9f07175 100644 --- a/surfsense_web/components/sidebar/all-notes-sidebar.tsx +++ b/surfsense_web/components/sidebar/all-notes-sidebar.tsx @@ -4,7 +4,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { format } from "date-fns"; import { FileText, Loader2, MoreHorizontal, Plus, Search, Trash2, X } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; -import { useRouter } from "next/navigation"; +import { useParams, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useCallback, useEffect, useMemo, useState } from "react"; import { createPortal } from "react-dom"; @@ -37,7 +37,11 @@ export function AllNotesSidebar({ }: AllNotesSidebarProps) { const t = useTranslations("sidebar"); const router = useRouter(); + const params = useParams(); const queryClient = useQueryClient(); + + // Get the current note ID from URL to highlight the open note + const currentNoteId = params.note_id ? Number(params.note_id) : null; const [deletingNoteId, setDeletingNoteId] = useState(null); const [searchQuery, setSearchQuery] = useState(""); const [mounted, setMounted] = useState(false); @@ -208,7 +212,7 @@ export function AllNotesSidebar({ aria-label={t("all_notes") || "All Notes"} > {/* Header */} -
+

{t("all_notes") || "All Notes"}

+ + + + + + )} + + {actionArray.length > 0 && ( +
+ {actionArray.map((action) => ( + + ))} +
+ )} + + + ); +}; diff --git a/surfsense_web/components/tool-ui/plan/schema.ts b/surfsense_web/components/tool-ui/plan/schema.ts new file mode 100644 index 000000000..a8263cf71 --- /dev/null +++ b/surfsense_web/components/tool-ui/plan/schema.ts @@ -0,0 +1,91 @@ +import { z } from "zod"; + +/** + * Todo item status + */ +export const TodoStatusSchema = z.enum(["pending", "in_progress", "completed", "cancelled"]); +export type TodoStatus = z.infer; + +/** + * Single todo item in a plan + * Matches deepagents TodoListMiddleware output: { content, status } + * id is auto-generated if not provided + */ +export const PlanTodoSchema = z.object({ + id: z.string().optional(), + content: z.string(), + status: TodoStatusSchema, +}); + +export type PlanTodo = z.infer; + +/** + * Serializable plan schema for tool results + * Matches deepagents TodoListMiddleware output format + * id/title are auto-generated if not provided + */ +export const SerializablePlanSchema = z.object({ + id: z.string().optional(), + title: z.string().optional(), + todos: z.array(PlanTodoSchema).min(1), + maxVisibleTodos: z.number().optional(), + showProgress: z.boolean().optional(), +}); + +export type SerializablePlan = z.infer; + +/** + * Normalized plan with required fields (after auto-generation) + */ +export interface NormalizedPlan { + id: string; + title: string; + todos: Array<{ id: string; content: string; status: TodoStatus }>; + maxVisibleTodos?: number; + showProgress?: boolean; +} + +/** + * Parse and normalize a plan from tool result + * Auto-generates id/title if not provided (for deepagents compatibility) + */ +export function parseSerializablePlan(data: unknown): NormalizedPlan { + const result = SerializablePlanSchema.safeParse(data); + + if (!result.success) { + console.warn("Invalid plan data:", result.error.issues); + + // Try to extract basic info for fallback + const obj = (data && typeof data === "object" ? data : {}) as Record; + + return { + id: typeof obj.id === "string" ? obj.id : `plan-${Date.now()}`, + title: typeof obj.title === "string" ? obj.title : "Plan", + todos: Array.isArray(obj.todos) + ? obj.todos.map((t: unknown, i: number) => { + const todo = t as Record; + return { + id: typeof todo?.id === "string" ? todo.id : `todo-${i}`, + content: typeof todo?.content === "string" ? todo.content : "Task", + status: TodoStatusSchema.safeParse(todo?.status).success + ? (todo.status as TodoStatus) + : ("pending" as const), + }; + }) + : [{ id: "1", content: "No tasks", status: "pending" as const }], + }; + } + + // Normalize: add id/title if missing + return { + id: result.data.id || `plan-${Date.now()}`, + title: result.data.title || "Plan", + todos: result.data.todos.map((t, i) => ({ + id: t.id || `todo-${i}`, + content: t.content, + status: t.status, + })), + maxVisibleTodos: result.data.maxVisibleTodos, + showProgress: result.data.showProgress, + }; +} diff --git a/surfsense_web/components/tool-ui/scrape-webpage.tsx b/surfsense_web/components/tool-ui/scrape-webpage.tsx index 025328235..29e7094db 100644 --- a/surfsense_web/components/tool-ui/scrape-webpage.tsx +++ b/surfsense_web/components/tool-ui/scrape-webpage.tsx @@ -2,6 +2,7 @@ import { makeAssistantToolUI } from "@assistant-ui/react"; import { AlertCircleIcon, FileTextIcon } from "lucide-react"; +import { z } from "zod"; import { Article, ArticleErrorBoundary, @@ -9,30 +10,44 @@ import { parseSerializableArticle, } from "@/components/tool-ui/article"; -/** - * Type definitions for the scrape_webpage tool - */ -interface ScrapeWebpageArgs { - url: string; - max_length?: number; -} +// ============================================================================ +// Zod Schemas +// ============================================================================ -interface ScrapeWebpageResult { - id: string; - assetId: string; - kind: "article"; - href: string; - title: string; - description?: string; - content?: string; - domain?: string; - author?: string; - date?: string; - word_count?: number; - was_truncated?: boolean; - crawler_type?: string; - error?: string; -} +/** + * Schema for scrape_webpage tool arguments + */ +const ScrapeWebpageArgsSchema = z.object({ + url: z.string(), + max_length: z.number().nullish(), +}); + +/** + * Schema for scrape_webpage tool result + */ +const ScrapeWebpageResultSchema = z.object({ + id: z.string(), + assetId: z.string(), + kind: z.literal("article"), + href: z.string(), + title: z.string(), + description: z.string().nullish(), + content: z.string().nullish(), + domain: z.string().nullish(), + author: z.string().nullish(), + date: z.string().nullish(), + word_count: z.number().nullish(), + was_truncated: z.boolean().nullish(), + crawler_type: z.string().nullish(), + error: z.string().nullish(), +}); + +// ============================================================================ +// Types +// ============================================================================ + +type ScrapeWebpageArgs = z.infer; +type ScrapeWebpageResult = z.infer; /** * Error state component shown when webpage scraping fails @@ -154,4 +169,9 @@ export const ScrapeWebpageToolUI = makeAssistantToolUI void; + disabled?: boolean; +} + +export const ActionButtons: FC = ({ actions, onAction, disabled }) => { + if (!actions) return null; + + // Normalize actions to array format + const actionArray: Action[] = Array.isArray(actions) + ? actions + : ([ + actions.confirm && { ...actions.confirm, id: "confirm" }, + actions.cancel && { ...actions.cancel, id: "cancel" }, + ].filter(Boolean) as Action[]); + + if (actionArray.length === 0) return null; + + return ( +
+ {actionArray.map((action) => ( + + ))} +
+ ); +}; diff --git a/surfsense_web/components/tool-ui/shared/index.ts b/surfsense_web/components/tool-ui/shared/index.ts new file mode 100644 index 000000000..23f5a27dd --- /dev/null +++ b/surfsense_web/components/tool-ui/shared/index.ts @@ -0,0 +1,2 @@ +export * from "./action-buttons"; +export * from "./schema"; diff --git a/surfsense_web/components/tool-ui/shared/schema.ts b/surfsense_web/components/tool-ui/shared/schema.ts new file mode 100644 index 000000000..8076a8e45 --- /dev/null +++ b/surfsense_web/components/tool-ui/shared/schema.ts @@ -0,0 +1,23 @@ +import { z } from "zod"; + +/** + * Shared action schema for tool UI components + */ +export const ActionSchema = z.object({ + id: z.string(), + label: z.string(), + variant: z.enum(["default", "secondary", "destructive", "outline", "ghost", "link"]).optional(), + disabled: z.boolean().optional(), +}); + +export type Action = z.infer; + +/** + * Actions configuration schema + */ +export const ActionsConfigSchema = z.object({ + confirm: ActionSchema.optional(), + cancel: ActionSchema.optional(), +}); + +export type ActionsConfig = z.infer; diff --git a/surfsense_web/components/tool-ui/write-todos.tsx b/surfsense_web/components/tool-ui/write-todos.tsx new file mode 100644 index 000000000..a5da31e9e --- /dev/null +++ b/surfsense_web/components/tool-ui/write-todos.tsx @@ -0,0 +1,158 @@ +"use client"; + +import { makeAssistantToolUI, useAssistantState } from "@assistant-ui/react"; +import { useAtomValue, useSetAtom } from "jotai"; +import { Loader2 } from "lucide-react"; +import { useEffect, useMemo } from "react"; +import { z } from "zod"; +import { + getCanonicalPlanTitle, + planStatesAtom, + registerPlanOwner, + updatePlanStateAtom, +} from "@/atoms/chat/plan-state.atom"; +import { Plan, PlanErrorBoundary, parseSerializablePlan, TodoStatusSchema } from "./plan"; + +// ============================================================================ +// Zod Schemas - Matching deepagents TodoListMiddleware output +// ============================================================================ + +/** + * Schema for a single todo item (matches deepagents output) + */ +const TodoItemSchema = z.object({ + content: z.string(), + status: TodoStatusSchema, +}); + +/** + * Schema for write_todos tool args/result (matches deepagents output) + * deepagents provides: { todos: [{ content, status }] } + */ +const WriteTodosSchema = z.object({ + todos: z.array(TodoItemSchema).nullish(), +}); + +// ============================================================================ +// Types +// ============================================================================ + +type WriteTodosData = z.infer; + +/** + * Loading state component + */ +function WriteTodosLoading() { + return ( +
+
+ + Creating plan... +
+
+ ); +} + +/** + * WriteTodos Tool UI Component + * + * Displays the agent's planning/todo list with a beautiful UI. + * Uses deepagents TodoListMiddleware output directly: { todos: [{ content, status }] } + * + * FIXED POSITION: When multiple write_todos calls happen in a conversation, + * only the FIRST component renders. Subsequent updates just update the + * shared state, and the first component reads from it. + */ +export const WriteTodosToolUI = makeAssistantToolUI({ + toolName: "write_todos", + render: function WriteTodosUI({ args, result, status, toolCallId }) { + const updatePlanState = useSetAtom(updatePlanStateAtom); + const planStates = useAtomValue(planStatesAtom); + + // Check if the THREAD is running + const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); + + // Use result if available, otherwise args (for streaming) + const data = result || args; + const hasTodos = data?.todos && data.todos.length > 0; + + // Fixed title for all plans in conversation + const planTitle = "Plan"; + + // SYNCHRONOUS ownership check + const isOwner = useMemo(() => { + return registerPlanOwner(planTitle, toolCallId); + }, [planTitle, toolCallId]); + + // Get canonical title + const canonicalTitle = useMemo(() => getCanonicalPlanTitle(planTitle), [planTitle]); + + // Register/update the plan state + useEffect(() => { + if (hasTodos) { + const normalizedPlan = parseSerializablePlan({ todos: data.todos }); + updatePlanState({ + id: normalizedPlan.id, + title: canonicalTitle, + todos: normalizedPlan.todos, + toolCallId, + }); + } + }, [data, hasTodos, canonicalTitle, updatePlanState, toolCallId]); + + // Get the current plan state + const currentPlanState = planStates.get(canonicalTitle); + + // If we're NOT the owner, render nothing + if (!isOwner) { + return null; + } + + // Loading state + if (status.type === "running" || status.type === "requires-action") { + if (hasTodos) { + const plan = parseSerializablePlan({ todos: data.todos }); + return ( +
+ + + +
+ ); + } + return ; + } + + // Incomplete/cancelled state + if (status.type === "incomplete") { + if (currentPlanState || hasTodos) { + const plan = currentPlanState || parseSerializablePlan({ todos: data?.todos || [] }); + return ( +
+ + + +
+ ); + } + return null; + } + + // Success - render the plan + const planToRender = + currentPlanState || (hasTodos ? parseSerializablePlan({ todos: data.todos }) : null); + if (!planToRender) { + return ; + } + + return ( +
+ + + +
+ ); + }, +}); + +export { WriteTodosSchema, type WriteTodosData }; diff --git a/surfsense_web/components/ui/sidebar.tsx b/surfsense_web/components/ui/sidebar.tsx index caafa6b6e..46280e1e3 100644 --- a/surfsense_web/components/ui/sidebar.tsx +++ b/surfsense_web/components/ui/sidebar.tsx @@ -449,7 +449,7 @@ function SidebarMenuItem({ className, ...props }: React.ComponentProps<"li">) { } const sidebarMenuButtonVariants = cva( - "peer/menu-button flex w-full items-center gap-2 overflow-hidden rounded-md p-2 text-left text-sm outline-hidden ring-sidebar-ring transition-[width,height,padding] hover:bg-sidebar-accent hover:text-sidebar-accent-foreground focus-visible:ring-2 active:bg-sidebar-accent active:text-sidebar-accent-foreground disabled:pointer-events-none disabled:opacity-50 group-has-data-[sidebar=menu-action]/menu-item:pr-8 aria-disabled:pointer-events-none aria-disabled:opacity-50 data-[active=true]:bg-sidebar-accent data-[active=true]:font-medium data-[active=true]:text-sidebar-accent-foreground data-[state=open]:hover:bg-sidebar-accent data-[state=open]:hover:text-sidebar-accent-foreground group-data-[collapsible=icon]:size-8! group-data-[collapsible=icon]:p-2! [&>span:last-child]:truncate [&>svg]:size-4 [&>svg]:shrink-0", + "peer/menu-button flex w-full items-center gap-2 overflow-hidden rounded-md p-2 text-left text-sm outline-hidden ring-sidebar-ring transition-[width,height,padding] hover:bg-sidebar-accent hover:text-sidebar-accent-foreground focus-visible:ring-2 active:bg-sidebar-accent active:text-sidebar-accent-foreground disabled:pointer-events-none disabled:opacity-50 group-has-data-[sidebar=menu-action]/menu-item:pr-8 aria-disabled:pointer-events-none aria-disabled:opacity-50 data-[active=true]:bg-sidebar-accent data-[active=true]:text-sidebar-accent-foreground data-[state=open]:hover:bg-sidebar-accent data-[state=open]:hover:text-sidebar-accent-foreground group-data-[collapsible=icon]:size-8! group-data-[collapsible=icon]:p-2! [&>span:last-child]:truncate [&>svg]:size-4 [&>svg]:shrink-0", { variants: { variant: { diff --git a/surfsense_web/contracts/types/connector.types.ts b/surfsense_web/contracts/types/connector.types.ts new file mode 100644 index 000000000..4e09ba067 --- /dev/null +++ b/surfsense_web/contracts/types/connector.types.ts @@ -0,0 +1,159 @@ +import { z } from "zod"; +import { paginationQueryParams } from "."; + +export const searchSourceConnectorTypeEnum = z.enum([ + "SERPER_API", + "TAVILY_API", + "SEARXNG_API", + "LINKUP_API", + "BAIDU_SEARCH_API", + "SLACK_CONNECTOR", + "NOTION_CONNECTOR", + "GITHUB_CONNECTOR", + "LINEAR_CONNECTOR", + "DISCORD_CONNECTOR", + "JIRA_CONNECTOR", + "CONFLUENCE_CONNECTOR", + "CLICKUP_CONNECTOR", + "GOOGLE_CALENDAR_CONNECTOR", + "GOOGLE_GMAIL_CONNECTOR", + "AIRTABLE_CONNECTOR", + "LUMA_CONNECTOR", + "ELASTICSEARCH_CONNECTOR", + "WEBCRAWLER_CONNECTOR", + "BOOKSTACK_CONNECTOR", +]); + +export const searchSourceConnector = z.object({ + id: z.number(), + name: z.string(), + connector_type: searchSourceConnectorTypeEnum, + is_indexable: z.boolean(), + last_indexed_at: z.string().nullable(), + config: z.record(z.string(), z.any()), + periodic_indexing_enabled: z.boolean(), + indexing_frequency_minutes: z.number().nullable(), + next_scheduled_at: z.string().nullable(), + search_space_id: z.number(), + user_id: z.string(), + created_at: z.string(), +}); + +/** + * Get connectors + */ +export const getConnectorsRequest = z.object({ + queryParams: paginationQueryParams + .pick({ skip: true, limit: true }) + .extend({ + search_space_id: z.number().or(z.string()).nullish(), + }) + .nullish(), +}); + +export const getConnectorsResponse = z.array(searchSourceConnector); + +/** + * Get connector + */ +export const getConnectorRequest = searchSourceConnector.pick({ id: true }); + +export const getConnectorResponse = searchSourceConnector; + +/** + * Create connector + */ +export const createConnectorRequest = z.object({ + data: searchSourceConnector.pick({ + name: true, + connector_type: true, + is_indexable: true, + last_indexed_at: true, + config: true, + periodic_indexing_enabled: true, + indexing_frequency_minutes: true, + next_scheduled_at: true, + }), + queryParams: z.object({ + search_space_id: z.number().or(z.string()), + }), +}); + +export const createConnectorResponse = searchSourceConnector; + +/** + * Update connector + */ +export const updateConnectorRequest = z.object({ + id: z.number(), + data: searchSourceConnector + .pick({ + name: true, + connector_type: true, + is_indexable: true, + last_indexed_at: true, + config: true, + periodic_indexing_enabled: true, + indexing_frequency_minutes: true, + next_scheduled_at: true, + }) + .partial(), +}); + +export const updateConnectorResponse = searchSourceConnector; + +/** + * Delete connector + */ +export const deleteConnectorRequest = searchSourceConnector.pick({ id: true }); + +export const deleteConnectorResponse = z.object({ + message: z.literal("Search source connector deleted successfully"), +}); + +/** + * Index connector + */ +export const indexConnectorRequest = z.object({ + connector_id: z.number(), + queryParams: z.object({ + search_space_id: z.number().or(z.string()), + start_date: z.string().optional(), + end_date: z.string().optional(), + }), +}); + +export const indexConnectorResponse = z.object({ + message: z.string(), + connector_id: z.number(), + search_space_id: z.number(), + indexing_from: z.string(), + indexing_to: z.string(), +}); + +/** + * List GitHub repositories + */ +export const listGitHubRepositoriesRequest = z.object({ + github_pat: z.string(), +}); + +export const listGitHubRepositoriesResponse = z.array(z.record(z.string(), z.any())); + +// Inferred types +export type SearchSourceConnectorType = z.infer; +export type SearchSourceConnector = z.infer; +export type GetConnectorsRequest = z.infer; +export type GetConnectorsResponse = z.infer; +export type GetConnectorRequest = z.infer; +export type GetConnectorResponse = z.infer; +export type CreateConnectorRequest = z.infer; +export type CreateConnectorResponse = z.infer; +export type UpdateConnectorRequest = z.infer; +export type UpdateConnectorResponse = z.infer; +export type DeleteConnectorRequest = z.infer; +export type DeleteConnectorResponse = z.infer; +export type IndexConnectorRequest = z.infer; +export type IndexConnectorResponse = z.infer; +export type ListGitHubRepositoriesRequest = z.infer; +export type ListGitHubRepositoriesResponse = z.infer; diff --git a/surfsense_web/contracts/types/log.types.ts b/surfsense_web/contracts/types/log.types.ts new file mode 100644 index 000000000..ac81d2d0d --- /dev/null +++ b/surfsense_web/contracts/types/log.types.ts @@ -0,0 +1,134 @@ +import { z } from "zod"; +import { paginationQueryParams } from "."; + +/** + * ENUMS + */ +export const logLevelEnum = z.enum(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]); + +export const logStatusEnum = z.enum(["IN_PROGRESS", "SUCCESS", "FAILED"]); + +/** + * Base log schema + */ +export const log = z.object({ + id: z.number(), + level: logLevelEnum, + status: logStatusEnum, + message: z.string(), + source: z.string().nullable().optional(), + log_metadata: z.record(z.string(), z.any()).nullable().optional(), + created_at: z.string(), + search_space_id: z.number(), +}); + +export const logBase = log.omit({ id: true, created_at: true }); + +/** + * Create log + */ +export const createLogRequest = logBase.extend({ search_space_id: z.number() }); +export const createLogResponse = log; + +/** + * Update log + */ +export const updateLogRequest = logBase.partial(); +export const updateLogResponse = log; + +/** + * Delete log + */ +export const deleteLogRequest = z.object({ id: z.number() }); +export const deleteLogResponse = z.object({ + message: z.string().default("Log deleted successfully"), +}); + +/** + * Get logs (list) + */ +export const logFilters = z.object({ + search_space_id: z.number().optional(), + level: logLevelEnum.optional(), + status: logStatusEnum.optional(), + source: z.string().optional(), + start_date: z.string().optional(), + end_date: z.string().optional(), +}); + +export const getLogsRequest = z.object({ + queryParams: paginationQueryParams + .extend({ + search_space_id: z.number().optional(), + level: logLevelEnum.optional(), + status: logStatusEnum.optional(), + source: z.string().optional(), + start_date: z.string().optional(), + end_date: z.string().optional(), + }) + .nullish(), +}); +export const getLogsResponse = z.array(log); + +/** + * Get single log + */ +export const getLogRequest = z.object({ id: z.number() }); +export const getLogResponse = log; + +/** + * Log summary (used for summary dashboard) + */ +export const logActiveTask = z.object({ + id: z.number(), + task_name: z.string(), + message: z.string(), + started_at: z.string(), + source: z.string().nullable().optional(), + document_id: z.number().nullable().optional(), +}); +export const logFailure = z.object({ + id: z.number(), + task_name: z.string(), + message: z.string(), + failed_at: z.string(), + source: z.string().nullable().optional(), + error_details: z.string().nullable().optional(), +}); +export const logSummary = z.object({ + total_logs: z.number(), + time_window_hours: z.number(), + by_status: z.record(z.string(), z.number()), + by_level: z.record(z.string(), z.number()), + by_source: z.record(z.string(), z.number()), + active_tasks: z.array(logActiveTask), + recent_failures: z.array(logFailure), +}); +export const getLogSummaryRequest = z.object({ + search_space_id: z.number(), + hours: z.number().optional(), +}); +export const getLogSummaryResponse = logSummary; + +/** + * Typescript types + */ +export type Log = z.infer; +export type LogLevelEnum = z.infer; +export type LogStatusEnum = z.infer; +export type LogFilters = z.infer; +export type CreateLogRequest = z.infer; +export type CreateLogResponse = z.infer; +export type UpdateLogRequest = z.infer; +export type UpdateLogResponse = z.infer; +export type DeleteLogRequest = z.infer; +export type DeleteLogResponse = z.infer; +export type GetLogsRequest = z.infer; +export type GetLogsResponse = z.infer; +export type GetLogRequest = z.infer; +export type GetLogResponse = z.infer; +export type LogSummary = z.infer; +export type LogFailure = z.infer; +export type LogActiveTask = z.infer; +export type GetLogSummaryRequest = z.infer; +export type GetLogSummaryResponse = z.infer; diff --git a/surfsense_web/hooks/use-connector-edit-page.ts b/surfsense_web/hooks/use-connector-edit-page.ts index 80a7b4add..05f5abcc2 100644 --- a/surfsense_web/hooks/use-connector-edit-page.ts +++ b/surfsense_web/hooks/use-connector-edit-page.ts @@ -1,8 +1,11 @@ import { zodResolver } from "@hookform/resolvers/zod"; +import { useAtomValue } from "jotai"; import { useRouter } from "next/navigation"; import { useCallback, useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; +import { updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { type EditConnectorFormValues, type EditMode, @@ -11,10 +14,8 @@ import { type GithubRepo, githubPatSchema, } from "@/components/editConnector/types"; -import { - type SearchSourceConnector, - useSearchSourceConnectors, -} from "@/hooks/use-search-source-connectors"; +import type { EnumConnectorName } from "@/contracts/enums/connector"; +import type { SearchSourceConnector } from "@/hooks/use-search-source-connectors"; import { authenticatedFetch } from "@/lib/auth-utils"; const normalizeListInput = (value: unknown): string[] => { @@ -51,11 +52,8 @@ const normalizeBoolean = (value: unknown): boolean | null => { export function useConnectorEditPage(connectorId: number, searchSpaceId: string) { const router = useRouter(); - const { - connectors, - updateConnector, - isLoading: connectorsLoading, - } = useSearchSourceConnectors(false, parseInt(searchSpaceId)); + const { data: connectors = [], isLoading: connectorsLoading } = useAtomValue(connectorsAtom); + const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom); // State managed by the hook const [connector, setConnector] = useState(null); @@ -532,7 +530,13 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) } try { - await updateConnector(connectorId, updatePayload); + await updateConnector({ + id: connectorId, + data: { + ...updatePayload, + connector_type: connector.connector_type as EnumConnectorName, + }, + }); toast.success("Connector updated!"); const newlySavedConfig = updatePayload.config || originalConfig; setOriginalConfig(newlySavedConfig); diff --git a/surfsense_web/hooks/use-connectors.ts b/surfsense_web/hooks/use-connectors.ts deleted file mode 100644 index 211a5e815..000000000 --- a/surfsense_web/hooks/use-connectors.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { authenticatedFetch } from "@/lib/auth-utils"; - -// Types for connector API -export interface ConnectorConfig { - [key: string]: string; -} - -export interface Connector { - id: number; - name: string; - connector_type: string; - config: ConnectorConfig; - created_at: string; - user_id: string; -} - -export interface CreateConnectorRequest { - name: string; - connector_type: string; - config: ConnectorConfig; -} - -// Get connector type display name -export const getConnectorTypeDisplay = (type: string): string => { - const typeMap: Record = { - TAVILY_API: "Tavily API", - SEARXNG_API: "SearxNG", - }; - return typeMap[type] || type; -}; - -// API service for connectors -export const ConnectorService = { - // Create a new connector - async createConnector(data: CreateConnectorRequest): Promise { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(data), - } - ); - - if (!response.ok) { - const errorData = await response.json(); - throw new Error(errorData.detail || "Failed to create connector"); - } - - return response.json(); - }, - - // Get all connectors - async getConnectors(skip = 0, limit = 100): Promise { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors?skip=${skip}&limit=${limit}`, - { method: "GET" } - ); - - if (!response.ok) { - const errorData = await response.json(); - throw new Error(errorData.detail || "Failed to fetch connectors"); - } - - return response.json(); - }, - - // Get a specific connector - async getConnector(connectorId: number): Promise { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/${connectorId}`, - { method: "GET" } - ); - - if (!response.ok) { - const errorData = await response.json(); - throw new Error(errorData.detail || "Failed to fetch connector"); - } - - return response.json(); - }, - - // Update a connector - async updateConnector(connectorId: number, data: CreateConnectorRequest): Promise { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/${connectorId}`, - { - method: "PUT", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(data), - } - ); - - if (!response.ok) { - const errorData = await response.json(); - throw new Error(errorData.detail || "Failed to update connector"); - } - - return response.json(); - }, - - // Delete a connector - async deleteConnector(connectorId: number): Promise { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/${connectorId}`, - { method: "DELETE" } - ); - - if (!response.ok) { - const errorData = await response.json(); - throw new Error(errorData.detail || "Failed to delete connector"); - } - }, -}; diff --git a/surfsense_web/hooks/use-logs.ts b/surfsense_web/hooks/use-logs.ts index cfd161de0..41b2660e5 100644 --- a/surfsense_web/hooks/use-logs.ts +++ b/surfsense_web/hooks/use-logs.ts @@ -1,7 +1,8 @@ "use client"; -import { useCallback, useEffect, useMemo, useState } from "react"; -import { toast } from "sonner"; -import { authenticatedFetch } from "@/lib/auth-utils"; +import { useQuery } from "@tanstack/react-query"; +import { useCallback, useMemo } from "react"; +import { logsApiService } from "@/lib/apis/logs-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; export type LogLevel = "DEBUG" | "INFO" | "WARNING" | "ERROR" | "CRITICAL"; export type LogStatus = "IN_PROGRESS" | "SUCCESS" | "FAILED"; @@ -38,6 +39,7 @@ export interface LogSummary { message: string; started_at: string; source?: string; + document_id?: number; }>; recent_failures: Array<{ id: number; @@ -50,267 +52,96 @@ export interface LogSummary { } export function useLogs(searchSpaceId?: number, filters: LogFilters = {}) { - const [logs, setLogs] = useState([]); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); - // Memoize filters to prevent infinite re-renders const memoizedFilters = useMemo(() => filters, [JSON.stringify(filters)]); const buildQueryParams = useCallback( (customFilters: LogFilters = {}) => { - const params = new URLSearchParams(); + const params: Record = {}; const allFilters = { ...memoizedFilters, ...customFilters }; if (allFilters.search_space_id) { - params.append("search_space_id", allFilters.search_space_id.toString()); + params["search_space_id"] = allFilters.search_space_id.toString(); } if (allFilters.level) { - params.append("level", allFilters.level); + params["level"] = allFilters.level; } if (allFilters.status) { - params.append("status", allFilters.status); + params["status"] = allFilters.status; } if (allFilters.source) { - params.append("source", allFilters.source); + params["source"] = allFilters.source; } if (allFilters.start_date) { - params.append("start_date", allFilters.start_date); + params["start_date"] = allFilters.start_date; } if (allFilters.end_date) { - params.append("end_date", allFilters.end_date); + params["end_date"] = allFilters.end_date; } - return params.toString(); + return params; }, [memoizedFilters] ); - const fetchLogs = useCallback( - async (customFilters: LogFilters = {}, options: { skip?: number; limit?: number } = {}) => { - try { - setLoading(true); - - const params = new URLSearchParams(buildQueryParams(customFilters)); - if (options.skip !== undefined) params.append("skip", options.skip.toString()); - if (options.limit !== undefined) params.append("limit", options.limit.toString()); - - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs?${params}`, - { method: "GET" } - ); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.detail || "Failed to fetch logs"); - } - - const data = await response.json(); - setLogs(data); - setError(null); - return data; - } catch (err: any) { - setError(err.message || "Failed to fetch logs"); - console.error("Error fetching logs:", err); - throw err; - } finally { - setLoading(false); - } - }, - [buildQueryParams] - ); - - // Initial fetch - useEffect(() => { - const initialFilters = searchSpaceId - ? { ...memoizedFilters, search_space_id: searchSpaceId } - : memoizedFilters; - fetchLogs(initialFilters); - }, [searchSpaceId, fetchLogs, memoizedFilters]); - - // Function to refresh the logs list - const refreshLogs = useCallback( - async (customFilters: LogFilters = {}) => { - const finalFilters = searchSpaceId - ? { ...customFilters, search_space_id: searchSpaceId } - : customFilters; - return await fetchLogs(finalFilters); - }, - [searchSpaceId, fetchLogs] - ); - - // Function to create a new log - // Use silent: true to suppress toast notifications (for internal/background operations) - const createLog = useCallback( - async (logData: Omit, options?: { silent?: boolean }) => { - const { silent = false } = options || {}; - try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs`, - { - headers: { "Content-Type": "application/json" }, - method: "POST", - body: JSON.stringify(logData), - } - ); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.detail || "Failed to create log"); - } - - const newLog = await response.json(); - setLogs((prevLogs) => [newLog, ...prevLogs]); - // Only show toast if not silent - if (!silent) { - toast.success("Log created successfully"); - } - return newLog; - } catch (err: any) { - // Only show error toast if not silent - if (!silent) { - toast.error(err.message || "Failed to create log"); - } - console.error("Error creating log:", err); - throw err; - } - }, - [] - ); - - // Function to update a log - const updateLog = useCallback( - async ( - logId: number, - updateData: Partial> - ) => { - try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs/${logId}`, - { - headers: { "Content-Type": "application/json" }, - method: "PUT", - body: JSON.stringify(updateData), - } - ); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.detail || "Failed to update log"); - } - - const updatedLog = await response.json(); - setLogs((prevLogs) => prevLogs.map((log) => (log.id === logId ? updatedLog : log))); - toast.success("Log updated successfully"); - return updatedLog; - } catch (err: any) { - toast.error(err.message || "Failed to update log"); - console.error("Error updating log:", err); - throw err; - } - }, - [] - ); - - // Function to delete a log - const deleteLog = useCallback(async (logId: number) => { - try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs/${logId}`, - { method: "DELETE" } - ); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.detail || "Failed to delete log"); - } - - setLogs((prevLogs) => prevLogs.filter((log) => log.id !== logId)); - toast.success("Log deleted successfully"); - return true; - } catch (err: any) { - toast.error(err.message || "Failed to delete log"); - console.error("Error deleting log:", err); - return false; - } - }, []); - - // Function to get a single log - const getLog = useCallback(async (logId: number) => { - try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs/${logId}`, - { method: "GET" } - ); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.detail || "Failed to fetch log"); - } - - return await response.json(); - } catch (err: any) { - toast.error(err.message || "Failed to fetch log"); - console.error("Error fetching log:", err); - throw err; - } - }, []); + const { + data: logs, + isLoading: loading, + error, + refetch, + } = useQuery({ + queryKey: cacheKeys.logs.withQueryParams({ + search_space_id: searchSpaceId, + skip: 0, + limit: 5, + ...buildQueryParams(filters ?? {}), + }), + queryFn: () => + logsApiService.getLogs({ + queryParams: { + search_space_id: searchSpaceId, + skip: 0, + limit: 5, + ...buildQueryParams(filters ?? {}), + }, + }), + enabled: !!searchSpaceId, + staleTime: 3 * 60 * 1000, + }); return { - logs, + logs: logs ?? [], loading, error, - refreshLogs, - createLog, - updateLog, - deleteLog, - getLog, - fetchLogs, + refreshLogs: refetch, }; } -// Separate hook for log summary -export function useLogsSummary(searchSpaceId: number, hours: number = 24) { - const [summary, setSummary] = useState(null); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); +// Separate hook for log summary with optional polling support for document processing indicator UI +export function useLogsSummary( + searchSpaceId: number, + hours: number = 24, + options: { refetchInterval?: number } = {} +) { + const { + data: summary, + isLoading: loading, + error, + refetch, + } = useQuery({ + queryKey: cacheKeys.logs.summary(searchSpaceId), + queryFn: () => + logsApiService.getLogSummary({ + search_space_id: searchSpaceId, + hours: hours, + }), + enabled: !!searchSpaceId, + staleTime: 3 * 60 * 1000, + // Enable refetch interval for document processing indicator polling + refetchInterval: + options.refetchInterval && options.refetchInterval > 0 ? options.refetchInterval : undefined, + }); - const fetchSummary = useCallback(async () => { - if (!searchSpaceId) return; - - try { - setLoading(true); - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/logs/search-space/${searchSpaceId}/summary?hours=${hours}`, - { method: "GET" } - ); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.detail || "Failed to fetch logs summary"); - } - - const data = await response.json(); - setSummary(data); - setError(null); - return data; - } catch (err: any) { - setError(err.message || "Failed to fetch logs summary"); - console.error("Error fetching logs summary:", err); - throw err; - } finally { - setLoading(false); - } - }, [searchSpaceId, hours]); - - useEffect(() => { - fetchSummary(); - }, [fetchSummary]); - - const refreshSummary = useCallback(() => { - return fetchSummary(); - }, [fetchSummary]); - - return { summary, loading, error, refreshSummary }; + return { summary, loading, error, refreshSummary: refetch }; } diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts index 0f3c17d4e..ff71fe14c 100644 --- a/surfsense_web/lib/apis/base-api.service.ts +++ b/surfsense_web/lib/apis/base-api.service.ts @@ -21,18 +21,23 @@ export type RequestOptions = { }; class BaseApiService { - bearerToken: string; baseUrl: string; noAuthEndpoints: string[] = ["/auth/jwt/login", "/auth/register", "/auth/refresh"]; // Add more endpoints as needed - constructor(bearerToken: string, baseUrl: string) { - this.bearerToken = bearerToken; + // Use a getter to always read fresh token from localStorage + // This ensures the token is always up-to-date after login/logout + get bearerToken(): string { + return typeof window !== "undefined" ? getBearerToken() || "" : ""; + } + + constructor(baseUrl: string) { this.baseUrl = baseUrl; } - setBearerToken(bearerToken: string) { - this.bearerToken = bearerToken; + // Keep for backward compatibility, but token is now always read from localStorage + setBearerToken(_bearerToken: string) { + // No-op: token is now always read fresh from localStorage via the getter } async request( @@ -293,7 +298,4 @@ class BaseApiService { } } -export const baseApiService = new BaseApiService( - typeof window !== "undefined" ? getBearerToken() || "" : "", - process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "" -); +export const baseApiService = new BaseApiService(process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || ""); diff --git a/surfsense_web/lib/apis/connectors-api.service.ts b/surfsense_web/lib/apis/connectors-api.service.ts new file mode 100644 index 000000000..eeee5e6a1 --- /dev/null +++ b/surfsense_web/lib/apis/connectors-api.service.ts @@ -0,0 +1,200 @@ +import { + type CreateConnectorRequest, + createConnectorRequest, + createConnectorResponse, + type DeleteConnectorRequest, + deleteConnectorRequest, + deleteConnectorResponse, + type GetConnectorRequest, + type GetConnectorsRequest, + getConnectorRequest, + getConnectorResponse, + getConnectorsRequest, + getConnectorsResponse, + type IndexConnectorRequest, + indexConnectorRequest, + indexConnectorResponse, + type ListGitHubRepositoriesRequest, + listGitHubRepositoriesRequest, + listGitHubRepositoriesResponse, + type UpdateConnectorRequest, + updateConnectorRequest, + updateConnectorResponse, +} from "@/contracts/types/connector.types"; +import { ValidationError } from "../error"; +import { baseApiService } from "./base-api.service"; + +class ConnectorsApiService { + /** + * Get all connectors for a search space + */ + getConnectors = async (request: GetConnectorsRequest) => { + const parsedRequest = getConnectorsRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + // Transform query params to be string values + const transformedQueryParams = parsedRequest.data.queryParams + ? Object.fromEntries( + Object.entries(parsedRequest.data.queryParams).map(([k, v]) => { + return [k, String(v)]; + }) + ) + : undefined; + + const queryParams = transformedQueryParams + ? new URLSearchParams(transformedQueryParams).toString() + : ""; + + return baseApiService.get( + `/api/v1/search-source-connectors?${queryParams}`, + getConnectorsResponse + ); + }; + + /** + * Get a single connector by ID + */ + getConnector = async (request: GetConnectorRequest) => { + const parsedRequest = getConnectorRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + return baseApiService.get( + `/api/v1/search-source-connectors/${request.id}`, + getConnectorResponse + ); + }; + + /** + * Create a new connector + */ + createConnector = async (request: CreateConnectorRequest) => { + const parsedRequest = createConnectorRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + const { data, queryParams } = parsedRequest.data; + + // Transform query params to be string values + const transformedQueryParams = Object.fromEntries( + Object.entries(queryParams).map(([k, v]) => { + return [k, String(v)]; + }) + ); + + const queryString = new URLSearchParams(transformedQueryParams).toString(); + + return baseApiService.post( + `/api/v1/search-source-connectors?${queryString}`, + createConnectorResponse, + { + body: data, + } + ); + }; + + /** + * Update an existing connector + */ + updateConnector = async (request: UpdateConnectorRequest) => { + const parsedRequest = updateConnectorRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + const { id, data } = parsedRequest.data; + + return baseApiService.put(`/api/v1/search-source-connectors/${id}`, updateConnectorResponse, { + body: data, + }); + }; + + /** + * Delete a connector + */ + deleteConnector = async (request: DeleteConnectorRequest) => { + const parsedRequest = deleteConnectorRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + return baseApiService.delete( + `/api/v1/search-source-connectors/${request.id}`, + deleteConnectorResponse + ); + }; + + /** + * Index connector content + */ + indexConnector = async (request: IndexConnectorRequest) => { + const parsedRequest = indexConnectorRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + const { connector_id, queryParams } = parsedRequest.data; + + // Transform query params to be string values + const transformedQueryParams = Object.fromEntries( + Object.entries(queryParams).map(([k, v]) => { + return [k, String(v)]; + }) + ); + + const queryString = new URLSearchParams(transformedQueryParams).toString(); + + return baseApiService.post( + `/api/v1/search-source-connectors/${connector_id}/index?${queryString}`, + indexConnectorResponse + ); + }; + + /** + * List GitHub repositories using a Personal Access Token + */ + listGitHubRepositories = async (request: ListGitHubRepositoriesRequest) => { + const parsedRequest = listGitHubRepositoriesRequest.safeParse(request); + + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + + const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + + return baseApiService.post(`/api/v1/github/repositories`, listGitHubRepositoriesResponse, { + body: parsedRequest.data, + }); + }; +} + +export const connectorsApiService = new ConnectorsApiService(); diff --git a/surfsense_web/lib/apis/logs-api.service.ts b/surfsense_web/lib/apis/logs-api.service.ts new file mode 100644 index 000000000..115f50497 --- /dev/null +++ b/surfsense_web/lib/apis/logs-api.service.ts @@ -0,0 +1,128 @@ +import { + type CreateLogRequest, + createLogRequest, + createLogResponse, + type DeleteLogRequest, + deleteLogRequest, + deleteLogResponse, + type GetLogRequest, + type GetLogSummaryRequest, + type GetLogsRequest, + getLogRequest, + getLogResponse, + getLogSummaryRequest, + getLogSummaryResponse, + getLogsRequest, + getLogsResponse, + type Log, + log, + type UpdateLogRequest, + updateLogRequest, + updateLogResponse, +} from "@/contracts/types/log.types"; +import { ValidationError } from "../error"; +import { baseApiService } from "./base-api.service"; + +class LogsApiService { + /** + * Get a list of logs with optional filtering and pagination + */ + getLogs = async (request: GetLogsRequest) => { + const parsedRequest = getLogsRequest.safeParse(request); + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + // Transform query params to be string values + const transformedQueryParams = parsedRequest.data.queryParams + ? Object.fromEntries( + Object.entries(parsedRequest.data.queryParams).map(([k, v]) => { + // Handle array values (document_type) + if (Array.isArray(v)) { + return [k, v.join(",")]; + } + return [k, String(v)]; + }) + ) + : undefined; + + const queryParams = transformedQueryParams + ? new URLSearchParams(transformedQueryParams).toString() + : ""; + return baseApiService.get(`/api/v1/logs?${queryParams}`, getLogsResponse); + }; + + /** + * Get a single log by ID + */ + getLog = async (request: GetLogRequest) => { + const parsedRequest = getLogRequest.safeParse(request); + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + return baseApiService.get(`/api/v1/logs/${request.id}`, getLogResponse); + }; + + /** + * Create a log entry + */ + createLog = async (request: CreateLogRequest) => { + const parsedRequest = createLogRequest.safeParse(request); + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + return baseApiService.post(`/api/v1/logs`, createLogResponse, { + body: parsedRequest.data, + }); + }; + + /** + * Update a log entry + */ + updateLog = async (logId: number, request: UpdateLogRequest) => { + const parsedRequest = updateLogRequest.safeParse(request); + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + return baseApiService.put(`/api/v1/logs/${logId}`, updateLogResponse, { + body: parsedRequest.data, + }); + }; + + /** + * Delete a log entry + */ + deleteLog = async (request: DeleteLogRequest) => { + const parsedRequest = deleteLogRequest.safeParse(request); + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + return baseApiService.delete(`/api/v1/logs/${parsedRequest.data.id}`, deleteLogResponse); + }; + + /** + * Get summary for logs by search space + */ + getLogSummary = async (request: GetLogSummaryRequest) => { + const parsedRequest = getLogSummaryRequest.safeParse(request); + if (!parsedRequest.success) { + console.error("Invalid request:", parsedRequest.error); + const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); + throw new ValidationError(`Invalid request: ${errorMessage}`); + } + const { search_space_id, hours } = parsedRequest.data; + const url = `/api/v1/logs/search-space/${search_space_id}/summary${hours ? `?hours=${hours}` : ""}`; + return baseApiService.get(url, getLogSummaryResponse); + }; +} + +export const logsApiService = new LogsApiService(); diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index 8e0f1431e..7722ec01e 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -1,4 +1,6 @@ +import type { GetConnectorsRequest } from "@/contracts/types/connector.types"; import type { GetDocumentsRequest } from "@/contracts/types/document.types"; +import type { GetLogsRequest } from "@/contracts/types/log.types"; import type { GetSearchSpacesRequest } from "@/contracts/types/search-space.types"; export const cacheKeys = { @@ -18,6 +20,13 @@ export const cacheKeys = { typeCounts: (searchSpaceId?: string) => ["documents", "type-counts", searchSpaceId] as const, byChunk: (chunkId: string) => ["documents", "by-chunk", chunkId] as const, }, + logs: { + list: (searchSpaceId?: number | string) => ["logs", "list", searchSpaceId] as const, + detail: (logId: number | string) => ["logs", "detail", logId] as const, + summary: (searchSpaceId?: number | string) => ["logs", "summary", searchSpaceId] as const, + withQueryParams: (queries: GetLogsRequest["queryParams"]) => + ["logs", "with-query-params", ...(queries ? Object.values(queries) : [])] as const, + }, newLLMConfigs: { all: (searchSpaceId: number) => ["new-llm-configs", searchSpaceId] as const, byId: (configId: number) => ["new-llm-configs", "detail", configId] as const, @@ -52,4 +61,11 @@ export const cacheKeys = { all: (searchSpaceId: string) => ["invites", searchSpaceId] as const, info: (inviteCode: string) => ["invites", "info", inviteCode] as const, }, + connectors: { + all: (searchSpaceId: string) => ["connectors", searchSpaceId] as const, + withQueryParams: (queries: GetConnectorsRequest["queryParams"]) => + ["connectors", ...(queries ? Object.values(queries) : [])] as const, + byId: (connectorId: string) => ["connector", connectorId] as const, + index: () => ["connector", "index"] as const, + }, }; diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json index f70c854e0..0fa6e461b 100644 --- a/surfsense_web/messages/en.json +++ b/surfsense_web/messages/en.json @@ -267,7 +267,11 @@ "content_summary": "Content Summary", "view_full": "View Full Content", "filter_placeholder": "Filter by title...", - "rows_per_page": "Rows per page" + "rows_per_page": "Rows per page", + "refresh": "Refresh", + "refresh_success": "Documents refreshed", + "processing_documents": "Processing documents...", + "active_tasks_count": "{count} active task(s)" }, "add_connector": { "title": "Connect Your Tools", diff --git a/surfsense_web/messages/zh.json b/surfsense_web/messages/zh.json index 483a10a10..625c8a31e 100644 --- a/surfsense_web/messages/zh.json +++ b/surfsense_web/messages/zh.json @@ -267,7 +267,11 @@ "content_summary": "内容摘要", "view_full": "查看完整内容", "filter_placeholder": "按标题筛选...", - "rows_per_page": "每页行数" + "rows_per_page": "每页行数", + "refresh": "刷新", + "refresh_success": "文档已刷新", + "processing_documents": "正在处理文档...", + "active_tasks_count": "{count} 个正在进行的工作项" }, "add_connector": { "title": "连接您的工具", diff --git a/surfsense_web/package.json b/surfsense_web/package.json index 5d602c2ab..7419f55d3 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -30,7 +30,6 @@ "@blocknote/react": "^0.45.0", "@blocknote/server-util": "^0.45.0", "@hookform/resolvers": "^5.2.2", - "@next/third-parties": "^16.1.0", "@number-flow/react": "^0.5.10", "@posthog/react": "^1.5.2", "@radix-ui/react-accordion": "^1.2.11", diff --git a/surfsense_web/pnpm-lock.yaml b/surfsense_web/pnpm-lock.yaml index a94b63c0d..f11cb77b9 100644 --- a/surfsense_web/pnpm-lock.yaml +++ b/surfsense_web/pnpm-lock.yaml @@ -35,9 +35,6 @@ importers: '@hookform/resolvers': specifier: ^5.2.2 version: 5.2.2(react-hook-form@7.69.0(react@19.2.3)) - '@next/third-parties': - specifier: ^16.1.0 - version: 16.1.0(next@16.1.0(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) '@number-flow/react': specifier: ^0.5.10 version: 0.5.10(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -1383,12 +1380,6 @@ packages: cpu: [x64] os: [win32] - '@next/third-parties@16.1.0': - resolution: {integrity: sha512-VxD1UxwXNgCnDDBW+oinysZORkzir2B/MvCYF8S02r78VnGr37cbkMlM0LESrE9Nc/qlo2bLBgvpvnyOy4vleg==} - peerDependencies: - next: ^13.0.0 || ^14.0.0 || ^15.0.0 || ^16.0.0-beta.0 - react: ^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0 - '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -5882,9 +5873,6 @@ packages: resolution: {integrity: sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==} engines: {node: '>=6'} - third-party-capital@1.0.20: - resolution: {integrity: sha512-oB7yIimd8SuGptespDAZnNkzIz+NWaJCu2RMsbs4Wmp9zSDUM8Nhi3s2OOcqYuv3mN4hitXc8DVx+LyUmbUDiA==} - throttleit@2.1.0: resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==} engines: {node: '>=18'} @@ -7214,12 +7202,6 @@ snapshots: '@next/swc-win32-x64-msvc@16.1.0': optional: true - '@next/third-parties@16.1.0(next@16.1.0(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)': - dependencies: - next: 16.1.0(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) - react: 19.2.3 - third-party-capital: 1.0.20 - '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -12560,8 +12542,6 @@ snapshots: tapable@2.3.0: {} - third-party-capital@1.0.20: {} - throttleit@2.1.0: {} tinyexec@1.0.2: {}