diff --git a/README.md b/README.md index 9b5d78c3e..9fe07eb35 100644 --- a/README.md +++ b/README.md @@ -136,14 +136,6 @@ Check out our public roadmap and contribute your ideas or feedback: **View the Roadmap:** [SurfSense Roadmap on GitHub Projects](https://github.com/users/MODSetter/projects/2) -## ⚠️ Important Announcement - -**AWS and Vercel are currently experiencing outages.** We deployed a major update to SurfSense last night and have updated our documentation accordingly with important setup and configuration changes. Unfortunately, these documentation updates cannot be deployed to our main site (surfsense.com) due to the ongoing outages. - -**Please view our documentation directly on GitHub:** -📚 [SurfSense Documentation](https://github.com/MODSetter/SurfSense/tree/main/surfsense_web/content/docs) - -We apologize for any inconvenience and appreciate your patience! ## How to get started? diff --git a/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py b/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py index c9966599c..444d8239b 100644 --- a/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py +++ b/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py @@ -55,30 +55,45 @@ def upgrade() -> None: # ===== STEP 2: Populate search_space_id with user's first search space ===== # This ensures existing LLM configs are assigned to a valid search space - op.execute( - """ - UPDATE llm_configs lc - SET search_space_id = ( - SELECT id - FROM searchspaces ss - WHERE ss.user_id = lc.user_id - ORDER BY ss.created_at ASC - LIMIT 1 + # Only run this if user_id column exists on llm_configs + if "user_id" in llm_config_columns: + op.execute( + """ + UPDATE llm_configs lc + SET search_space_id = ( + SELECT id + FROM searchspaces ss + WHERE ss.user_id = lc.user_id + ORDER BY ss.created_at ASC + LIMIT 1 + ) + WHERE search_space_id IS NULL AND user_id IS NOT NULL + """ ) - WHERE search_space_id IS NULL AND user_id IS NOT NULL - """ - ) # ===== STEP 3: Make search_space_id NOT NULL and add FK constraint ===== - op.alter_column( - "llm_configs", - "search_space_id", - nullable=False, + # Check if there are any rows with NULL search_space_id + # If llm_configs table is empty or all rows have search_space_id, we can proceed + result = conn.execute( + sa.text("SELECT COUNT(*) FROM llm_configs WHERE search_space_id IS NULL") ) + null_count = result.scalar() - # Add foreign key constraint + if null_count == 0 or "user_id" in llm_config_columns: + # Safe to make NOT NULL + op.alter_column( + "llm_configs", + "search_space_id", + nullable=False, + ) + else: + # If there are NULL values and no user_id to migrate from, skip making it NOT NULL + # This would happen if llm_configs already exists without user_id + pass + + # Add foreign key constraint only if search_space_id is NOT NULL foreign_keys = [fk["name"] for fk in inspector.get_foreign_keys("llm_configs")] - if "fk_llm_configs_search_space_id" not in foreign_keys: + if "fk_llm_configs_search_space_id" not in foreign_keys and null_count == 0: op.create_foreign_key( "fk_llm_configs_search_space_id", "llm_configs", diff --git a/surfsense_backend/alembic/versions/32_add_periodic_indexing_fields.py b/surfsense_backend/alembic/versions/32_add_periodic_indexing_fields.py new file mode 100644 index 000000000..30656739a --- /dev/null +++ b/surfsense_backend/alembic/versions/32_add_periodic_indexing_fields.py @@ -0,0 +1,94 @@ +"""Add periodic indexing fields to search_source_connectors + +Revision ID: 32 +Revises: 31 + +Changes: +1. Add periodic_indexing_enabled column (Boolean, default False) +2. Add indexing_frequency_minutes column (Integer, nullable) +3. Add next_scheduled_at column (TIMESTAMP with timezone, nullable) +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "32" +down_revision: str | None = "31" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Add periodic indexing fields to search_source_connectors table.""" + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns + connector_columns = [ + col["name"] for col in inspector.get_columns("search_source_connectors") + ] + + # Add periodic_indexing_enabled column if it doesn't exist + if "periodic_indexing_enabled" not in connector_columns: + op.add_column( + "search_source_connectors", + sa.Column( + "periodic_indexing_enabled", + sa.Boolean(), + nullable=False, + server_default="false", + ), + ) + + # Add indexing_frequency_minutes column if it doesn't exist + if "indexing_frequency_minutes" not in connector_columns: + op.add_column( + "search_source_connectors", + sa.Column( + "indexing_frequency_minutes", + sa.Integer(), + nullable=True, + ), + ) + + # Add next_scheduled_at column if it doesn't exist + if "next_scheduled_at" not in connector_columns: + op.add_column( + "search_source_connectors", + sa.Column( + "next_scheduled_at", + sa.TIMESTAMP(timezone=True), + nullable=True, + ), + ) + + +def downgrade() -> None: + """Remove periodic indexing fields from search_source_connectors table.""" + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns + connector_columns = [ + col["name"] for col in inspector.get_columns("search_source_connectors") + ] + + # Drop columns if they exist + if "next_scheduled_at" in connector_columns: + op.drop_column("search_source_connectors", "next_scheduled_at") + + if "indexing_frequency_minutes" in connector_columns: + op.drop_column("search_source_connectors", "indexing_frequency_minutes") + + if "periodic_indexing_enabled" in connector_columns: + op.drop_column("search_source_connectors", "periodic_indexing_enabled") diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index db5ea73ea..5183adc93 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -285,6 +285,11 @@ class SearchSourceConnector(BaseModel, TimestampMixin): last_indexed_at = Column(TIMESTAMP(timezone=True), nullable=True) config = Column(JSON, nullable=False) + # Periodic indexing fields + periodic_indexing_enabled = Column(Boolean, nullable=False, default=False) + indexing_frequency_minutes = Column(Integer, nullable=True) + next_scheduled_at = Column(TIMESTAMP(timezone=True), nullable=True) + search_space_id = Column( Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False ) diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index d0ee44f51..4a6774113 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -11,7 +11,7 @@ Note: Each search space can have only one connector of each type per user (based """ import logging -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from typing import Any from fastapi import APIRouter, Depends, HTTPException, Query @@ -124,8 +124,22 @@ async def create_search_source_connector( status_code=409, detail=f"A connector with type {connector.connector_type} already exists in this search space. Each search space can have only one connector of each type per user.", ) + + # Prepare connector data + connector_data = connector.model_dump() + + # Automatically set next_scheduled_at if periodic indexing is enabled + if ( + connector.periodic_indexing_enabled + and connector.indexing_frequency_minutes + and connector.next_scheduled_at is None + ): + connector_data["next_scheduled_at"] = datetime.now(UTC) + timedelta( + minutes=connector.indexing_frequency_minutes + ) + db_connector = SearchSourceConnector( - **connector.model_dump(), search_space_id=search_space_id, user_id=user.id + **connector_data, search_space_id=search_space_id, user_id=user.id ) session.add(db_connector) await session.commit() @@ -224,6 +238,50 @@ async def update_search_source_connector( # Convert the sparse update data (only fields present in request) to a dict update_data = connector_update.model_dump(exclude_unset=True) + # Validate periodic indexing fields + # Get the effective values after update + effective_is_indexable = update_data.get("is_indexable", db_connector.is_indexable) + effective_periodic_enabled = update_data.get( + "periodic_indexing_enabled", db_connector.periodic_indexing_enabled + ) + effective_frequency = update_data.get( + "indexing_frequency_minutes", db_connector.indexing_frequency_minutes + ) + + # Validate periodic indexing configuration + if effective_periodic_enabled: + if not effective_is_indexable: + raise HTTPException( + status_code=422, + detail="periodic_indexing_enabled can only be True for indexable connectors", + ) + if effective_frequency is None: + raise HTTPException( + status_code=422, + detail="indexing_frequency_minutes is required when periodic_indexing_enabled is True", + ) + if effective_frequency <= 0: + raise HTTPException( + status_code=422, + detail="indexing_frequency_minutes must be greater than 0", + ) + + # Automatically set next_scheduled_at if not provided and periodic indexing is being enabled + if ( + "periodic_indexing_enabled" in update_data + or "indexing_frequency_minutes" in update_data + ) and "next_scheduled_at" not in update_data: + # Schedule the next indexing based on the frequency + update_data["next_scheduled_at"] = datetime.now(UTC) + timedelta( + minutes=effective_frequency + ) + elif ( + effective_periodic_enabled is False + and "periodic_indexing_enabled" in update_data + ): + # If disabling periodic indexing, clear the next_scheduled_at + update_data["next_scheduled_at"] = None + # Special handling for 'config' field if "config" in update_data: incoming_config = update_data["config"] # Config data from the request diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index b84eeb07d..1e8a7a38d 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -2,7 +2,7 @@ import uuid from datetime import datetime from typing import Any -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, field_validator, model_validator from app.db import SearchSourceConnectorType from app.utils.validators import validate_connector_config @@ -16,6 +16,9 @@ class SearchSourceConnectorBase(BaseModel): is_indexable: bool last_indexed_at: datetime | None = None config: dict[str, Any] + periodic_indexing_enabled: bool = False + indexing_frequency_minutes: int | None = None + next_scheduled_at: datetime | None = None @field_validator("config") @classmethod @@ -25,6 +28,22 @@ class SearchSourceConnectorBase(BaseModel): connector_type = values.data.get("connector_type") return validate_connector_config(connector_type, config) + @model_validator(mode="after") + def validate_periodic_indexing(self): + """Validate that periodic indexing configuration is consistent.""" + if self.periodic_indexing_enabled: + if not self.is_indexable: + raise ValueError( + "periodic_indexing_enabled can only be True for indexable connectors" + ) + if self.indexing_frequency_minutes is None: + raise ValueError( + "indexing_frequency_minutes is required when periodic_indexing_enabled is True" + ) + if self.indexing_frequency_minutes <= 0: + raise ValueError("indexing_frequency_minutes must be greater than 0") + return self + class SearchSourceConnectorCreate(SearchSourceConnectorBase): pass @@ -36,6 +55,9 @@ class SearchSourceConnectorUpdate(BaseModel): is_indexable: bool | None = None last_indexed_at: datetime | None = None config: dict[str, Any] | None = None + periodic_indexing_enabled: bool | None = None + indexing_frequency_minutes: int | None = None + next_scheduled_at: datetime | None = None class SearchSourceConnectorRead(SearchSourceConnectorBase, IDModel, TimestampModel): diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index eb1d53895..3537885ca 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -1,7 +1,15 @@ "use client"; import { format } from "date-fns"; -import { Calendar as CalendarIcon, Edit, Plus, RefreshCw, Trash2 } from "lucide-react"; +import { + Calendar as CalendarIcon, + Clock, + Edit, + Loader2, + Plus, + RefreshCw, + Trash2, +} from "lucide-react"; import { motion } from "motion/react"; import { useParams, useRouter } from "next/navigation"; import { useEffect, useState } from "react"; @@ -28,8 +36,17 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; import { Table, TableBody, @@ -64,7 +81,7 @@ export default function ConnectorsPage() { const searchSpaceId = params.search_space_id as string; const today = new Date(); - const { connectors, isLoading, error, deleteConnector, indexConnector } = + const { connectors, isLoading, error, deleteConnector, indexConnector, updateConnector } = useSearchSourceConnectors(false, parseInt(searchSpaceId)); const [connectorToDelete, setConnectorToDelete] = useState(null); const [indexingConnectorId, setIndexingConnectorId] = useState(null); @@ -75,6 +92,16 @@ export default function ConnectorsPage() { const [startDate, setStartDate] = useState(undefined); const [endDate, setEndDate] = useState(undefined); + // Periodic indexing state + const [periodicDialogOpen, setPeriodicDialogOpen] = useState(false); + const [selectedConnectorForPeriodic, setSelectedConnectorForPeriodic] = useState( + null + ); + const [periodicEnabled, setPeriodicEnabled] = useState(false); + const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); + const [customFrequency, setCustomFrequency] = useState(""); + const [isSavingPeriodic, setIsSavingPeriodic] = useState(false); + useEffect(() => { if (error) { toast.error("Failed to load connectors"); @@ -141,6 +168,84 @@ export default function ConnectorsPage() { } }; + // Handle opening periodic indexing dialog + const handleOpenPeriodicDialog = (connectorId: number) => { + const connector = connectors.find((c) => c.id === connectorId); + if (!connector) return; + + setSelectedConnectorForPeriodic(connectorId); + setPeriodicEnabled(connector.periodic_indexing_enabled); + + if (connector.indexing_frequency_minutes) { + // Check if it's a preset value + const presetValues = ["15", "60", "360", "720", "1440", "10080"]; + if (presetValues.includes(connector.indexing_frequency_minutes.toString())) { + setFrequencyMinutes(connector.indexing_frequency_minutes.toString()); + setCustomFrequency(""); + } else { + setFrequencyMinutes("custom"); + setCustomFrequency(connector.indexing_frequency_minutes.toString()); + } + } else { + setFrequencyMinutes("1440"); + setCustomFrequency(""); + } + + setPeriodicDialogOpen(true); + }; + + // Handle saving periodic indexing configuration + const handleSavePeriodicIndexing = async () => { + if (selectedConnectorForPeriodic === null) return; + + const connector = connectors.find((c) => c.id === selectedConnectorForPeriodic); + if (!connector) return; + + setIsSavingPeriodic(true); + try { + // Determine the frequency value + let frequency: number | null = null; + if (periodicEnabled) { + if (frequencyMinutes === "custom") { + frequency = parseInt(customFrequency, 10); + if (isNaN(frequency) || frequency <= 0) { + toast.error("Please enter a valid frequency in minutes"); + setIsSavingPeriodic(false); + return; + } + } else { + frequency = parseInt(frequencyMinutes, 10); + } + } + + await updateConnector(selectedConnectorForPeriodic, { + periodic_indexing_enabled: periodicEnabled, + indexing_frequency_minutes: frequency, + }); + + toast.success( + periodicEnabled + ? "Periodic indexing enabled successfully" + : "Periodic indexing disabled successfully" + ); + setPeriodicDialogOpen(false); + } catch (error) { + console.error("Error updating periodic indexing:", error); + toast.error(error instanceof Error ? error.message : "Failed to update periodic indexing"); + } finally { + setIsSavingPeriodic(false); + setSelectedConnectorForPeriodic(null); + } + }; + + // Format frequency for display + const formatFrequency = (minutes: number): string => { + if (minutes < 60) return `${minutes}m`; + if (minutes < 1440) return `${Math.floor(minutes / 60)}h`; + if (minutes < 10080) return `${Math.floor(minutes / 1440)}d`; + return `${Math.floor(minutes / 10080)}w`; + }; + return (
Name Type Last Indexed + Periodic Actions @@ -206,6 +312,41 @@ export default function ConnectorsPage() { ? formatDateTime(connector.last_indexed_at) : "Not indexable"} + + {connector.is_indexable ? ( + connector.periodic_indexing_enabled ? ( + + + +
+ + + {connector.indexing_frequency_minutes + ? formatFrequency(connector.indexing_frequency_minutes) + : "Enabled"} + +
+ + +

+ Runs every {connector.indexing_frequency_minutes} minutes + {connector.next_scheduled_at && ( + <> +
+ Next: {formatDateTime(connector.next_scheduled_at)} + + )} +

+ + + + ) : ( + Disabled + ) + ) : ( + - + )} +
{connector.is_indexable && ( @@ -256,6 +397,25 @@ export default function ConnectorsPage() {
)} + {connector.is_indexable && ( + + + + + + +

Configure Periodic Indexing

+ + + + )} + + + +
); } diff --git a/surfsense_web/hooks/use-search-source-connectors.ts b/surfsense_web/hooks/use-search-source-connectors.ts index 5a45fd761..752eda978 100644 --- a/surfsense_web/hooks/use-search-source-connectors.ts +++ b/surfsense_web/hooks/use-search-source-connectors.ts @@ -10,6 +10,9 @@ export interface SearchSourceConnector { search_space_id: number; user_id?: string; created_at?: string; + periodic_indexing_enabled: boolean; + indexing_frequency_minutes: number | null; + next_scheduled_at: string | null; } export interface ConnectorSourceItem {