diff --git a/surfsense_backend/app/connectors/elasticsearch_connector.py b/surfsense_backend/app/connectors/elasticsearch_connector.py index 930d140ce..8a8c259e1 100644 --- a/surfsense_backend/app/connectors/elasticsearch_connector.py +++ b/surfsense_backend/app/connectors/elasticsearch_connector.py @@ -31,35 +31,47 @@ class ElasticsearchConnector: async def connect(self) -> bool: """Establish connection to Elasticsearch""" try: + # Build the URL with proper scheme + scheme = "https" if self.config.get("ssl_enabled", True) else "http" + port = self.config.get("port", 443 if scheme == "https" else 9200) + hostname = self.config["hostname"] + + # Construct the full URL + url = f"{scheme}://{hostname}:{port}" + connection_params = { - "hosts": [f"{self.config.hostname}:{self.config.port}"], - "verify_certs": self.config.ssl_enabled - if hasattr(self.config, "ssl_enabled") - else True, + "hosts": [url], + "verify_certs": self.config.get("ssl_enabled", True), "request_timeout": 30, } - # Add authentication if provided + # Handle different authentication methods + auth_method = self.config.get("auth_method", "none") + if ( - hasattr(self.config, "username") - and hasattr(self.config, "password") - and self.config.username - and self.config.password + auth_method == "basic" + and self.config.get("username") + and self.config.get("password") ): connection_params["basic_auth"] = ( - self.config.username, - self.config.password, + self.config["username"], + self.config["password"], ) + elif auth_method == "api_key" and self.config.get("api_key"): + api_key = self.config["api_key"] + if api_key.startswith("ApiKey "): + connection_params["headers"] = {"Authorization": api_key} + else: + connection_params["api_key"] = api_key - # Add API key authentication if provided - if hasattr(self.config, "api_key") and self.config.api_key: - connection_params["api_key"] = self.config.api_key - + logger.info(f"Connecting to Elasticsearch at {url}") self.client = AsyncElasticsearch(**connection_params) # Test connection - await self.client.info() - logger.info("Successfully connected to Elasticsearch") + info = await self.client.info() + logger.info( + f"Successfully connected to Elasticsearch cluster: {info.get('cluster_name', 'Unknown')}" + ) return True except (ConnectionError, AuthenticationException) as e: @@ -92,6 +104,9 @@ class ElasticsearchConnector: "cluster_name": info.get("cluster_name"), "version": info.get("version", {}).get("number"), "indices_count": len(indices) if indices else 0, + "indices": [ + idx["index"] for idx in indices if not idx["index"].startswith(".") + ], } except Exception as e: return {"success": False, "error": str(e)} diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index f6c377c46..1c7e3505f 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -5,7 +5,6 @@ from .airtable_add_connector_route import ( ) from .chats_routes import router as chats_router from .documents_routes import router as documents_router -from .elasticsearch_add_connector_route import router as elasticsearch_router from .google_calendar_add_connector_route import ( router as google_calendar_add_connector_router, ) @@ -25,7 +24,6 @@ router.include_router(search_spaces_router) router.include_router(documents_router) router.include_router(podcasts_router) router.include_router(chats_router) -router.include_router(elasticsearch_router) router.include_router(search_source_connectors_router) router.include_router(google_calendar_add_connector_router) router.include_router(google_gmail_add_connector_router) diff --git a/surfsense_backend/app/routes/elasticsearch_add_connector_route.py b/surfsense_backend/app/routes/elasticsearch_add_connector_route.py deleted file mode 100644 index 4a6f654cb..000000000 --- a/surfsense_backend/app/routes/elasticsearch_add_connector_route.py +++ /dev/null @@ -1,392 +0,0 @@ -import contextlib -import logging -from typing import Any - -from fastapi import APIRouter, Depends, HTTPException, status -from pydantic import BaseModel -from sqlalchemy import select -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession - -from app.connectors.elasticsearch_connector import ElasticsearchConnector -from app.db import ( - SearchSourceConnector, - SearchSourceConnectorType, - User, - get_async_session, -) -from app.users import current_active_user - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/elasticsearch", tags=["elasticsearch"]) - - -class ElasticsearchConnectorConfig(BaseModel): - hostname: str - port: int = 9200 - username: str | None = None - password: str | None = None - api_key: str | None = None - ssl_enabled: bool = True - indices: list[str] | None = None - query: str = "*" - search_fields: list[str] | None = None - max_documents: int = 1000 - - -@router.post("/add-connector") -async def add_elasticsearch_connector( - connector_data: ElasticsearchConnectorConfig, - current_user: User = Depends(current_active_user), - session: AsyncSession = Depends(get_async_session), -) -> dict[str, Any]: - """Add a new Elasticsearch connector for the current user""" - - try: - elasticsearch_connector = ElasticsearchConnector(connector_data) - connection_test = await elasticsearch_connector.test_connection() - if not connection_test.get("success"): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Failed to connect to Elasticsearch: {connection_test.get('error')}", - ) - except Exception as e: - logger.error(f"Error initializing or testing Elasticsearch connector: {e}") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Failed to initialize or test Elasticsearch connector: {e!s}", - ) from e - finally: - with contextlib.suppress(Exception): - await elasticsearch_connector.disconnect() - - try: - # Check if connector already exists for this user - existing_connector_result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.user_id == current_user.id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR, - ) - ) - existing_connector = existing_connector_result.scalars().first() - - if existing_connector: - # Update existing connector - existing_connector.config = connector_data.model_dump() - existing_connector.name = ( - f"Elasticsearch - {connector_data.hostname}:{connector_data.port}" - ) - existing_connector.is_indexable = True - logger.info( - f"Updated existing Elasticsearch connector for user {current_user.id}" - ) - else: - # Create new connector - new_connector = SearchSourceConnector( - name=f"Elasticsearch - {connector_data.hostname}:{connector_data.port}", - connector_type=SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR, - is_indexable=True, - config=connector_data.model_dump(), - user_id=current_user.id, - ) - session.add(new_connector) - logger.info( - f"Created new Elasticsearch connector for user {current_user.id}" - ) - - await session.commit() - - # Get the connector (either existing or new) to return - if existing_connector: - connector = existing_connector - else: - await session.refresh(new_connector) - connector = new_connector - - logger.info( - f"Successfully saved Elasticsearch connector for user {current_user.id}" - ) - - return { - "success": True, - "message": "Elasticsearch connector added successfully", - "connector": { - "id": connector.id, - "name": connector.name, - "connector_type": connector.connector_type.value, - "is_indexable": connector.is_indexable, - "created_at": connector.created_at.isoformat() - if connector.created_at - else None, - "last_indexed_at": connector.last_indexed_at.isoformat() - if connector.last_indexed_at - else None, - }, - } - - except IntegrityError as e: - await session.rollback() - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=f"Integrity error: A connector with this configuration already exists. {e!s}", - ) from e - except Exception as e: - logger.error(f"Failed to create Elasticsearch connector: {e!s}") - await session.rollback() - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to create Elasticsearch connector: {e!s}", - ) from e - - -@router.get("/connectors") -async def get_elasticsearch_connectors( - current_user: User = Depends(current_active_user), - session: AsyncSession = Depends(get_async_session), -) -> dict[str, Any]: - """Get all Elasticsearch connectors for the current user""" - - try: - result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.user_id == current_user.id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR, - ) - ) - connectors = result.scalars().all() - - connector_list = [] - for connector in connectors: - connector_list.append( - { - "id": connector.id, - "name": connector.name, - "connector_type": connector.connector_type.value, - "is_indexable": connector.is_indexable, - "created_at": connector.created_at.isoformat() - if connector.created_at - else None, - "last_indexed_at": connector.last_indexed_at.isoformat() - if connector.last_indexed_at - else None, - "config": { - "hostname": connector.config.get("hostname"), - "port": connector.config.get("port"), - "indices": connector.config.get("indices"), - # Don't expose sensitive credentials - }, - } - ) - - return {"success": True, "connectors": connector_list} - - except Exception as e: - logger.error(f"Error fetching Elasticsearch connectors: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Failed to fetch Elasticsearch connectors", - ) from e - - -@router.put("/connectors/{connector_id}") -async def update_elasticsearch_connector( - connector_id: int, - connector_data: ElasticsearchConnectorConfig, - current_user: User = Depends(current_active_user), - session: AsyncSession = Depends(get_async_session), -) -> dict[str, Any]: - """Update an existing Elasticsearch connector""" - - try: - # Get the connector - result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == connector_id, - SearchSourceConnector.user_id == current_user.id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR, - ) - ) - connector = result.scalar_one_or_none() - - if not connector: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Elasticsearch connector not found", - ) - - # Test the new connection with robust error handling - try: - elasticsearch_connector = ElasticsearchConnector(connector_data) - connection_test = await elasticsearch_connector.test_connection() - if not connection_test.get("success"): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Failed to connect to Elasticsearch: {connection_test.get('error')}", - ) - except Exception as e: - logger.error(f"Error initializing or testing Elasticsearch connector: {e}") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Failed to initialize or test Elasticsearch connector: {e!s}", - ) from e - finally: - with contextlib.suppress(Exception): - await elasticsearch_connector.disconnect() - - # Update the connector - connector.config = connector_data.model_dump() - connector.name = ( - f"Elasticsearch - {connector_data.hostname}:{connector_data.port}" - ) - - await session.commit() - await session.refresh(connector) - - return { - "success": True, - "message": "Elasticsearch connector updated successfully", - "connector": { - "id": connector.id, - "name": connector.name, - "connector_type": connector.connector_type.value, - "is_indexable": connector.is_indexable, - "created_at": connector.created_at.isoformat() - if connector.created_at - else None, - "last_indexed_at": connector.last_indexed_at.isoformat() - if connector.last_indexed_at - else None, - }, - } - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error updating Elasticsearch connector: {e}") - await session.rollback() - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Failed to update Elasticsearch connector", - ) from e - - -@router.delete("/connectors/{connector_id}") -async def delete_elasticsearch_connector( - connector_id: int, - current_user: User = Depends(current_active_user), - session: AsyncSession = Depends(get_async_session), -) -> dict[str, Any]: - """Delete an Elasticsearch connector""" - - try: - # Get the connector - result = await session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == connector_id, - SearchSourceConnector.user_id == current_user.id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR, - ) - ) - connector = result.scalar_one_or_none() - - if not connector: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Elasticsearch connector not found", - ) - - # Delete the connector - await session.delete(connector) - await session.commit() - - return { - "success": True, - "message": "Elasticsearch connector deleted successfully", - } - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error deleting Elasticsearch connector: {e}") - await session.rollback() - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Failed to delete Elasticsearch connector", - ) from e - - -@router.get("/test-connection") -async def test_elasticsearch_connection( - hostname: str, - port: int = 9200, - username: str | None = None, - password: str | None = None, - api_key: str | None = None, - ssl_enabled: bool = True, - current_user: User = Depends(current_active_user), -) -> dict[str, Any]: - """Test Elasticsearch connection with provided credentials""" - - config_data = ElasticsearchConnectorConfig( - hostname=hostname, - port=port, - username=username, - password=password, - api_key=api_key, - ssl_enabled=ssl_enabled, - ) - - elasticsearch_connector = ElasticsearchConnector(config_data) - - try: - result = await elasticsearch_connector.test_connection() - return result - except Exception as e: - logger.error(f"Error testing Elasticsearch connection: {e}") - return {"success": False, "error": str(e)} - finally: - await elasticsearch_connector.disconnect() - - -@router.get("/indices") -async def get_elasticsearch_indices( - hostname: str, - port: int = 9200, - username: str | None = None, - password: str | None = None, - api_key: str | None = None, - ssl_enabled: bool = True, - current_user: User = Depends(current_active_user), -) -> dict[str, Any]: - """Get list of available Elasticsearch indices""" - - config_data = ElasticsearchConnectorConfig( - hostname=hostname, - port=port, - username=username, - password=password, - api_key=api_key, - ssl_enabled=ssl_enabled, - ) - - elasticsearch_connector = ElasticsearchConnector(config_data) - - try: - if not await elasticsearch_connector.connect(): - return {"success": False, "error": "Failed to connect to Elasticsearch"} - - indices = await elasticsearch_connector.get_indices() - return {"success": True, "indices": indices} - except Exception as e: - logger.error(f"Error fetching Elasticsearch indices: {e}") - return {"success": False, "error": str(e)} - finally: - await elasticsearch_connector.disconnect() - - -# Note: Removed the index_elasticsearch_connector endpoint - now handled by universal system diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index 022be65e4..f5dbb0e3d 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -216,6 +216,7 @@ class SearchSourceConnectorBase(BaseModel): "username", "password", "api_key", + "auth_method", "ssl_enabled", "indices", "query", @@ -227,6 +228,18 @@ class SearchSourceConnectorBase(BaseModel): if not config.get("hostname"): raise ValueError("Elasticsearch connector must have a hostname") + # Validate authentication method + auth_method = config.get("auth_method", "none") + if auth_method not in ["none", "basic", "api_key"]: + raise ValueError("auth_method must be one of: none, basic, api_key") + + # Validate auth credentials based on method + if auth_method == "basic": + if not config.get("username") or not config.get("password"): + raise ValueError("Username and password required for basic auth") + elif auth_method == "api_key" and not config.get("api_key"): + raise ValueError("API key required for api_key auth method") + # Validate that all config keys are allowed for key in config: if key not in allowed_keys: diff --git a/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py b/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py index 1e4b3d7f0..5fbe409ad 100644 --- a/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py @@ -32,7 +32,7 @@ class ElasticsearchIndexer: def __init__(self, connector_config: dict): self.connector_config = connector_config - self.connector = ElasticsearchConnector(self.config) + self.connector = ElasticsearchConnector(self.connector_config) async def get_documents(self): """Get documents from Elasticsearch (kept for backward compatibility)""" diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx new file mode 100644 index 000000000..f03053c3a --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx @@ -0,0 +1,712 @@ +"use client"; + +import { zodResolver } from "@hookform/resolvers/zod"; +import * as RadioGroup from "@radix-ui/react-radio-group"; +import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; +import { motion } from "motion/react"; +import { useParams, useRouter } from "next/navigation"; +import { useId, useState } from "react"; +import { useForm } from "react-hook-form"; +import { toast } from "sonner"; +import * as z from "zod"; + +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@/components/ui/accordion"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Separator } from "@/components/ui/separator"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; + +import { EnumConnectorName } from "@/contracts/enums/connector"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; + +// Define the form schema with Zod +const elasticsearchConnectorFormSchema = z + .object({ + name: z.string().min(3, { + message: "Connector name must be at least 3 characters.", + }), + endpoint_url: z.string().url({ message: "Please enter a valid Elasticsearch endpoint URL." }), + auth_method: z.enum(["basic", "api_key"]).default("api_key"), + username: z.string().optional(), + password: z.string().optional(), + api_key: z.string().optional(), + indices: z.string().optional(), + query: z.string().default("*"), + search_fields: z.array(z.string()).optional(), + max_documents: z.number().min(1).max(10000), + }) + .refine( + (data) => { + if (data.auth_method === "basic") { + return data.username && data.password; + } + if (data.auth_method === "api_key") { + return data.api_key; + } + return true; + }, + { + message: "Authentication credentials are required for the selected method.", + path: ["auth_method"], + } + ); + +// Define the type for the form values +type ElasticsearchConnectorFormValues = z.infer; + +export default function ElasticsearchConnectorPage() { + const router = useRouter(); + const params = useParams(); + const searchSpaceId = params.search_space_id as string; + const [isSubmitting, setIsSubmitting] = useState(false); + + const authBasicId = useId(); + const authApiKeyId = useId(); + + const { createConnector } = useSearchSourceConnectors(); + + // Initialize the form + const form = useForm({ + resolver: zodResolver(elasticsearchConnectorFormSchema), + defaultValues: { + name: "Elasticsearch Connector", + endpoint_url: "", + auth_method: "api_key", + username: "", + password: "", + api_key: "", + indices: "", + query: "*", + search_fields: "", + }, + }); + + const stringToArray = (str: string): string[] => { + return str + .split(",") + .map((item) => item.trim()) + .filter((item) => item.length > 0); + }; + + // Handle form submission + const onSubmit = async (values: ElasticsearchConnectorFormValues) => { + setIsSubmitting(true); + try { + // Parse the endpoint URL to extract hostname, port, and SSL + const url = new URL(values.endpoint_url); + const hostname = url.hostname; + const port = url.port ? parseInt(url.port, 10) : url.protocol === "https:" ? 443 : 80; + const ssl_enabled = url.protocol === "https:"; + + const config: any = { + hostname, + port, + ssl_enabled, + }; + + if (values.auth_method === "basic") { + config.username = values.username; + config.password = values.password; + } else if (values.auth_method === "api_key") { + config.api_key = values.api_key; + } + + if (values.indices?.trim()) { + config.indices = stringToArray(values.indices); + } + if (values.query && values.query !== "*") { + config.query = values.query; + } + if (values.search_fields?.trim()) { + config.search_fields = stringToArray(values.search_fields); + } + if (values.max_documents !== undefined && values.max_documents > 0) { + config.max_documents = values.max_documents; + } + + // Ensure the connector payload has the correct structure + const connectorPayload = { + name: values.name, + connector_type: "ELASTICSEARCH_CONNECTOR", + is_indexable: true, + config, + }; + + // console.log("Connector payload:", JSON.stringify(connectorPayload, null, 2)); + + // Use existing hook method + await createConnector(connectorPayload); + + toast.success("Elasticsearch connector created successfully!"); + router.push(`/dashboard/${searchSpaceId}/connectors`); + } catch (error) { + console.error("Error creating connector:", error); + toast.error(error instanceof Error ? error.message : "Failed to create connector"); + } finally { + setIsSubmitting(false); + } + }; + + return ( +
+ + + {/* Header */} +
+
+
+ {getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6")} +
+
+

Connect Elasticsearch

+

+ Connect to your Elasticsearch cluster to search and index documents. +

+
+
+
+ + + + + Connect + Documentation + + + + + + Connect Elasticsearch Cluster + + Connect to your Elasticsearch instance to search and index documents for enhanced + search capabilities. + + + +
+ + {/* Connector Name */} + ( + + Connector Name + + + + + A friendly name to identify this connector. + + + + )} + /> + + {/* Connection Details */} +
+

Connection Details

+ + ( + + Elasticsearch Endpoint URL + + + + + Enter the complete Elasticsearch endpoint URL. We'll automatically + extract the hostname, port, and SSL settings. + + + + )} + /> + + {/* Show parsed URL details */} + {form.watch("endpoint_url") && ( +
+

Parsed Connection Details:

+
+ {(() => { + try { + const url = new URL(form.watch("endpoint_url")); + return ( + <> +
+ Hostname: {url.hostname} +
+
+ Port:{" "} + {url.port || (url.protocol === "https:" ? "443" : "80")} +
+
+ SSL/TLS:{" "} + {url.protocol === "https:" ? "Enabled" : "Disabled"} +
+ + ); + } catch { + return
Invalid URL format
; + } + })()} +
+
+ )} +
+ + {/* Authentication */} +
+

Authentication

+ + ( + + + { + field.onChange(value); + // Clear auth fields when method changes + if (value !== "basic") { + form.setValue("username", ""); + form.setValue("password", ""); + } + if (value !== "api_key") { + form.setValue("api_key", ""); + } + }} + value={field.value} + className="flex flex-col space-y-2" + > +
+ + +
+ + + +
+ +
+ + +
+ + + +
+ + + + + )} + /> + + {/* Basic Auth Fields */} + {form.watch("auth_method") === "basic" && ( +
+ ( + + Username + + + + + + )} + /> + + ( + + Password + + + + + + )} + /> +
+ )} + + {/* API Key Field */} + {form.watch("auth_method") === "api_key" && ( + ( + + API Key + + + + + Enter your Elasticsearch API key (base64 encoded). This will be + stored securely. + + + + )} + /> + )} +
+ + {/* Advanced Configuration */} + + + Advanced Configuration + + {/* Index Selection */} + ( + + + Index Selection{" "} + (Optional) + + + + + + Comma-separated indices to search (e.g., "logs-*, documents-*"). + Leave empty for all indices. Supports wildcards. + + + + )} + /> + + {/* Show parsed indices as badges */} + {form.watch("indices")?.trim() && ( +
+

Selected Indices:

+
+ {stringToArray(form.watch("indices")).map((index) => ( + + {index} + + ))} +
+
+ )} + + ( + + + Default Search Query{" "} + (Optional) + + + + + + Default Elasticsearch query to use for searches. Use "*" to match + all documents. + + + + )} + /> + + ( + + + Search Fields{" "} + (Optional) + + + + + + Comma-separated list of specific fields to search in (e.g., + "title, content, description"). Leave empty to search all fields. + + + + )} + /> + + {/* Show parsed search fields as badges */} + {form.watch("search_fields")?.trim() && ( +
+

Search Fields:

+
+ {stringToArray(form.watch("search_fields")).map((field) => ( + + {field} + + ))} +
+
+ )} + + ( + + + Maximum Documents{" "} + (Optional) + + + field.onChange(parseInt(e.target.value, 10))} + /> + + + Maximum number of documents to retrieve per search (1-10,000). + Leave empty to use Elasticsearch's default limit. + + + + )} + /> + + + + Index Selection Tips + +
    +
  • Use wildcards like "logs-*" to match multiple indices
    • +
  • Separate multiple indices with commas
    • +
  • Leave empty to search all accessible indices
    • +
  • Choosing specific indices improves search performance
    • +
+ + + + + + + + +
+ +
+ + + + +

+ What you get with Elasticsearch integration: +

+
    +
  • Search across your indexed documents and logs
    • +
  • Access structured and unstructured data from your cluster
    • +
  • Leverage existing Elasticsearch indices for enhanced search
    • +
  • Real-time search capabilities with powerful query features
    • +
  • Integration with your existing Elasticsearch infrastructure
    • +
+ + + + + + + + + Elasticsearch Connector Documentation + + + Learn how to set up and use the Elasticsearch connector to search your data. + + + +
+

How it works

+

+ The Elasticsearch connector allows you to search and retrieve documents from + your Elasticsearch cluster. Configure connection details, select specific + indices, and set search parameters to make your existing data searchable within + SurfSense. +

+
+ + + + + Connection Setup + + +
    +
  1. + Endpoint URL: Enter the complete Elasticsearch endpoint + URL (e.g., https://your-cluster.es.region.aws.com:443). We'll + automatically extract hostname, port, and SSL settings. +
    2. +
  2. + Authentication: Choose the appropriate method: +
      +
    • + API Key: Base64 encoded API key (recommended for + security) +
      • +
    • + Username/Password: Basic authentication credentials +
      • +
    +
    3. +
+ + + + + + Advanced Configuration + + +

+ Fine-tune your Elasticsearch connector with these optional settings: +

+
    +
  • + Index Selection: Specify which indices to search using + comma-separated patterns (e.g., "logs-*, documents-*") +
    • +
  • + Search Fields: Limit searches to specific fields (e.g., + "title, content") for better relevance +
    • +
  • + Default Query: Set a default Elasticsearch query pattern +
    • +
  • + Max Documents: Limit the number of documents returned per + search (1-10,000) +
    • +
+ + + + + + Troubleshooting + + +
+
+

Common Connection Issues:

+
    +
  • + Connection Refused: Check hostname and port. Ensure + Elasticsearch is running. +
    • +
  • + Authentication Failed: Verify credentials. For API + keys, ensure they have proper permissions. +
    • +
  • + SSL Errors: Try disabling SSL for local development + or check certificate validity. +
    • +
  • + No Indices Found: Ensure your credentials have + permission to list and read indices. +
    • +
+
+ + + + Security Note + + For production environments, use API keys with minimal required + permissions: cluster monitoring and read access to specific indices. + + +
+ + + + + + + + +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx index a8bb9cfe8..4593541bb 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx @@ -52,6 +52,13 @@ const connectorCategories: ConnectorCategory[] = [ icon: getConnectorIcon(EnumConnectorName.LINKUP_API, "h-6 w-6"), status: "available", }, + { + id: "elasticsearch-connector", + title: "Elasticsearch", + description: "Connect to your Elasticsearch cluster to search and index data.", + icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"), + status: "available", + }, ], }, { diff --git a/surfsense_web/components/dashboard-breadcrumb.tsx b/surfsense_web/components/dashboard-breadcrumb.tsx index 6eb71ef5d..e77a1fdb3 100644 --- a/surfsense_web/components/dashboard-breadcrumb.tsx +++ b/surfsense_web/components/dashboard-breadcrumb.tsx @@ -88,6 +88,7 @@ export function DashboardBreadcrumb() { "serper-api": "Serper API", "linkup-api": "LinkUp API", "luma-connector": "Luma", + "elasticsearch-connector": "Elasticsearch", }; const connectorLabel = connectorLabels[connectorType] || connectorType; diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index f34a10398..7310edfad 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -1,6 +1,7 @@ import { IconBook, IconBrandDiscord, + IconBrandElastic, IconBrandGithub, IconBrandNotion, IconBrandSlack, @@ -17,7 +18,6 @@ import { } from "@tabler/icons-react"; import { File, Globe, Link, Microscope, Search, Sparkles, Telescope, Webhook } from "lucide-react"; import { EnumConnectorName } from "./connector"; - export const getConnectorIcon = (connectorType: EnumConnectorName | string, className?: string) => { const iconProps = { className: className || "h-4 w-4" }; @@ -52,6 +52,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas return ; case EnumConnectorName.LUMA_CONNECTOR: return ; + case EnumConnectorName.ELASTICSEARCH_CONNECTOR: + return ; // Additional cases for non-enum connector types case "YOUTUBE_VIDEO": return ; diff --git a/surfsense_web/package.json b/surfsense_web/package.json index f65904784..8897fd6ef 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -32,6 +32,7 @@ "@radix-ui/react-dropdown-menu": "^2.1.15", "@radix-ui/react-label": "^2.1.7", "@radix-ui/react-popover": "^1.1.14", + "@radix-ui/react-radio-group": "^1.3.8", "@radix-ui/react-scroll-area": "^1.2.9", "@radix-ui/react-select": "^2.2.5", "@radix-ui/react-separator": "^1.1.7", diff --git a/surfsense_web/pnpm-lock.yaml b/surfsense_web/pnpm-lock.yaml index 3463929ec..562a7a7bd 100644 --- a/surfsense_web/pnpm-lock.yaml +++ b/surfsense_web/pnpm-lock.yaml @@ -47,6 +47,9 @@ importers: '@radix-ui/react-popover': specifier: ^1.1.14 version: 1.1.14(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + '@radix-ui/react-radio-group': + specifier: ^1.3.8 + version: 1.3.8(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) '@radix-ui/react-scroll-area': specifier: ^1.2.9 version: 1.2.9(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) @@ -118,7 +121,7 @@ importers: version: 15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0) fumadocs-mdx: specifier: ^11.7.1 - version: 11.7.1(acorn@8.14.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0) + version: 11.7.1(acorn@8.15.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0) fumadocs-ui: specifier: ^15.6.6 version: 15.6.6(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(tailwindcss@4.1.11) @@ -1735,6 +1738,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-presence@1.1.5': + resolution: {integrity: sha512-/jfEwNDdQVBCNvjkGit4h6pMOzq8bHkopq458dPt2lMjx+eBQUohZNG9A7DtO/O5ukSbxuaNGXMjHicgwy6rQQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-primitive@1.0.0': resolution: {integrity: sha512-EyXe6mnRlHZ8b6f4ilTDrXmkLShICIuOTTj0GX4w1rp+wSxf3+TD05u1UOITC8VsJ2a9nwHvdXtOXEOl0Cw/zQ==} peerDependencies: @@ -1793,6 +1809,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-radio-group@1.3.8': + resolution: {integrity: sha512-VBKYIYImA5zsxACdisNQ3BjCBfmbGH3kQlnFVqlWU4tXwjy7cGX8ta80BcrO+WJXIn5iBylEH3K6ZTlee//lgQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-roving-focus@1.1.10': resolution: {integrity: sha512-dT9aOXUen9JSsxnMPv/0VqySQf5eDQ6LCk5Sw28kamz8wSOW2bJdlX2Bg5VUIIcV+6XlHpWTIuTPCf/UNIyq8Q==} peerDependencies: @@ -1806,6 +1835,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-roving-focus@1.1.11': + resolution: {integrity: sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-scroll-area@1.2.9': resolution: {integrity: sha512-YSjEfBXnhUELsO2VzjdtYYD4CfQjvao+lhhrX5XsHD7/cyUNzljF1FHEbgTPN7LH2MClfwRMIsYlqTYpKTTe2A==} peerDependencies: @@ -6554,7 +6596,7 @@ snapshots: '@marijn/find-cluster-break@1.0.2': {} - '@mdx-js/mdx@3.1.0(acorn@8.14.0)': + '@mdx-js/mdx@3.1.0(acorn@8.15.0)': dependencies: '@types/estree': 1.0.8 '@types/estree-jsx': 1.0.5 @@ -6568,7 +6610,7 @@ snapshots: hast-util-to-jsx-runtime: 2.3.6 markdown-extensions: 2.0.0 recma-build-jsx: 1.0.0 - recma-jsx: 1.0.0(acorn@8.14.0) + recma-jsx: 1.0.0(acorn@8.15.0) recma-stringify: 1.0.0 rehype-recma: 1.0.0 remark-mdx: 3.1.0 @@ -7260,6 +7302,16 @@ snapshots: '@types/react': 19.1.8 '@types/react-dom': 19.1.6(@types/react@19.1.8) + '@radix-ui/react-presence@1.1.5(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.1.8)(react@19.1.0) + react: 19.1.0 + react-dom: 19.1.0(react@19.1.0) + optionalDependencies: + '@types/react': 19.1.8 + '@types/react-dom': 19.1.6(@types/react@19.1.8) + '@radix-ui/react-primitive@1.0.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: '@babel/runtime': 7.26.9 @@ -7305,6 +7357,24 @@ snapshots: '@types/react': 19.1.8 '@types/react-dom': 19.1.6(@types/react@19.1.8) + '@radix-ui/react-radio-group@1.3.8(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-direction': 1.1.1(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + '@radix-ui/react-roving-focus': 1.1.11(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-use-previous': 1.1.1(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-use-size': 1.1.1(@types/react@19.1.8)(react@19.1.0) + react: 19.1.0 + react-dom: 19.1.0(react@19.1.0) + optionalDependencies: + '@types/react': 19.1.8 + '@types/react-dom': 19.1.6(@types/react@19.1.8) + '@radix-ui/react-roving-focus@1.1.10(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: '@radix-ui/primitive': 1.1.2 @@ -7322,6 +7392,23 @@ snapshots: '@types/react': 19.1.8 '@types/react-dom': 19.1.6(@types/react@19.1.8) + '@radix-ui/react-roving-focus@1.1.11(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-collection': 1.1.7(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-direction': 1.1.1(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-id': 1.1.1(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.1.8)(react@19.1.0) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.1.8)(react@19.1.0) + react: 19.1.0 + react-dom: 19.1.0(react@19.1.0) + optionalDependencies: + '@types/react': 19.1.8 + '@types/react-dom': 19.1.6(@types/react@19.1.8) + '@radix-ui/react-scroll-area@1.2.9(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: '@radix-ui/number': 1.1.1 @@ -9073,9 +9160,9 @@ snapshots: transitivePeerDependencies: - supports-color - fumadocs-mdx@11.7.1(acorn@8.14.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0): + fumadocs-mdx@11.7.1(acorn@8.15.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0): dependencies: - '@mdx-js/mdx': 3.1.0(acorn@8.14.0) + '@mdx-js/mdx': 3.1.0(acorn@8.15.0) '@standard-schema/spec': 1.0.0 chokidar: 4.0.3 esbuild: 0.25.8 @@ -11137,9 +11224,9 @@ snapshots: estree-util-build-jsx: 3.0.1 vfile: 6.0.3 - recma-jsx@1.0.0(acorn@8.14.0): + recma-jsx@1.0.0(acorn@8.15.0): dependencies: - acorn-jsx: 5.3.2(acorn@8.14.0) + acorn-jsx: 5.3.2(acorn@8.15.0) estree-util-to-js: 2.0.0 recma-parse: 1.0.0 recma-stringify: 1.0.0