Merge pull request #388 from AnishSarkar22/feature/elasticsearch-connector

[Feature] Add elasticsearch connector

README.md

@@ -10,7 +10,7 @@
 
 
 # SurfSense
-While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic/query, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as Search Engines (SearxNG, Tavily, LinkUp), Slack, Linear, Jira, ClickUp, Confluence, Gmail, Notion, YouTube, GitHub, Discord, Airtable, Google Calendar, Luma and more to come.
+While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic/query, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as Search Engines (SearxNG, Tavily, LinkUp), Slack, Linear, Jira, ClickUp, Confluence, Gmail, Notion, YouTube, GitHub, Discord, Airtable, Google Calendar, Luma, Elasticsearch and more to come.
 
 <div align="center">
 <a href="https://trendshift.io/repositories/13606" target="_blank"><img src="https://trendshift.io/api/badge/repositories/13606" alt="MODSetter%2FSurfSense | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
@@ -76,6 +76,7 @@ Open source and easy to deploy locally.
 - Airtable
 - Google Calendar
 - Luma
+- Elasticsearch
 - and more to come.....
 
 ## 📄 **Supported File Extensions**

surfsense_backend/alembic/versions/31_add_elasticsearch_connector_enums.py

@@ -0,0 +1,60 @@
+"""Add ElasticSearch connector enums
+
+Revision ID: 31
+Revises: 30
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers
+revision: str = "31"
+down_revision: str | None = "30"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # Add enum values
+    op.execute(
+        """
+    DO $$
+    BEGIN
+        IF NOT EXISTS (
+            SELECT 1 FROM pg_type t
+            JOIN pg_enum e ON t.oid = e.enumtypid
+            WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'ELASTICSEARCH_CONNECTOR'
+        ) THEN
+            ALTER TYPE searchsourceconnectortype ADD VALUE 'ELASTICSEARCH_CONNECTOR';
+        END IF;
+    END
+    $$;
+    """
+    )
+    op.execute(
+        """
+    DO $$
+    BEGIN
+        IF NOT EXISTS (
+            SELECT 1 FROM pg_type t
+            JOIN pg_enum e ON t.oid = e.enumtypid
+            WHERE t.typname = 'documenttype' AND e.enumlabel = 'ELASTICSEARCH_CONNECTOR'
+        ) THEN
+            ALTER TYPE documenttype ADD VALUE 'ELASTICSEARCH_CONNECTOR';
+        END IF;
+    END
+    $$;
+    """
+    )
+
+
+def downgrade() -> None:
+    """Remove 'ELASTICSEARCH_CONNECTOR' from enum types.
+
+    Note: PostgreSQL does not support removing enum values that may be in use.
+    Manual intervention would be required if rollback is necessary:
+    1. Delete all rows using ELASTICSEARCH_CONNECTOR
+    2. Manually remove the enum value using ALTER TYPE ... DROP VALUE (requires no dependencies)
+    """
+    pass

surfsense_backend/app/agents/researcher/nodes.py

@@ -443,6 +443,25 @@ async def fetch_documents_by_ids(
                     )
                     url = metadata.get("url", "")
 
+                elif doc_type == "ELASTICSEARCH_CONNECTOR":
+                    # Prefer explicit title in metadata/source, otherwise fallback to doc.title
+                    es_title = (
+                        metadata.get("title")
+                        or metadata.get("es_title")
+                        or doc.title
+                        or f"Elasticsearch: {metadata.get('elasticsearch_index', '')}"
+                    )
+                    title = es_title
+                    description = metadata.get("description") or (
+                        doc.content[:100] + "..."
+                        if len(doc.content) > 100
+                        else doc.content
+                    )
+                    # If a link or index info is stored, surface it
+                    url = metadata.get("url", "") or metadata.get(
+                        "elasticsearch_index", ""
+                    )
+
                 else:  # FILE and other types
                     title = doc.title
                     description = doc.content
@@ -464,6 +483,7 @@ async def fetch_documents_by_ids(
                 "SLACK_CONNECTOR": "Slack (Selected)",
                 "NOTION_CONNECTOR": "Notion (Selected)",
                 "GITHUB_CONNECTOR": "GitHub (Selected)",
+                "ELASTICSEARCH_CONNECTOR": "Elasticsearch (Selected)",
                 "YOUTUBE_VIDEO": "YouTube Videos (Selected)",
                 "DISCORD_CONNECTOR": "Discord (Selected)",
                 "JIRA_CONNECTOR": "Jira Issues (Selected)",
@@ -1269,6 +1289,33 @@ async def fetch_relevant_documents(
                             }
                         )
 
+                elif connector == "ELASTICSEARCH_CONNECTOR":
+                    (
+                        source_object,
+                        elasticsearch_chunks,
+                    ) = await connector_service.search_elasticsearch(
+                        user_query=reformulated_query,
+                        user_id=user_id,
+                        search_space_id=search_space_id,
+                        top_k=top_k,
+                        search_mode=search_mode,
+                    )
+
+                    # Add to sources and raw documents
+                    if source_object:
+                        all_sources.append(source_object)
+                    all_raw_documents.extend(elasticsearch_chunks)
+
+                    # Stream found document count
+                    if streaming_service and writer:
+                        writer(
+                            {
+                                "yield_value": streaming_service.format_terminal_info_delta(
+                                    f"🔎 Found {len(elasticsearch_chunks)} Elasticsearch chunks related to your query"
+                                )
+                            }
+                        )
+
             except Exception as e:
                 logging.error("Error in search_airtable: %s", traceback.format_exc())
                 error_message = f"Error searching connector {connector}: {e!s}"

surfsense_backend/app/agents/researcher/qna_agent/prompts.py

@@ -34,6 +34,7 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel
 - NOTION_CONNECTOR: "Notion workspace pages and databases" (personal knowledge management)
 - YOUTUBE_VIDEO: "YouTube video transcripts and metadata" (personally saved videos)
 - GITHUB_CONNECTOR: "GitHub repository content and issues" (personal repositories and interactions)
+- ELASTICSEARCH_CONNECTOR: "Elasticsearch indexed documents and data" (personal Elasticsearch instances and custom data sources)
 - LINEAR_CONNECTOR: "Linear project issues and discussions" (personal project management)
 - JIRA_CONNECTOR: "Jira project issues, tickets, and comments" (personal project tracking)
 - CONFLUENCE_CONNECTOR: "Confluence pages and comments" (personal project documentation)

surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py

@@ -35,6 +35,7 @@ You are SurfSense, an advanced AI research assistant that synthesizes informatio
 - NOTION_CONNECTOR: "Notion workspace pages and databases" (personal knowledge management)
 - YOUTUBE_VIDEO: "YouTube video transcripts and metadata" (personally saved videos)
 - GITHUB_CONNECTOR: "GitHub repository content and issues" (personal repositories and interactions)
+- ELASTICSEARCH_CONNECTOR: "Elasticsearch documents and indices (indexed content from your ES connector)" (personal search index)
 - LINEAR_CONNECTOR: "Linear project issues and discussions" (personal project management)
 - JIRA_CONNECTOR: "Jira project issues, tickets, and comments" (personal project tracking)
 - CONFLUENCE_CONNECTOR: "Confluence pages and comments" (personal project documentation)

surfsense_backend/app/agents/researcher/utils.py

@@ -52,6 +52,7 @@ def get_connector_emoji(connector_name: str) -> str:
         "GOOGLE_CALENDAR_CONNECTOR": "📅",
         "AIRTABLE_CONNECTOR": "🗃️",
         "LUMA_CONNECTOR": "✨",
+        "ELASTICSEARCH_CONNECTOR": "⚡",
     }
     return connector_emojis.get(connector_name, "🔎")
 
@@ -76,6 +77,7 @@ def get_connector_friendly_name(connector_name: str) -> str:
         "BAIDU_SEARCH_API": "Baidu Search",
         "AIRTABLE_CONNECTOR": "Airtable",
         "LUMA_CONNECTOR": "Luma",
+        "ELASTICSEARCH_CONNECTOR": "Elasticsearch",
     }
     return connector_friendly_names.get(connector_name, connector_name)
 

surfsense_backend/app/connectors/elasticsearch_connector.py

@@ -0,0 +1,264 @@
+"""
+Elasticsearch connector for SurfSense
+"""
+
+import logging
+from typing import Any
+
+from elasticsearch import AsyncElasticsearch
+from elasticsearch.exceptions import (
+    AuthenticationException,
+    ConnectionError,
+    NotFoundError,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ElasticsearchConnector:
+    """
+    Connector for Elasticsearch instances
+    """
+
+    def __init__(
+        self,
+        url: str,
+        api_key: str | None = None,
+        username: str | None = None,
+        password: str | None = None,
+        verify_certs: bool = True,
+        ca_certs: str | None = None,
+    ):
+        """
+        Initialize Elasticsearch connector
+
+        Args:
+            url: Full Elasticsearch URL (e.g., https://host:port or cloud endpoint)
+            api_key: API key for authentication (preferred method)
+            username: Username for basic authentication
+            password: Password for basic authentication
+            verify_certs: Whether to verify SSL certificates
+            ca_certs: Path to CA certificates file
+        """
+        self.url = url
+        self.api_key = api_key
+        self.username = username
+        self.password = password
+        self.verify_certs = verify_certs
+        self.ca_certs = ca_certs
+
+        # Build connection configuration
+        self.es_config = self._build_config()
+
+        # Initialize Elasticsearch client
+        try:
+            self.client = AsyncElasticsearch(**self.es_config)
+        except Exception as e:
+            logger.error(f"Failed to initialize Elasticsearch client: {e}")
+            raise
+
+    def _build_config(self) -> dict[str, Any]:
+        """Build Elasticsearch client configuration"""
+        config = {
+            "hosts": [self.url],
+            "verify_certs": self.verify_certs,
+            "request_timeout": 30,
+            "max_retries": 3,
+            "retry_on_timeout": True,
+        }
+
+        # Authentication - API key takes precedence
+        if self.api_key:
+            config["api_key"] = self.api_key
+        elif self.username and self.password:
+            config["basic_auth"] = (self.username, self.password)
+
+        # SSL configuration
+        if self.ca_certs:
+            config["ca_certs"] = self.ca_certs
+
+        return config
+
+    async def search(
+        self,
+        index: str | list[str],
+        query: dict[str, Any],
+        size: int = 100,
+        from_: int = 0,
+        fields: list[str] | None = None,
+        sort: list[dict[str, Any]] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Search documents in Elasticsearch
+
+        Args:
+            index: Elasticsearch index name or list of indices
+            query: Elasticsearch query DSL
+            size: Number of results to return
+            from_: Starting offset for pagination
+            fields: List of fields to include in response
+            sort: Sort configuration
+
+        Returns:
+            Elasticsearch search response
+        """
+        try:
+            search_body: dict[str, Any] = {
+                "query": query,
+                "size": size,
+                "from": from_,
+            }
+
+            if fields:
+                search_body["_source"] = fields
+
+            if sort:
+                search_body["sort"] = sort
+
+            response = await self.client.search(index=index, body=search_body)
+
+            total_hits = response.get("hits", {}).get("total", {})
+            # normalize total value (could be dict or int depending on server)
+            total_val = (
+                total_hits.get("value", total_hits)
+                if isinstance(total_hits, dict)
+                else total_hits
+            )
+            logger.info(
+                f"Successfully searched index '{index}', found {total_val} results"
+            )
+            return response
+
+        except NotFoundError:
+            logger.error(f"Index '{index}' not found")
+            raise
+        except AuthenticationException:
+            logger.error("Authentication failed")
+            raise
+        except ConnectionError:
+            logger.error("Failed to connect to Elasticsearch")
+            raise
+        except Exception as e:
+            logger.error(f"Search failed: {e}")
+            raise
+
+    async def get_indices(self) -> list[str]:
+        """
+        Get list of available indices
+
+        Returns:
+            List of index names
+        """
+        try:
+            indices = await self.client.indices.get_alias(index="*")
+            return list(indices.keys())
+        except Exception as e:
+            logger.error(f"Failed to get indices: {e}")
+            raise
+
+    async def get_mapping(self, index: str) -> dict[str, Any]:
+        """
+        Get mapping for an index
+
+        Args:
+            index: Index name
+
+        Returns:
+            Index mapping
+        """
+        try:
+            mapping = await self.client.indices.get_mapping(index=index)
+            return mapping[index]["mappings"] if index in mapping else {}
+        except Exception as e:
+            logger.error(f"Failed to get mapping for index '{index}': {e}")
+            raise
+
+    async def scroll_search(
+        self,
+        index: str | list[str],
+        query: dict[str, Any],
+        size: int = 1000,
+        scroll_timeout: str = "5m",
+        fields: list[str] | None = None,
+    ):
+        """
+        Perform a scroll search for large result sets
+
+        Args:
+            index: Elasticsearch index name or list of indices
+            query: Elasticsearch query DSL
+            size: Number of results per scroll
+            scroll_timeout: Scroll timeout
+            fields: List of fields to include in response
+
+        Yields:
+            Document hits from Elasticsearch
+        """
+        try:
+            search_body: dict[str, Any] = {
+                "query": query,
+                "size": size,
+            }
+
+            if fields:
+                search_body["_source"] = fields
+
+            # Initial search
+            response = await self.client.search(
+                index=index, body=search_body, scroll=scroll_timeout
+            )
+
+            scroll_id = response.get("_scroll_id")
+            hits = response.get("hits", {}).get("hits", [])
+
+            while hits:
+                for hit in hits:
+                    yield hit
+
+                # Continue scrolling
+                if scroll_id:
+                    response = await self.client.scroll(
+                        scroll_id=scroll_id, scroll=scroll_timeout
+                    )
+                    scroll_id = response.get("_scroll_id")
+                    hits = response.get("hits", {}).get("hits", [])
+
+            # Clear scroll
+            if scroll_id:
+                try:
+                    await self.client.clear_scroll(scroll_id=scroll_id)
+                except Exception:
+                    logger.debug("Failed to clear scroll id (non-fatal)")
+
+        except Exception as e:
+            logger.error(f"Scroll search failed: {e}", exc_info=True)
+            raise
+
+    async def count_documents(
+        self, index: str | list[str], query: dict[str, Any] | None = None
+    ) -> int:
+        """
+        Count documents in an index
+
+        Args:
+            index: Index name or list of indices
+            query: Optional query to filter documents
+
+        Returns:
+            Number of documents
+        """
+        try:
+            if query:
+                response = await self.client.count(index=index, body={"query": query})
+            else:
+                response = await self.client.count(index=index)
+
+            return response["count"]
+        except Exception as e:
+            logger.error(f"Failed to count documents in index '{index}': {e}")
+            raise
+
+    async def close(self):
+        """Close the Elasticsearch client connection"""
+        if hasattr(self, "client"):
+            await self.client.close()

surfsense_backend/app/db.py

@@ -50,6 +50,7 @@ class DocumentType(str, Enum):
     GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR"
     AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR"
     LUMA_CONNECTOR = "LUMA_CONNECTOR"
+    ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR"
 
 
 class SearchSourceConnectorType(str, Enum):
@@ -70,6 +71,7 @@ class SearchSourceConnectorType(str, Enum):
     GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR"
     AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR"
     LUMA_CONNECTOR = "LUMA_CONNECTOR"
+    ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR"
 
 
 class ChatType(str, Enum):

surfsense_backend/app/routes/search_source_connectors_routes.py

@@ -40,6 +40,7 @@ from app.tasks.connector_indexers import (
     index_clickup_tasks,
     index_confluence_pages,
     index_discord_messages,
+    index_elasticsearch_documents,
     index_github_repos,
     index_google_calendar_events,
     index_google_gmail_messages,
@@ -363,6 +364,7 @@ async def index_connector_content(
     - JIRA_CONNECTOR: Indexes issues and comments from Jira
     - DISCORD_CONNECTOR: Indexes messages from all accessible Discord channels
     - LUMA_CONNECTOR: Indexes events from Luma
+    - ELASTICSEARCH_CONNECTOR: Indexes documents from Elasticsearch
 
     Args:
         connector_id: ID of the connector to use
@@ -589,6 +591,24 @@ async def index_connector_content(
             )
             response_message = "Luma indexing started in the background."
 
+        elif (
+            connector.connector_type
+            == SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR
+        ):
+            # Run indexing in background
+            logger.info(
+                f"Triggering Elasticsearch indexing for connector {connector_id} into search space {search_space_id}"
+            )
+            background_tasks.add_task(
+                run_elasticsearch_indexing_with_new_session,
+                connector_id,
+                search_space_id,
+                str(user.id),
+                indexing_from,
+                indexing_to,
+            )
+            response_message = "Elasticsearch indexing started in the background."
+
         else:
             raise HTTPException(
                 status_code=400,
@@ -1358,3 +1378,61 @@ async def run_luma_indexing(
             )
     except Exception as e:
         logger.error(f"Error in background Luma indexing task: {e!s}")
+
+
+async def run_elasticsearch_indexing_with_new_session(
+    connector_id: int,
+    search_space_id: int,
+    user_id: str,
+    start_date: str,
+    end_date: str,
+):
+    """Wrapper to run Elasticsearch indexing with its own database session."""
+    logger.info(
+        f"Background task started: Indexing Elasticsearch connector {connector_id} into space {search_space_id}"
+    )
+    async with async_session_maker() as session:
+        await run_elasticsearch_indexing(
+            session, connector_id, search_space_id, user_id, start_date, end_date
+        )
+    logger.info(
+        f"Background task finished: Indexing Elasticsearch connector {connector_id}"
+    )
+
+
+async def run_elasticsearch_indexing(
+    session: AsyncSession,
+    connector_id: int,
+    search_space_id: int,
+    user_id: str,
+    start_date: str,
+    end_date: str,
+):
+    """Runs the Elasticsearch indexing task and updates the timestamp."""
+    try:
+        indexed_count, error_message = await index_elasticsearch_documents(
+            session,
+            connector_id,
+            search_space_id,
+            user_id,
+            start_date,
+            end_date,
+            update_last_indexed=False,
+        )
+        if error_message:
+            logger.error(
+                f"Elasticsearch indexing failed for connector {connector_id}: {error_message}"
+            )
+        else:
+            logger.info(
+                f"Elasticsearch indexing successful for connector {connector_id}. Indexed {indexed_count} documents."
+            )
+            # Update the last indexed timestamp only on success
+            await update_connector_last_indexed(session, connector_id)
+            await session.commit()
+    except Exception as e:
+        await session.rollback()
+        logger.error(
+            f"Critical error in run_elasticsearch_indexing for connector {connector_id}: {e}",
+            exc_info=True,
+        )

surfsense_backend/app/services/connector_service.py

@@ -2421,3 +2421,117 @@ class ConnectorService:
         }
 
         return result_object, luma_chunks
+
+    async def search_elasticsearch(
+        self,
+        user_query: str,
+        user_id: str,
+        search_space_id: int,
+        top_k: int = 20,
+        search_mode: SearchMode = SearchMode.CHUNKS,
+    ) -> tuple:
+        """
+        Search for Elasticsearch documents and return both the source information and langchain documents
+
+        Args:
+            user_query: The user's query
+            user_id: The user's ID
+            search_space_id: The search space ID to search in
+            top_k: Maximum number of results to return
+            search_mode: Search mode (CHUNKS or DOCUMENTS)
+
+        Returns:
+            tuple: (sources_info, langchain_documents)
+        """
+        if search_mode == SearchMode.CHUNKS:
+            elasticsearch_chunks = await self.chunk_retriever.hybrid_search(
+                query_text=user_query,
+                top_k=top_k,
+                user_id=user_id,
+                search_space_id=search_space_id,
+                document_type="ELASTICSEARCH_CONNECTOR",
+            )
+        elif search_mode == SearchMode.DOCUMENTS:
+            elasticsearch_chunks = await self.document_retriever.hybrid_search(
+                query_text=user_query,
+                top_k=top_k,
+                user_id=user_id,
+                search_space_id=search_space_id,
+                document_type="ELASTICSEARCH_CONNECTOR",
+            )
+            # Transform document retriever results to match expected format
+            elasticsearch_chunks = self._transform_document_results(
+                elasticsearch_chunks
+            )
+
+        # Early return if no results
+        if not elasticsearch_chunks:
+            return {
+                "id": 34,
+                "name": "Elasticsearch",
+                "type": "ELASTICSEARCH_CONNECTOR",
+                "sources": [],
+            }, []
+
+        # Process each chunk and create sources directly without deduplication
+        sources_list = []
+        async with self.counter_lock:
+            for _i, chunk in enumerate(elasticsearch_chunks):
+                # Extract document metadata
+                document = chunk.get("document", {})
+                metadata = document.get("metadata", {})
+
+                # Extract Elasticsearch-specific metadata
+                es_id = metadata.get("elasticsearch_id", "")
+                es_index = metadata.get("elasticsearch_index", "")
+                es_score = metadata.get("elasticsearch_score", "")
+
+                # Create a more descriptive title for Elasticsearch documents
+                title = document.get("title", "Elasticsearch Document")
+                if es_index:
+                    title = f"{title} (Index: {es_index})"
+
+                # Create a more descriptive description for Elasticsearch documents
+                description = chunk.get("content", "")[:150]
+                if len(description) == 150:
+                    description += "..."
+
+                # Add Elasticsearch info to description
+                info_parts = []
+                if es_id:
+                    info_parts.append(f"ID: {es_id}")
+                if es_score:
+                    info_parts.append(f"Score: {es_score}")
+
+                if info_parts:
+                    if description:
+                        description = f"{description} | {' | '.join(info_parts)}"
+                    else:
+                        description = " | ".join(info_parts)
+
+                # For URL, we could construct a URL to view the document if we have the Elasticsearch UI URL
+                url = ""
+                # Could be extended to include Kibana or other UI URLs if configured
+
+                source = {
+                    "id": chunk.get("chunk_id", self.source_id_counter),
+                    "title": title,
+                    "description": description,
+                    "url": url,
+                    "elasticsearch_id": es_id,
+                    "elasticsearch_index": es_index,
+                    "elasticsearch_score": es_score,
+                }
+
+                self.source_id_counter += 1
+                sources_list.append(source)
+
+        # Create result object
+        result_object = {
+            "id": 34,  # Assign a unique ID for the Elasticsearch connector
+            "name": "Elasticsearch",
+            "type": "ELASTICSEARCH_CONNECTOR",
+            "sources": sources_list,
+        }
+
+        return result_object, elasticsearch_chunks

surfsense_backend/app/tasks/connector_indexers/__init__.py

@@ -17,6 +17,7 @@ Available indexers:
 - Google Gmail: Index messages from Google Gmail
 - Google Calendar: Index events from Google Calendar
 - Luma: Index events from Luma
+- Elasticsearch: Index documents from Elasticsearch instances
 """
 
 # Communication platforms
@@ -27,6 +28,7 @@ from .confluence_indexer import index_confluence_pages
 from .discord_indexer import index_discord_messages
 
 # Development platforms
+from .elasticsearch_indexer import index_elasticsearch_documents
 from .github_indexer import index_github_repos
 from .google_calendar_indexer import index_google_calendar_events
 from .google_gmail_indexer import index_google_gmail_messages
@@ -46,6 +48,7 @@ __all__ = [  # noqa: RUF022
     "index_confluence_pages",
     "index_discord_messages",
     # Development platforms
+    "index_elasticsearch_documents",
     "index_github_repos",
     # Calendar and scheduling
     "index_google_calendar_events",

surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py

@@ -0,0 +1,399 @@
+"""
+Elasticsearch indexer for SurfSense
+"""
+
+import hashlib
+import json
+import logging
+from datetime import UTC, datetime
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.future import select
+
+from app.connectors.elasticsearch_connector import ElasticsearchConnector
+from app.db import Document, DocumentType, SearchSourceConnector
+from app.services.task_logging_service import TaskLoggingService
+from app.utils.document_converters import (
+    create_document_chunks,
+    generate_content_hash,
+    generate_unique_identifier_hash,
+)
+
+from .base import check_document_by_unique_identifier, check_duplicate_document_by_hash
+
+logger = logging.getLogger(__name__)
+
+
+async def index_elasticsearch_documents(
+    session: AsyncSession,
+    connector_id: int,
+    search_space_id: int,
+    user_id: str,
+    start_date: str,
+    end_date: str,
+    update_last_indexed: bool = True,
+) -> tuple[int, str | None]:
+    """
+    Index documents from Elasticsearch into SurfSense
+
+    Args:
+        session: Database session
+        connector_id: Elasticsearch connector ID
+        search_space_id: Search space ID
+        user_id: User ID
+        start_date: Start date for indexing (not used for Elasticsearch, kept for compatibility)
+        end_date: End date for indexing (not used for Elasticsearch, kept for compatibility)
+        update_last_indexed: Whether to update the last indexed timestamp
+
+    Returns:
+        Tuple of (number of documents processed, error message if any)
+    """
+    task_logger = TaskLoggingService(session, search_space_id)
+    log_entry = await task_logger.log_task_start(
+        task_name="elasticsearch_indexing",
+        source="connector_indexing_task",
+        message=f"Starting Elasticsearch indexing for connector {connector_id}",
+        metadata={
+            "connector_id": connector_id,
+            "user_id": str(user_id),
+            "index": None,
+            "start_date": start_date,
+            "end_date": end_date,
+        },
+    )
+    es_connector = None
+    try:
+        # Get the connector configuration
+        result = await session.execute(
+            select(SearchSourceConnector).filter(
+                SearchSourceConnector.id == connector_id
+            )
+        )
+        connector = result.scalars().first()
+
+        if not connector:
+            error_msg = f"Elasticsearch connector with ID {connector_id} not found"
+            logger.error(error_msg)
+            await task_logger.log_task_failure(
+                log_entry,
+                "Connector not found",
+                error_msg,
+                {"connector_id": connector_id},
+            )
+            return 0, error_msg
+
+        # Get connector configuration
+        config = connector.config
+
+        # Validate required fields - now only URL and INDEX are required
+        # Authentication can be either API key OR username/password
+        if "ELASTICSEARCH_URL" not in config:
+            error_msg = "Missing required field in connector config: ELASTICSEARCH_URL"
+            logger.error(error_msg)
+            return 0, error_msg
+
+        # Allow missing/empty index: default to searching all indices ("*" or "_all")
+        index_name = config.get("ELASTICSEARCH_INDEX")
+        if not index_name:
+            index_name = "*"
+            logger.info(
+                "ELASTICSEARCH_INDEX missing or empty in connector config; defaulting to '*' (search all indices)"
+            )
+            await task_logger.log_task_progress(
+                log_entry,
+                "Using default index",
+                {"index": index_name, "stage": "index_defaulted"},
+            )
+
+        # Check authentication - must have either API key or username+password
+        has_api_key = (
+            "ELASTICSEARCH_API_KEY" in config and config["ELASTICSEARCH_API_KEY"]
+        )
+        has_basic_auth = (
+            "ELASTICSEARCH_USERNAME" in config
+            and config["ELASTICSEARCH_USERNAME"]
+            and "ELASTICSEARCH_PASSWORD" in config
+            and config["ELASTICSEARCH_PASSWORD"]
+        )
+
+        if not has_api_key and not has_basic_auth:
+            error_msg = "Missing authentication: provide either ELASTICSEARCH_API_KEY or ELASTICSEARCH_USERNAME + ELASTICSEARCH_PASSWORD"
+            logger.error(error_msg)
+            return 0, error_msg
+
+        # Initialize document service
+        # document_service = _DocumentService(session)
+
+        # Initialize Elasticsearch connector
+        es_connector = ElasticsearchConnector(
+            url=config["ELASTICSEARCH_URL"],
+            api_key=config.get("ELASTICSEARCH_API_KEY"),
+            username=config.get("ELASTICSEARCH_USERNAME"),
+            password=config.get("ELASTICSEARCH_PASSWORD"),
+            verify_certs=config.get("ELASTICSEARCH_VERIFY_CERTS", True),
+            ca_certs=config.get("ELASTICSEARCH_CA_CERTS"),
+        )
+        await task_logger.log_task_progress(
+            log_entry,
+            "Initialized Elasticsearch connector",
+            {"index": index_name, "stage": "connector_initialized"},
+        )
+
+        # Build query based on configuration
+        query = _build_elasticsearch_query(config)
+
+        # Get max documents to index
+        max_documents = config.get("ELASTICSEARCH_MAX_DOCUMENTS", 1000)
+
+        logger.info(
+            f"Starting Elasticsearch indexing for index '{index_name}' with max {max_documents} documents"
+        )
+
+        documents_processed = 0
+
+        try:
+            await task_logger.log_task_progress(
+                log_entry,
+                "Starting scroll search",
+                {
+                    "index": index_name,
+                    "stage": "scroll_start",
+                    "max_documents": max_documents,
+                },
+            )
+            # Use scroll search for large result sets
+            async for hit in es_connector.scroll_search(
+                index=index_name,
+                query=query,
+                size=min(max_documents, 100),  # Scroll in batches
+                fields=config.get("ELASTICSEARCH_FIELDS"),
+            ):
+                if documents_processed >= max_documents:
+                    break
+
+                try:
+                    # Extract document data
+                    doc_id = hit["_id"]
+                    source = hit.get("_source", {})
+
+                    # Build document title
+                    title_field = config.get("ELASTICSEARCH_TITLE_FIELD")
+                    if not title_field:
+                        for candidate in ("title", "name", "subject"):
+                            if candidate in source:
+                                title_field = candidate
+                                break
+                    title = (
+                        str(source.get(title_field, doc_id))
+                        if title_field is not None
+                        else str(doc_id)
+                    )
+
+                    # Build document content
+                    content = _build_document_content(source, config)
+
+                    if not content.strip():
+                        logger.warning(f"Skipping document {doc_id} - no content found")
+                        continue
+
+                    # Create content hash
+                    content_hash = generate_content_hash(content, search_space_id)
+
+                    # Build metadata
+                    metadata = {
+                        "elasticsearch_id": doc_id,
+                        "elasticsearch_index": hit.get("_index", index_name),
+                        "elasticsearch_score": hit.get("_score"),
+                        "indexed_at": datetime.now().isoformat(),
+                        "source": "ELASTICSEARCH_CONNECTOR",
+                    }
+
+                    # Add any additional metadata fields specified in config
+                    if "ELASTICSEARCH_METADATA_FIELDS" in config:
+                        for field in config["ELASTICSEARCH_METADATA_FIELDS"]:
+                            if field in source:
+                                metadata[f"es_{field}"] = source[field]
+
+                    # Build source-unique identifier and hash (prefer source id dedupe)
+                    source_identifier = f"{hit.get('_index', index_name)}:{doc_id}"
+                    unique_identifier_hash = generate_unique_identifier_hash(
+                        DocumentType.ELASTICSEARCH_CONNECTOR,
+                        source_identifier,
+                        search_space_id,
+                    )
+
+                    # Two-step duplicate detection: first by source-unique id, then by content hash
+                    existing_doc = await check_document_by_unique_identifier(
+                        session, unique_identifier_hash
+                    )
+                    if not existing_doc:
+                        existing_doc = await check_duplicate_document_by_hash(
+                            session, content_hash
+                        )
+
+                    if existing_doc:
+                        # If content is unchanged, skip. Otherwise update the existing document.
+                        if existing_doc.content_hash == content_hash:
+                            logger.info(
+                                f"Skipping ES doc {doc_id} — already indexed (doc id {existing_doc.id})"
+                            )
+                            continue
+                        else:
+                            logger.info(
+                                f"Updating existing document {existing_doc.id} for ES doc {doc_id}"
+                            )
+                            existing_doc.title = title
+                            existing_doc.content = content
+                            existing_doc.content_hash = content_hash
+                            existing_doc.document_metadata = metadata
+                            existing_doc.unique_identifier_hash = unique_identifier_hash
+                            chunks = await create_document_chunks(content)
+                            existing_doc.chunks = chunks
+                            await session.flush()
+                            documents_processed += 1
+                            if documents_processed % 10 == 0:
+                                await session.commit()
+                            continue
+
+                    # Create document
+                    document = Document(
+                        title=title,
+                        content=content,
+                        content_hash=content_hash,
+                        unique_identifier_hash=unique_identifier_hash,
+                        document_type=DocumentType.ELASTICSEARCH_CONNECTOR,
+                        document_metadata=metadata,
+                        search_space_id=search_space_id,
+                    )
+
+                    # Create chunks and attach to document (persist via relationship)
+                    chunks = await create_document_chunks(content)
+                    document.chunks = chunks
+                    session.add(document)
+                    await session.flush()
+
+                    documents_processed += 1
+
+                    if documents_processed % 10 == 0:
+                        logger.info(
+                            f"Processed {documents_processed} Elasticsearch documents"
+                        )
+                        await session.commit()
+
+                except Exception as e:
+                    msg = f"Error processing Elasticsearch document {hit.get('_id', 'unknown')}: {e}"
+                    logger.error(msg)
+                    await task_logger.log_task_failure(
+                        log_entry,
+                        "Document processing error",
+                        msg,
+                        {
+                            "document_id": hit.get("_id", "unknown"),
+                            "error_type": type(e).__name__,
+                        },
+                    )
+                    continue
+
+            # Final commit
+            await session.commit()
+
+            await task_logger.log_task_success(
+                log_entry,
+                f"Successfully indexed {documents_processed} documents from Elasticsearch",
+                {"documents_indexed": documents_processed, "index": index_name},
+            )
+            logger.info(
+                f"Successfully indexed {documents_processed} documents from Elasticsearch"
+            )
+
+            # Update last indexed timestamp if requested
+            if update_last_indexed and documents_processed > 0:
+                # connector.last_indexed_at = datetime.now()
+                connector.last_indexed_at = (
+                    datetime.now(UTC).isoformat().replace("+00:00", "Z")
+                )
+                await session.commit()
+                await task_logger.log_task_progress(
+                    log_entry,
+                    "Updated connector.last_indexed_at",
+                    {"last_indexed_at": connector.last_indexed_at},
+                )
+
+            return documents_processed, None
+
+        finally:
+            # Clean up Elasticsearch connection
+            if es_connector:
+                await es_connector.close()
+
+    except Exception as e:
+        error_msg = f"Error indexing Elasticsearch documents: {e}"
+        logger.error(error_msg, exc_info=True)
+        await task_logger.log_task_failure(
+            log_entry, "Indexing failed", error_msg, {"error_type": type(e).__name__}
+        )
+        await session.rollback()
+        if es_connector:
+            await es_connector.close()
+        return 0, error_msg
+
+
+def _build_elasticsearch_query(config: dict[str, Any]) -> dict[str, Any]:
+    """
+    Build Elasticsearch query from connector configuration
+
+    Args:
+        config: Connector configuration
+
+    Returns:
+        Elasticsearch query DSL
+    """
+    # Check if custom query is provided
+    if config.get("ELASTICSEARCH_QUERY"):
+        try:
+            if isinstance(config["ELASTICSEARCH_QUERY"], str):
+                return json.loads(config["ELASTICSEARCH_QUERY"])
+            else:
+                return config["ELASTICSEARCH_QUERY"]
+        except (json.JSONDecodeError, TypeError) as e:
+            logger.warning(f"Invalid custom query, using match_all: {e}")
+
+    # Default to match all documents
+    return {"match_all": {}}
+
+
+def _build_document_content(source: dict[str, Any], config: dict[str, Any]) -> str:
+    """
+    Build document content from Elasticsearch document source
+
+    Args:
+        source: Elasticsearch document source
+        config: Connector configuration
+
+    Returns:
+        Formatted document content
+    """
+    content_parts = []
+
+    # Get content fields from config
+    content_fields = config.get("ELASTICSEARCH_CONTENT_FIELDS", [])
+
+    if content_fields:
+        # Use specified content fields
+        for field in content_fields:
+            if field in source:
+                field_value = source[field]
+                if isinstance(field_value, str | int | float):
+                    content_parts.append(f"{field}: {field_value}")
+                elif isinstance(field_value, list | dict):
+                    content_parts.append(f"{field}: {json.dumps(field_value)}")
+    else:
+        # Use all fields if no specific content fields specified
+        for key, value in source.items():
+            if isinstance(value, str | int | float):
+                content_parts.append(f"{key}: {value}")
+            elif isinstance(value, list | dict):
+                content_parts.append(f"{key}: {json.dumps(value)}")
+
+    return "\n".join(content_parts)

surfsense_backend/pyproject.toml

@@ -43,6 +43,7 @@ dependencies = [
     "youtube-transcript-api>=1.0.3",
     "litellm>=1.77.5",
     "langchain-litellm>=0.2.3",
+    "elasticsearch>=9.1.1",
     "faster-whisper>=1.1.0",
 ]
 

surfsense_backend/uv.lock

@@ -1161,6 +1161,33 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9c/13/563119fe0af82aca5a3b89399c435953072c39515c2e818eb82793955c3b/effdet-0.4.1-py3-none-any.whl", hash = "sha256:10889a226228d515c948e3fcf811e64c0d78d7aa94823a300045653b9c284cb7", size = 112513, upload-time = "2023-05-21T22:17:58.47Z" },
 ]
 
+[[package]]
+name = "elastic-transport"
+version = "9.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "certifi" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ae/1f/2d1a1790df2b75e1e1eb90d8a3fe066a47ef95e34430657447e549cc274c/elastic_transport-9.1.0.tar.gz", hash = "sha256:1590e44a25b0fe208107d5e8d7dea15c070525f3ac9baafbe4cb659cd14f073d", size = 76483, upload-time = "2025-07-24T16:41:31.017Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ef/5d/dd5a919dd887fe20a91f18faf5b4345ee3a058e483d2aa84cef0f2567e17/elastic_transport-9.1.0-py3-none-any.whl", hash = "sha256:369fa56874c74daae4ea10cbf40636d139f38f42bec0e006b9cd45a168ee7fce", size = 65142, upload-time = "2025-07-24T16:41:29.648Z" },
+]
+
+[[package]]
+name = "elasticsearch"
+version = "9.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "elastic-transport" },
+    { name = "python-dateutil" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/59/6a/5eecef6f1ac8005b04714405cb65971d46031bd897e47c29af86e0f87353/elasticsearch-9.1.1.tar.gz", hash = "sha256:be20acda2a97591a9a6cf4981fc398ee6fca3291cf9e7a9e52b6a9f41a46d393", size = 857802, upload-time = "2025-09-12T13:27:38.62Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cf/4c/c0c95d3d881732a5d1b28e12c9be4dea5953ade71810f94565bd5bd2101a/elasticsearch-9.1.1-py3-none-any.whl", hash = "sha256:2a5c27c57ca3dd3365f665c82c9dcd8666ccfb550d5b07c688c21ec636c104e5", size = 937483, upload-time = "2025-09-12T13:27:34.948Z" },
+]
+
 [[package]]
 name = "email-validator"
 version = "2.2.0"
@@ -5402,6 +5429,7 @@ dependencies = [
     { name = "chonkie", extra = ["all"] },
     { name = "discord-py" },
     { name = "docling" },
+    { name = "elasticsearch" },
     { name = "en-core-web-sm" },
     { name = "fastapi" },
     { name = "fastapi-users", extra = ["oauth", "sqlalchemy"] },
@@ -5450,6 +5478,7 @@ requires-dist = [
     { name = "chonkie", extras = ["all"], specifier = ">=1.0.6" },
     { name = "discord-py", specifier = ">=2.5.2" },
     { name = "docling", specifier = ">=2.15.0" },
+    { name = "elasticsearch", specifier = ">=9.1.1" },
     { name = "en-core-web-sm", url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" },
     { name = "fastapi", specifier = ">=0.115.8" },
     { name = "fastapi-users", extras = ["oauth", "sqlalchemy"], specifier = ">=14.0.1" },

surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx

@@ -271,6 +271,17 @@ export default function EditConnectorPage() {
 										placeholder="API Key..."
 									/>
 								)}
+
+								{/* == Elasticsearch == */}
+								{connector.connector_type === "ELASTICSEARCH_CONNECTOR" && (
+									<EditSimpleTokenForm
+										control={editForm.control}
+										fieldName="ELASTICSEARCH_API_KEY"
+										fieldLabel="Elasticsearch API Key"
+										fieldDescription="Update your Elasticsearch API Key if needed."
+										placeholder="Your Elasticsearch API Key"
+									/>
+								)}
 							</CardContent>
 							<CardFooter className="border-t pt-6">
 								<Button type="submit" disabled={isSaving} className="w-full sm:w-auto">

surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx

@@ -54,6 +54,7 @@ const getConnectorTypeDisplay = (type: string): string => {
 		GOOGLE_GMAIL_CONNECTOR: "Google Gmail Connector",
 		AIRTABLE_CONNECTOR: "Airtable Connector",
 		LUMA_CONNECTOR: "Luma Connector",
+		ELASTICSEARCH_CONNECTOR: "Elasticsearch Connector",
 		// Add other connector types here as needed
 	};
 	return typeMap[type] || type;
@@ -73,6 +74,7 @@ const getApiKeyFieldName = (connectorType: string): string => {
 		DISCORD_CONNECTOR: "DISCORD_BOT_TOKEN",
 		LINKUP_API: "LINKUP_API_KEY",
 		LUMA_CONNECTOR: "LUMA_API_KEY",
+		ELASTICSEARCH_CONNECTOR: "ELASTICSEARCH_API_KEY",
 	};
 	return fieldMap[connectorType] || "";
 };
@@ -233,7 +235,9 @@ export default function EditConnectorPage() {
 															? "GitHub Personal Access Token (PAT)"
 															: connector?.connector_type === "LINKUP_API"
 																? "Linkup API Key"
-																: "API Key"}
+																: connector?.connector_type === "ELASTICSEARCH_CONNECTOR"
+																	? "Elasticsearch API Key"
+																	: "API Key"}
 											</FormLabel>
 											<FormControl>
 												<Input
@@ -247,7 +251,9 @@ export default function EditConnectorPage() {
 																	? "Enter new GitHub PAT (optional)"
 																	: connector?.connector_type === "LINKUP_API"
 																		? "Enter new Linkup API Key (optional)"
-																		: "Enter new API key (optional)"
+																		: connector?.connector_type === "ELASTICSEARCH_CONNECTOR"
+																			? "Enter new Elasticsearch API Key (optional)"
+																			: "Enter new API key (optional)"
 													}
 													{...field}
 												/>
@@ -261,7 +267,9 @@ export default function EditConnectorPage() {
 															? "Enter a new GitHub PAT or leave blank to keep your existing token."
 															: connector?.connector_type === "LINKUP_API"
 																? "Enter a new Linkup API Key or leave blank to keep your existing key."
-																: "Enter a new API key or leave blank to keep your existing key."}
+																: connector?.connector_type === "ELASTICSEARCH_CONNECTOR"
+																	? "Enter a new Elasticsearch API Key or leave blank to keep your existing key."
+																	: "Enter a new API key or leave blank to keep your existing key."}
 											</FormDescription>
 											<FormMessage />
 										</FormItem>

surfsense_web/app/dashboard/[search_space_id]/connectors/add/elasticsearch-connector/page.tsx

@@ -0,0 +1,751 @@
+"use client";
+
+import { zodResolver } from "@hookform/resolvers/zod";
+import * as RadioGroup from "@radix-ui/react-radio-group";
+import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
+import { motion } from "motion/react";
+import { useParams, useRouter, useSearchParams } from "next/navigation";
+import { useId, useState } from "react";
+import { useForm } from "react-hook-form";
+import { toast } from "sonner";
+import * as z from "zod";
+
+import {
+	Accordion,
+	AccordionContent,
+	AccordionItem,
+	AccordionTrigger,
+} from "@/components/ui/accordion";
+import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import {
+	Card,
+	CardContent,
+	CardDescription,
+	CardFooter,
+	CardHeader,
+	CardTitle,
+} from "@/components/ui/card";
+import {
+	Form,
+	FormControl,
+	FormDescription,
+	FormField,
+	FormItem,
+	FormLabel,
+	FormMessage,
+} from "@/components/ui/form";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { Separator } from "@/components/ui/separator";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
+
+import { EnumConnectorName } from "@/contracts/enums/connector";
+import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
+import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
+
+// Define the form schema with Zod
+const elasticsearchConnectorFormSchema = z
+	.object({
+		name: z.string().min(3, {
+			message: "Connector name must be at least 3 characters.",
+		}),
+		endpoint_url: z.string().url({ message: "Please enter a valid Elasticsearch endpoint URL." }),
+		auth_method: z.enum(["basic", "api_key"]).default("api_key"),
+		username: z.string().optional(),
+		password: z.string().optional(),
+		ELASTICSEARCH_API_KEY: z.string().optional(),
+		indices: z.string().optional(),
+		query: z.string().default("*"),
+		search_fields: z.string().optional(),
+		max_documents: z.number().min(1).max(10000).optional(),
+	})
+	.refine(
+		(data) => {
+			if (data.auth_method === "basic") {
+				return Boolean(data.username?.trim() && data.password?.trim());
+			}
+			if (data.auth_method === "api_key") {
+				return Boolean(data.ELASTICSEARCH_API_KEY?.trim());
+			}
+			return true;
+		},
+		{
+			message: "Authentication credentials are required for the selected method.",
+			path: ["auth_method"],
+		}
+	);
+
+// Define the type for the form values
+type ElasticsearchConnectorFormValues = z.infer<typeof elasticsearchConnectorFormSchema>;
+
+export default function ElasticsearchConnectorPage() {
+	const router = useRouter();
+	const params = useParams();
+	const searchParams = useSearchParams();
+	// match pattern used in other connector pages: prefer route param, fallback to query param
+	const searchSpaceId = (params.search_space_id ?? searchParams?.get("search_space_id")) as string;
+	const [isSubmitting, setIsSubmitting] = useState(false);
+
+	const authBasicId = useId();
+	const authApiKeyId = useId();
+
+	const { createConnector } = useSearchSourceConnectors();
+
+	// Initialize the form
+	const form = useForm<ElasticsearchConnectorFormValues>({
+		resolver: zodResolver(elasticsearchConnectorFormSchema),
+		defaultValues: {
+			name: "Elasticsearch Connector",
+			endpoint_url: "",
+			auth_method: "api_key",
+			username: "",
+			password: "",
+			ELASTICSEARCH_API_KEY: "",
+			indices: "",
+			query: "*",
+			search_fields: "",
+			max_documents: undefined,
+		},
+	});
+
+	const stringToArray = (str: string): string[] => {
+		const items = str
+			.split(",")
+			.map((item) => item.trim())
+			.filter((item) => item.length > 0);
+		return Array.from(new Set(items));
+	};
+
+	// Handle form submission
+	const onSubmit = async (values: ElasticsearchConnectorFormValues) => {
+		setIsSubmitting(true);
+		if (!searchSpaceId) {
+			toast.error(
+				"Missing search_space_id (route or ?search_space_id=). Provide it in the URL or pick a search space."
+			);
+			setIsSubmitting(false);
+			return;
+		}
+		const searchSpaceIdNum = Number(searchSpaceId);
+		if (!Number.isInteger(searchSpaceIdNum) || searchSpaceIdNum <= 0) {
+			toast.error("Invalid search_space_id. It must be a positive integer.");
+			setIsSubmitting(false);
+			return;
+		}
+		try {
+			// Send full URL to backend (backend expects ELASTICSEARCH_URL)
+			const config: Record<string, string | number | boolean | string[]> = {
+				ELASTICSEARCH_URL: values.endpoint_url,
+				// default to verifying certs; expose fields for CA/verify if UI added later
+				ELASTICSEARCH_VERIFY_CERTS: true,
+			};
+
+			if (values.auth_method === "basic") {
+				if (values.username) config.ELASTICSEARCH_USERNAME = values.username;
+				if (values.password) config.ELASTICSEARCH_PASSWORD = values.password;
+			} else if (values.auth_method === "api_key") {
+				if (values.ELASTICSEARCH_API_KEY)
+					config.ELASTICSEARCH_API_KEY = values.ELASTICSEARCH_API_KEY;
+			}
+
+			const indicesInput = values.indices?.trim() ?? "";
+			const indicesArr = stringToArray(indicesInput);
+			config.ELASTICSEARCH_INDEX =
+				indicesArr.length === 0 ? "*" : indicesArr.length === 1 ? indicesArr[0] : indicesArr;
+
+			if (values.query && values.query !== "*") {
+				config.ELASTICSEARCH_QUERY = values.query;
+			}
+
+			if (values.search_fields?.trim()) {
+				// config.ELASTICSEARCH_FIELDS = stringToArray(values.search_fields);
+				const fields = stringToArray(values.search_fields);
+				config.ELASTICSEARCH_FIELDS = fields;
+				config.ELASTICSEARCH_CONTENT_FIELDS = fields;
+				if (fields.includes("title")) {
+					config.ELASTICSEARCH_TITLE_FIELD = "title";
+				}
+			}
+
+			if (values.max_documents !== undefined && values.max_documents > 0) {
+				config.ELASTICSEARCH_MAX_DOCUMENTS = values.max_documents;
+			}
+
+			const connectorPayload = {
+				name: values.name,
+				connector_type: EnumConnectorName.ELASTICSEARCH_CONNECTOR,
+				is_indexable: true,
+				search_space_id: searchSpaceIdNum,
+				config,
+			};
+
+			// Use existing hook method
+			await createConnector(connectorPayload, searchSpaceIdNum);
+
+			toast.success("Elasticsearch connector created successfully!");
+			router.push(`/dashboard/${searchSpaceId}/connectors`);
+		} catch (error) {
+			console.error("Error creating connector:", error);
+			toast.error(error instanceof Error ? error.message : "Failed to create connector");
+		} finally {
+			setIsSubmitting(false);
+		}
+	};
+
+	return (
+		<div className="container mx-auto py-8 max-w-3xl">
+			<Button
+				variant="ghost"
+				className="mb-6"
+				onClick={() => router.push(`/dashboard/${searchSpaceId}/connectors/add`)}
+			>
+				<ArrowLeft className="mr-2 h-4 w-4" />
+				Back to Connectors
+			</Button>
+
+			{/* Header */}
+			<div className="mb-8">
+				<div className="flex items-center gap-4">
+					<div className="flex h-12 w-12 items-center justify-center rounded-lg">
+						{getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6")}
+					</div>
+					<div>
+						<h1 className="text-3xl font-bold tracking-tight">Connect Elasticsearch</h1>
+						<p className="text-muted-foreground">
+							Connect to your Elasticsearch cluster to search and index documents.
+						</p>
+					</div>
+				</div>
+			</div>
+
+			<motion.div
+				initial={{ opacity: 0, y: 20 }}
+				animate={{ opacity: 1, y: 0 }}
+				transition={{ duration: 0.5 }}
+			>
+				<Tabs defaultValue="connect" className="w-full">
+					<TabsList className="grid w-full grid-cols-2 mb-6">
+						<TabsTrigger value="connect">Connect</TabsTrigger>
+						<TabsTrigger value="documentation">Documentation</TabsTrigger>
+					</TabsList>
+
+					<TabsContent value="connect">
+						<Card className="border-2 border-border">
+							<CardHeader>
+								<CardTitle className="text-2xl font-bold">Connect Elasticsearch Cluster</CardTitle>
+								<CardDescription>
+									Connect to your Elasticsearch instance to search and index documents for enhanced
+									search capabilities.
+								</CardDescription>
+							</CardHeader>
+							<CardContent>
+								<Form {...form}>
+									<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-6">
+										{/* Connector Name */}
+										<FormField
+											control={form.control}
+											name="name"
+											render={({ field }) => (
+												<FormItem>
+													<FormLabel>Connector Name</FormLabel>
+													<FormControl>
+														<Input placeholder="My Elasticsearch Connector" {...field} />
+													</FormControl>
+													<FormDescription>
+														A friendly name to identify this connector.
+													</FormDescription>
+													<FormMessage />
+												</FormItem>
+											)}
+										/>
+
+										{/* Connection Details */}
+										<div className="space-y-4">
+											<h3 className="text-lg font-medium">Connection Details</h3>
+
+											<FormField
+												control={form.control}
+												name="endpoint_url"
+												render={({ field }) => (
+													<FormItem>
+														<FormLabel>Elasticsearch Endpoint URL</FormLabel>
+														<FormControl>
+															<Input
+																type="url"
+																autoComplete="off"
+																placeholder="https://your-cluster.es.region.aws.com:443"
+																{...field}
+															/>
+														</FormControl>
+														<FormDescription>
+															Enter the complete Elasticsearch endpoint URL. We'll automatically
+															extract the hostname, port, and SSL settings.
+														</FormDescription>
+														<FormMessage />
+													</FormItem>
+												)}
+											/>
+
+											{/* Show parsed URL details */}
+											{form.watch("endpoint_url") && (
+												<div className="rounded-lg border bg-muted/50 p-3">
+													<h4 className="text-sm font-medium mb-2">Parsed Connection Details:</h4>
+													<div className="text-sm text-muted-foreground space-y-1">
+														{(() => {
+															try {
+																const url = new URL(form.watch("endpoint_url"));
+																return (
+																	<>
+																		<div>
+																			<strong>Hostname:</strong> {url.hostname}
+																		</div>
+																		<div>
+																			<strong>Port:</strong>{" "}
+																			{url.port || (url.protocol === "https:" ? "443" : "80")}
+																		</div>
+																		<div>
+																			<strong>SSL/TLS:</strong>{" "}
+																			{url.protocol === "https:" ? "Enabled" : "Disabled"}
+																		</div>
+																	</>
+																);
+															} catch {
+																return <div className="text-destructive">Invalid URL format</div>;
+															}
+														})()}
+													</div>
+												</div>
+											)}
+										</div>
+
+										{/* Authentication */}
+										<div className="space-y-4">
+											<h3 className="text-lg font-medium">Authentication</h3>
+
+											<FormField
+												control={form.control}
+												name="auth_method"
+												render={({ field }) => (
+													<FormItem className="space-y-3">
+														<FormControl>
+															<RadioGroup.Root
+																onValueChange={(value) => {
+																	field.onChange(value);
+																	// Clear auth fields when method changes
+																	if (value !== "basic") {
+																		form.setValue("username", "");
+																		form.setValue("password", "");
+																	}
+																	if (value !== "api_key") {
+																		form.setValue("ELASTICSEARCH_API_KEY", "");
+																	}
+																}}
+																value={field.value}
+																className="flex flex-col space-y-2"
+															>
+																<div className="flex items-center space-x-2">
+																	<RadioGroup.Item
+																		value="api_key"
+																		id={authApiKeyId}
+																		className="aspect-square h-4 w-4 rounded-full border border-primary text-primary ring-offset-background focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground"
+																	>
+																		<RadioGroup.Indicator className="flex items-center justify-center">
+																			<div className="h-2.5 w-2.5 rounded-full bg-current" />
+																		</RadioGroup.Indicator>
+																	</RadioGroup.Item>
+																	<Label htmlFor={authApiKeyId}>API Key</Label>
+																</div>
+
+																<div className="flex items-center space-x-2">
+																	<RadioGroup.Item
+																		value="basic"
+																		id={authBasicId}
+																		className="aspect-square h-4 w-4 rounded-full border border-primary text-primary ring-offset-background focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground"
+																	>
+																		<RadioGroup.Indicator className="flex items-center justify-center">
+																			<div className="h-2.5 w-2.5 rounded-full bg-current" />
+																		</RadioGroup.Indicator>
+																	</RadioGroup.Item>
+																	<Label htmlFor={authBasicId}>Username & Password</Label>
+																</div>
+															</RadioGroup.Root>
+														</FormControl>
+														<FormMessage />
+													</FormItem>
+												)}
+											/>
+
+											{/* Basic Auth Fields */}
+											{form.watch("auth_method") === "basic" && (
+												<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
+													<FormField
+														control={form.control}
+														name="username"
+														render={({ field }) => (
+															<FormItem>
+																<FormLabel>Username</FormLabel>
+																<FormControl>
+																	<Input placeholder="elastic" autoComplete="username" {...field} />
+																</FormControl>
+																<FormMessage />
+															</FormItem>
+														)}
+													/>
+
+													<FormField
+														control={form.control}
+														name="password"
+														render={({ field }) => (
+															<FormItem>
+																<FormLabel>Password</FormLabel>
+																<FormControl>
+																	<Input
+																		type="password"
+																		placeholder="Password"
+																		autoComplete="current-password"
+																		{...field}
+																	/>
+																</FormControl>
+																<FormMessage />
+															</FormItem>
+														)}
+													/>
+												</div>
+											)}
+
+											{/* API Key Field */}
+											{form.watch("auth_method") === "api_key" && (
+												<FormField
+													control={form.control}
+													name="ELASTICSEARCH_API_KEY"
+													render={({ field }) => (
+														<FormItem>
+															<FormLabel>API Key</FormLabel>
+															<FormControl>
+																<Input
+																	type="password"
+																	placeholder="Your API Key Here"
+																	autoComplete="off"
+																	{...field}
+																/>
+															</FormControl>
+															<FormDescription>
+																Enter your Elasticsearch API key (base64 encoded). This will be
+																stored securely.
+															</FormDescription>
+															<FormMessage />
+														</FormItem>
+													)}
+												/>
+											)}
+
+											{/* Index Selection */}
+											<FormField
+												control={form.control}
+												name="indices"
+												render={({ field }) => (
+													<FormItem>
+														<FormLabel>Index Selection </FormLabel>
+														<FormControl>
+															<Input placeholder="logs-*, documents-*, app-logs" {...field} />
+														</FormControl>
+														<FormDescription>
+															Comma-separated indices to search (e.g., "logs-*, documents-*").
+														</FormDescription>
+														<FormMessage />
+													</FormItem>
+												)}
+											/>
+
+											{/* Show parsed indices as badges */}
+											{form.watch("indices")?.trim() && (
+												<div className="rounded-lg border bg-muted/50 p-3">
+													<h4 className="text-sm font-medium mb-2">Selected Indices:</h4>
+													<div className="flex flex-wrap gap-2">
+														{stringToArray(form.watch("indices")).map((index) => (
+															<Badge key={index} variant="secondary" className="text-xs">
+																{index}
+															</Badge>
+														))}
+													</div>
+												</div>
+											)}
+
+											<Alert className="bg-muted">
+												<Info className="h-4 w-4" />
+												<AlertTitle>Index Selection Tips</AlertTitle>
+												<AlertDescription className="mt-2">
+													<ul className="list-disc pl-4 space-y-1 text-sm">
+														<li>Use wildcards like "logs-*" to match multiple indices</li>
+														<li>Separate multiple indices with commas</li>
+														<li>
+															Leave empty to search all accessible indices including internal ones
+														</li>
+														<li>Choosing specific indices improves search performance</li>
+													</ul>
+												</AlertDescription>
+											</Alert>
+										</div>
+
+										{/* Advanced Configuration */}
+										<Accordion type="single" collapsible className="w-full">
+											<AccordionItem value="advanced">
+												<AccordionTrigger>Advanced Configuration</AccordionTrigger>
+												<AccordionContent className="space-y-4">
+													{/* Default Search Query */}
+													<FormField
+														control={form.control}
+														name="query"
+														render={({ field }) => (
+															<FormItem>
+																<FormLabel>
+																	Default Search Query{" "}
+																	<span className="text-muted-foreground">(Optional)</span>
+																</FormLabel>
+																<FormControl>
+																	<Input placeholder="*" {...field} />
+																</FormControl>
+																<FormDescription>
+																	Default Elasticsearch query to use for searches. Use "*" to match
+																	all documents.
+																</FormDescription>
+																<FormMessage />
+															</FormItem>
+														)}
+													/>
+
+													{/* Form Fields */}
+													<FormField
+														control={form.control}
+														name="search_fields"
+														render={({ field }) => (
+															<FormItem>
+																<FormLabel>
+																	Search Fields{" "}
+																	<span className="text-muted-foreground">(Optional)</span>
+																</FormLabel>
+																<FormControl>
+																	<Input placeholder="title, content, description" {...field} />
+																</FormControl>
+																<FormDescription>
+																	Comma-separated list of specific fields to search in (e.g.,
+																	"title, content, description"). Leave empty to search all fields.
+																</FormDescription>
+																<FormMessage />
+															</FormItem>
+														)}
+													/>
+
+													{/* Show parsed search fields as badges */}
+													{form.watch("search_fields")?.trim() && (
+														<div className="rounded-lg border bg-muted/50 p-3">
+															<h4 className="text-sm font-medium mb-2">Search Fields:</h4>
+															<div className="flex flex-wrap gap-2">
+																{stringToArray(form.watch("search_fields")).map((field) => (
+																	<Badge key={field} variant="outline" className="text-xs">
+																		{field}
+																	</Badge>
+																))}
+															</div>
+														</div>
+													)}
+
+													<FormField
+														control={form.control}
+														name="max_documents"
+														render={({ field }) => (
+															<FormItem>
+																<FormLabel>
+																	Maximum Documents{" "}
+																	<span className="text-muted-foreground">(Optional)</span>
+																</FormLabel>
+																<FormControl>
+																	<Input
+																		type="number"
+																		placeholder="1000"
+																		min="1"
+																		max="10000"
+																		{...field}
+																		onChange={(e) =>
+																			field.onChange(
+																				e.target.value === ""
+																					? undefined
+																					: parseInt(e.target.value, 10)
+																			)
+																		}
+																	/>
+																</FormControl>
+																<FormDescription>
+																	Maximum number of documents to retrieve per search (1-10,000).
+																	Leave empty to use Elasticsearch's default limit.
+																</FormDescription>
+																<FormMessage />
+															</FormItem>
+														)}
+													/>
+												</AccordionContent>
+											</AccordionItem>
+										</Accordion>
+
+										<Separator />
+
+										<div className="flex justify-end">
+											<Button type="submit" disabled={isSubmitting} className="w-full sm:w-auto">
+												{isSubmitting ? (
+													<>
+														<Loader2 className="mr-2 h-4 w-4 animate-spin" />
+														Connecting...
+													</>
+												) : (
+													<>
+														<Check className="mr-2 h-4 w-4" />
+														Connect Elasticsearch
+													</>
+												)}
+											</Button>
+										</div>
+									</form>
+								</Form>
+							</CardContent>
+							<CardFooter className="flex flex-col items-start border-t bg-muted/50 px-6 py-4">
+								<h4 className="text-sm font-medium">
+									What you get with Elasticsearch integration:
+								</h4>
+								<ul className="mt-2 list-disc pl-5 text-sm text-muted-foreground">
+									<li>Search across your indexed documents and logs</li>
+									<li>Access structured and unstructured data from your cluster</li>
+									<li>Leverage existing Elasticsearch indices for enhanced search</li>
+									<li>Real-time search capabilities with powerful query features</li>
+									<li>Integration with your existing Elasticsearch infrastructure</li>
+								</ul>
+							</CardFooter>
+						</Card>
+					</TabsContent>
+
+					<TabsContent value="documentation">
+						<Card className="border-2 border-border">
+							<CardHeader>
+								<CardTitle className="text-2xl font-bold">
+									Elasticsearch Connector Documentation
+								</CardTitle>
+								<CardDescription>
+									Learn how to set up and use the Elasticsearch connector to search your data.
+								</CardDescription>
+							</CardHeader>
+							<CardContent className="space-y-6">
+								<div>
+									<h3 className="text-xl font-semibold mb-2">How it works</h3>
+									<p className="text-muted-foreground">
+										The Elasticsearch connector allows you to search and retrieve documents from
+										your Elasticsearch cluster. Configure connection details, select specific
+										indices, and set search parameters to make your existing data searchable within
+										SurfSense.
+									</p>
+								</div>
+
+								<Accordion type="single" collapsible className="w-full">
+									<AccordionItem value="connection">
+										<AccordionTrigger className="text-lg font-medium">
+											Connection Setup
+										</AccordionTrigger>
+										<AccordionContent className="space-y-4">
+											<ol className="list-decimal pl-5 space-y-3">
+												<li>
+													<strong>Endpoint URL:</strong> Enter the complete Elasticsearch endpoint
+													URL (e.g., https://your-cluster.es.region.aws.com:443). We'll
+													automatically extract hostname, port, and SSL settings.
+												</li>
+												<li>
+													<strong>Authentication:</strong> Choose the appropriate method:
+													<ul className="list-disc pl-5 mt-1">
+														<li>
+															<strong>API Key:</strong> Base64 encoded API key (recommended for
+															security)
+														</li>
+														<li>
+															<strong>Username/Password:</strong> Basic authentication credentials
+														</li>
+													</ul>
+												</li>
+												<li>
+													<strong>Index Selection:</strong> Specify which indices to search using
+													comma-separated patterns (e.g., "logs-*, documents-*")
+												</li>
+											</ol>
+										</AccordionContent>
+									</AccordionItem>
+
+									<AccordionItem value="advanced">
+										<AccordionTrigger className="text-lg font-medium">
+											Advanced Configuration
+										</AccordionTrigger>
+										<AccordionContent className="space-y-4">
+											<p className="text-muted-foreground">
+												Fine-tune your Elasticsearch connector with these optional settings:
+											</p>
+											<ul className="list-disc pl-5 space-y-2">
+												<li>
+													<strong>Search Fields:</strong> Limit searches to specific fields (e.g.,
+													"title, content") for better relevance
+												</li>
+												<li>
+													<strong>Default Query:</strong> Set a default Elasticsearch query pattern
+												</li>
+												<li>
+													<strong>Max Documents:</strong> Limit the number of documents returned per
+													search (1-10,000)
+												</li>
+											</ul>
+										</AccordionContent>
+									</AccordionItem>
+
+									<AccordionItem value="troubleshooting">
+										<AccordionTrigger className="text-lg font-medium">
+											Troubleshooting
+										</AccordionTrigger>
+										<AccordionContent className="space-y-4">
+											<div className="space-y-4">
+												<div>
+													<h4 className="font-medium mb-2">Common Connection Issues:</h4>
+													<ul className="list-disc pl-5 space-y-2 text-muted-foreground">
+														<li>
+															<strong>Connection Refused:</strong> Check hostname and port. Ensure
+															Elasticsearch is running.
+														</li>
+														<li>
+															<strong>Authentication Failed:</strong> Verify credentials. For API
+															keys, ensure they have proper permissions.
+														</li>
+														<li>
+															<strong>SSL Errors:</strong> Try disabling SSL for local development
+															or check certificate validity.
+														</li>
+														<li>
+															<strong>No Indices Found:</strong> Ensure your credentials have
+															permission to list and read indices.
+														</li>
+													</ul>
+												</div>
+
+												<Alert className="bg-muted">
+													<Info className="h-4 w-4" />
+													<AlertTitle>Security Note</AlertTitle>
+													<AlertDescription>
+														For production environments, use API keys with minimal required
+														permissions: cluster monitoring and read access to specific indices.
+													</AlertDescription>
+												</Alert>
+											</div>
+										</AccordionContent>
+									</AccordionItem>
+								</Accordion>
+							</CardContent>
+						</Card>
+					</TabsContent>
+				</Tabs>
+			</motion.div>
+		</div>
+	);
+}

surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx

@@ -59,6 +59,13 @@ const connectorCategories: ConnectorCategory[] = [
 				icon: getConnectorIcon(EnumConnectorName.LINKUP_API, "h-6 w-6"),
 				status: "available",
 			},
+			{
+				id: "elasticsearch-connector",
+				title: "Elasticsearch",
+				description: "Connect to Elasticsearch to index and search documents, logs and metrics.",
+				icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"),
+				status: "available",
+			},	
 			{
 				id: "baidu-search-api",
 				title: "Baidu Search",

surfsense_web/components/dashboard-breadcrumb.tsx

@@ -88,6 +88,7 @@ export function DashboardBreadcrumb() {
 								"serper-api": "Serper API",
 								"linkup-api": "LinkUp API",
 								"luma-connector": "Luma",
+								"elasticsearch-connector": "Elasticsearch",
 							};
 
 							const connectorLabel = connectorLabels[connectorType] || connectorType;

surfsense_web/components/editConnector/types.ts

@@ -51,5 +51,6 @@ export const editConnectorSchema = z.object({
 	GOOGLE_CALENDAR_REFRESH_TOKEN: z.string().optional(),
 	GOOGLE_CALENDAR_CALENDAR_IDS: z.string().optional(),
 	LUMA_API_KEY: z.string().optional(),
+	ELASTICSEARCH_API_KEY: z.string().optional(),
 });
 export type EditConnectorFormValues = z.infer<typeof editConnectorSchema>;

surfsense_web/components/homepage/integrations.tsx

@@ -13,6 +13,7 @@ const INTEGRATIONS: Integration[] = [
 		name: "LinkUp",
 		icon: "https://framerusercontent.com/images/7zeIm6t3f1HaSltkw8upEvsD80.png?scale-down-to=512",
 	},
+	{ name: "Elasticsearch", icon: "https://cdn.simpleicons.org/elastic/00A9E5" },
 
 	// Communication
 	{ name: "Slack", icon: "https://cdn.simpleicons.org/slack/4A154B" },

surfsense_web/contracts/enums/connector.ts

@@ -16,4 +16,5 @@ export enum EnumConnectorName {
 	GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR",
 	AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR",
 	LUMA_CONNECTOR = "LUMA_CONNECTOR",
+	ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR",
 }

surfsense_web/contracts/enums/connectorIcons.tsx

@@ -1,6 +1,7 @@
 import {
 	IconBook,
 	IconBrandDiscord,
+	IconBrandElastic,
 	IconBrandGithub,
 	IconBrandNotion,
 	IconBrandSlack,
@@ -56,6 +57,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas
 			return <IconChecklist {...iconProps} />;
 		case EnumConnectorName.LUMA_CONNECTOR:
 			return <IconSparkles {...iconProps} />;
+		case EnumConnectorName.ELASTICSEARCH_CONNECTOR:
+			return <IconBrandElastic {...iconProps} />;
 		// Additional cases for non-enum connector types
 		case "YOUTUBE_VIDEO":
 			return <IconBrandYoutube {...iconProps} />;

surfsense_web/hooks/use-connector-edit-page.ts

@@ -96,6 +96,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
 			JIRA_EMAIL: "",
 			JIRA_API_TOKEN: "",
 			LUMA_API_KEY: "",
+			ELASTICSEARCH_API_KEY: "",
 		},
 	});
 
@@ -140,6 +141,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
 					JIRA_EMAIL: config.JIRA_EMAIL || "",
 					JIRA_API_TOKEN: config.JIRA_API_TOKEN || "",
 					LUMA_API_KEY: config.LUMA_API_KEY || "",
+					ELASTICSEARCH_API_KEY: config.ELASTICSEARCH_API_KEY || "",
 				});
 				if (currentConnector.connector_type === "GITHUB_CONNECTOR") {
 					const savedRepos = config.repo_full_names || [];
@@ -457,6 +459,16 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
 						newConfig = { LUMA_API_KEY: formData.LUMA_API_KEY };
 					}
 					break;
+				case "ELASTICSEARCH_CONNECTOR":
+					if (formData.ELASTICSEARCH_API_KEY !== originalConfig.ELASTICSEARCH_API_KEY) {
+						if (!formData.ELASTICSEARCH_API_KEY) {
+							toast.error("Elasticsearch API Key cannot be empty.");
+							setIsSaving(false);
+							return;
+						}
+						newConfig = { ELASTICSEARCH_API_KEY: formData.ELASTICSEARCH_API_KEY };
+					}
+					break;
 			}
 
 			if (newConfig !== null) {
@@ -545,6 +557,11 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
 						editForm.setValue("JIRA_API_TOKEN", newlySavedConfig.JIRA_API_TOKEN || "");
 					} else if (connector.connector_type === "LUMA_CONNECTOR") {
 						editForm.setValue("LUMA_API_KEY", newlySavedConfig.LUMA_API_KEY || "");
+					} else if (connector.connector_type === "ELASTICSEARCH_CONNECTOR") {
+						editForm.setValue(
+							"ELASTICSEARCH_API_KEY",
+							newlySavedConfig.ELASTICSEARCH_API_KEY || ""
+						);
 					}
 				}
 				if (connector.connector_type === "GITHUB_CONNECTOR") {

surfsense_web/hooks/use-document-by-chunk.ts

@@ -35,7 +35,8 @@ export type DocumentType =
 	| "CLICKUP_CONNECTOR"
 	| "GOOGLE_CALENDAR_CONNECTOR"
 	| "GOOGLE_GMAIL_CONNECTOR"
-	| "LUMA_CONNECTOR";
+	| "LUMA_CONNECTOR"
+	| "ELASTICSEARCH_CONNECTOR";
 
 export function useDocumentByChunk() {
 	const [document, setDocument] = useState<DocumentWithChunks | null>(null);

surfsense_web/hooks/use-documents.ts

@@ -29,7 +29,8 @@ export type DocumentType =
 	| "GOOGLE_CALENDAR_CONNECTOR"
 	| "GOOGLE_GMAIL_CONNECTOR"
 	| "AIRTABLE_CONNECTOR"
-	| "LUMA_CONNECTOR";
+	| "LUMA_CONNECTOR"
+	| "ELASTICSEARCH_CONNECTOR";
 
 export interface UseDocumentsOptions {
 	page?: number;

surfsense_web/lib/connectors/utils.ts

@@ -17,6 +17,7 @@ export const getConnectorTypeDisplay = (type: string): string => {
 		GOOGLE_GMAIL_CONNECTOR: "Google Gmail",
 		AIRTABLE_CONNECTOR: "Airtable",
 		LUMA_CONNECTOR: "Luma",
+		ELASTICSEARCH_CONNECTOR: "Elasticsearch",
 	};
 	return typeMap[type] || type;
 };

surfsense_web/package.json

@@ -32,6 +32,7 @@
 		"@radix-ui/react-dropdown-menu": "^2.1.15",
 		"@radix-ui/react-label": "^2.1.7",
 		"@radix-ui/react-popover": "^1.1.14",
+		"@radix-ui/react-radio-group": "^1.3.8",
 		"@radix-ui/react-scroll-area": "^1.2.9",
 		"@radix-ui/react-select": "^2.2.5",
 		"@radix-ui/react-separator": "^1.1.7",

surfsense_web/pnpm-lock.yaml

@@ -47,6 +47,9 @@ importers:
       '@radix-ui/react-popover':
         specifier: ^1.1.14
         version: 1.1.14(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
+      '@radix-ui/react-radio-group':
+        specifier: ^1.3.8
+        version: 1.3.8(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
       '@radix-ui/react-scroll-area':
         specifier: ^1.2.9
         version: 1.2.9(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
@@ -118,7 +121,7 @@ importers:
         version: 15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
       fumadocs-mdx:
         specifier: ^11.7.1
-        version: 11.7.1(acorn@8.14.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0)
+        version: 11.7.1(acorn@8.15.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0)
       fumadocs-ui:
         specifier: ^15.6.6
         version: 15.6.6(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(tailwindcss@4.1.11)
@@ -1735,6 +1738,19 @@ packages:
       '@types/react-dom':
         optional: true
 
+  '@radix-ui/react-presence@1.1.5':
+    resolution: {integrity: sha512-/jfEwNDdQVBCNvjkGit4h6pMOzq8bHkopq458dPt2lMjx+eBQUohZNG9A7DtO/O5ukSbxuaNGXMjHicgwy6rQQ==}
+    peerDependencies:
+      '@types/react': '*'
+      '@types/react-dom': '*'
+      react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+      react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+    peerDependenciesMeta:
+      '@types/react':
+        optional: true
+      '@types/react-dom':
+        optional: true
+
   '@radix-ui/react-primitive@1.0.0':
     resolution: {integrity: sha512-EyXe6mnRlHZ8b6f4ilTDrXmkLShICIuOTTj0GX4w1rp+wSxf3+TD05u1UOITC8VsJ2a9nwHvdXtOXEOl0Cw/zQ==}
     peerDependencies:
@@ -1793,6 +1809,19 @@ packages:
       '@types/react-dom':
         optional: true
 
+  '@radix-ui/react-radio-group@1.3.8':
+    resolution: {integrity: sha512-VBKYIYImA5zsxACdisNQ3BjCBfmbGH3kQlnFVqlWU4tXwjy7cGX8ta80BcrO+WJXIn5iBylEH3K6ZTlee//lgQ==}
+    peerDependencies:
+      '@types/react': '*'
+      '@types/react-dom': '*'
+      react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+      react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+    peerDependenciesMeta:
+      '@types/react':
+        optional: true
+      '@types/react-dom':
+        optional: true
+
   '@radix-ui/react-roving-focus@1.1.10':
     resolution: {integrity: sha512-dT9aOXUen9JSsxnMPv/0VqySQf5eDQ6LCk5Sw28kamz8wSOW2bJdlX2Bg5VUIIcV+6XlHpWTIuTPCf/UNIyq8Q==}
     peerDependencies:
@@ -1806,6 +1835,19 @@ packages:
       '@types/react-dom':
         optional: true
 
+  '@radix-ui/react-roving-focus@1.1.11':
+    resolution: {integrity: sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA==}
+    peerDependencies:
+      '@types/react': '*'
+      '@types/react-dom': '*'
+      react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+      react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+    peerDependenciesMeta:
+      '@types/react':
+        optional: true
+      '@types/react-dom':
+        optional: true
+
   '@radix-ui/react-scroll-area@1.2.9':
     resolution: {integrity: sha512-YSjEfBXnhUELsO2VzjdtYYD4CfQjvao+lhhrX5XsHD7/cyUNzljF1FHEbgTPN7LH2MClfwRMIsYlqTYpKTTe2A==}
     peerDependencies:
@@ -6554,7 +6596,7 @@ snapshots:
 
   '@marijn/find-cluster-break@1.0.2': {}
 
-  '@mdx-js/mdx@3.1.0(acorn@8.14.0)':
+  '@mdx-js/mdx@3.1.0(acorn@8.15.0)':
     dependencies:
       '@types/estree': 1.0.8
       '@types/estree-jsx': 1.0.5
@@ -6568,7 +6610,7 @@ snapshots:
       hast-util-to-jsx-runtime: 2.3.6
       markdown-extensions: 2.0.0
       recma-build-jsx: 1.0.0
-      recma-jsx: 1.0.0(acorn@8.14.0)
+      recma-jsx: 1.0.0(acorn@8.15.0)
       recma-stringify: 1.0.0
       rehype-recma: 1.0.0
       remark-mdx: 3.1.0
@@ -7260,6 +7302,16 @@ snapshots:
       '@types/react': 19.1.8
       '@types/react-dom': 19.1.6(@types/react@19.1.8)
 
+  '@radix-ui/react-presence@1.1.5(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
+    dependencies:
+      '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.1.8)(react@19.1.0)
+      react: 19.1.0
+      react-dom: 19.1.0(react@19.1.0)
+    optionalDependencies:
+      '@types/react': 19.1.8
+      '@types/react-dom': 19.1.6(@types/react@19.1.8)
+
   '@radix-ui/react-primitive@1.0.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
     dependencies:
       '@babel/runtime': 7.26.9
@@ -7305,6 +7357,24 @@ snapshots:
       '@types/react': 19.1.8
       '@types/react-dom': 19.1.6(@types/react@19.1.8)
 
+  '@radix-ui/react-radio-group@1.3.8(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
+    dependencies:
+      '@radix-ui/primitive': 1.1.3
+      '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-context': 1.1.2(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-direction': 1.1.1(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-presence': 1.1.5(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
+      '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
+      '@radix-ui/react-roving-focus': 1.1.11(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
+      '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-use-previous': 1.1.1(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-use-size': 1.1.1(@types/react@19.1.8)(react@19.1.0)
+      react: 19.1.0
+      react-dom: 19.1.0(react@19.1.0)
+    optionalDependencies:
+      '@types/react': 19.1.8
+      '@types/react-dom': 19.1.6(@types/react@19.1.8)
+
   '@radix-ui/react-roving-focus@1.1.10(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
     dependencies:
       '@radix-ui/primitive': 1.1.2
@@ -7322,6 +7392,23 @@ snapshots:
       '@types/react': 19.1.8
       '@types/react-dom': 19.1.6(@types/react@19.1.8)
 
+  '@radix-ui/react-roving-focus@1.1.11(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
+    dependencies:
+      '@radix-ui/primitive': 1.1.3
+      '@radix-ui/react-collection': 1.1.7(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
+      '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-context': 1.1.2(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-direction': 1.1.1(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-id': 1.1.1(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
+      '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.1.8)(react@19.1.0)
+      '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.1.8)(react@19.1.0)
+      react: 19.1.0
+      react-dom: 19.1.0(react@19.1.0)
+    optionalDependencies:
+      '@types/react': 19.1.8
+      '@types/react-dom': 19.1.6(@types/react@19.1.8)
+
   '@radix-ui/react-scroll-area@1.2.9(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
     dependencies:
       '@radix-ui/number': 1.1.1
@@ -9073,9 +9160,9 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  fumadocs-mdx@11.7.1(acorn@8.14.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0):
+  fumadocs-mdx@11.7.1(acorn@8.15.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0):
     dependencies:
-      '@mdx-js/mdx': 3.1.0(acorn@8.14.0)
+      '@mdx-js/mdx': 3.1.0(acorn@8.15.0)
       '@standard-schema/spec': 1.0.0
       chokidar: 4.0.3
       esbuild: 0.25.8
@@ -11137,9 +11224,9 @@ snapshots:
       estree-util-build-jsx: 3.0.1
       vfile: 6.0.3
 
-  recma-jsx@1.0.0(acorn@8.14.0):
+  recma-jsx@1.0.0(acorn@8.15.0):
     dependencies:
-      acorn-jsx: 5.3.2(acorn@8.14.0)
+      acorn-jsx: 5.3.2(acorn@8.15.0)
       estree-util-to-js: 2.0.0
       recma-parse: 1.0.0
       recma-stringify: 1.0.0