added initial backend files

surfsense_backend/app/connectors/elasticsearch_connector.py

@@ -0,0 +1,217 @@
+# import asyncio
+import json
+import logging
+from collections.abc import AsyncGenerator
+from typing import Any
+
+from elasticsearch import AsyncElasticsearch
+from elasticsearch.exceptions import AuthenticationException, ConnectionError
+
+from app.schemas.connector import ConnectorConfig
+from app.schemas.documents import DocumentCreate, DocumentType
+from app.utils.document_converter import DocumentConverter
+
+logger = logging.getLogger(__name__)
+
+
+class ElasticsearchConnector:
+    def __init__(self, config: ConnectorConfig):
+        self.config = config
+        self.client: AsyncElasticsearch | None = None
+        self.document_converter = DocumentConverter()
+
+    async def connect(self) -> bool:
+        """Establish connection to Elasticsearch"""
+        try:
+            connection_params = {
+                "hosts": [f"{self.config.hostname}:{self.config.port}"],
+                "verify_certs": self.config.ssl_enabled
+                if hasattr(self.config, "ssl_enabled")
+                else True,
+                "request_timeout": 30,
+            }
+
+            # Add authentication if provided
+            if (
+                hasattr(self.config, "username")
+                and hasattr(self.config, "password")
+                and self.config.username
+                and self.config.password
+            ):
+                connection_params["basic_auth"] = (
+                    self.config.username,
+                    self.config.password,
+                )
+
+            # Add API key authentication if provided
+            if hasattr(self.config, "api_key") and self.config.api_key:
+                connection_params["api_key"] = self.config.api_key
+
+            self.client = AsyncElasticsearch(**connection_params)
+
+            # Test connection
+            await self.client.info()
+            logger.info("Successfully connected to Elasticsearch")
+            return True
+
+        except (ConnectionError, AuthenticationException) as e:
+            logger.error(f"Failed to connect to Elasticsearch: {e}")
+            return False
+        except Exception as e:
+            logger.error(f"Unexpected error connecting to Elasticsearch: {e}")
+            return False
+
+    async def disconnect(self):
+        """Close Elasticsearch connection"""
+        if self.client:
+            await self.client.close()
+            self.client = None
+
+    async def test_connection(self) -> dict[str, Any]:
+        """Test the Elasticsearch connection and return cluster info"""
+        try:
+            if not self.client:
+                await self.connect()
+
+            if not self.client:
+                return {"success": False, "error": "Failed to establish connection"}
+
+            info = await self.client.info()
+            indices = await self.client.cat.indices(format="json")
+
+            return {
+                "success": True,
+                "cluster_name": info.get("cluster_name"),
+                "version": info.get("version", {}).get("number"),
+                "indices_count": len(indices) if indices else 0,
+            }
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    async def get_indices(self) -> list[str]:
+        """Get list of available indices"""
+        try:
+            if not self.client:
+                await self.connect()
+
+            indices = await self.client.cat.indices(format="json")
+            return [idx["index"] for idx in indices if not idx["index"].startswith(".")]
+        except Exception as e:
+            logger.error(f"Error fetching indices: {e}")
+            return []
+
+    async def search_documents(
+        self,
+        query: str,
+        indices: list[str] | None = None,
+        size: int = 100,
+        fields: list[str] | None = None,
+    ) -> AsyncGenerator[DocumentCreate, None]:
+        """Search documents in Elasticsearch and yield them as DocumentCreate objects"""
+        try:
+            if not self.client:
+                await self.connect()
+
+            if not self.client:
+                logger.error("No Elasticsearch client available")
+                return
+
+            # Build search query
+            search_body = {
+                "query": {
+                    "multi_match": {
+                        "query": query,
+                        "fields": fields or ["*"],
+                        "type": "best_fields",
+                        "fuzziness": "AUTO",
+                    }
+                },
+                "size": size,
+                "_source": True,
+            }
+
+            # Search across specified indices or all indices
+            index_pattern = ",".join(indices) if indices else "*"
+
+            response = await self.client.search(index=index_pattern, body=search_body)
+
+            for hit in response["hits"]["hits"]:
+                try:
+                    # Convert Elasticsearch document to SurfSense document
+                    document = await self._convert_es_document(hit)
+                    if document:
+                        yield document
+                except Exception as e:
+                    logger.warning(f"Failed to convert document {hit.get('_id')}: {e}")
+                    continue
+
+        except Exception as e:
+            logger.error(f"Error searching Elasticsearch: {e}")
+
+    async def get_document_by_id(
+        self, index: str, doc_id: str
+    ) -> DocumentCreate | None:
+        """Get a specific document by ID"""
+        try:
+            if not self.client:
+                await self.connect()
+
+            response = await self.client.get(index=index, id=doc_id)
+            return await self._convert_es_document(response)
+        except Exception as e:
+            logger.error(f"Error fetching document {doc_id} from {index}: {e}")
+            return None
+
+    async def _convert_es_document(
+        self, es_doc: dict[str, Any]
+    ) -> DocumentCreate | None:
+        """Convert Elasticsearch document to SurfSense DocumentCreate"""
+        try:
+            source = es_doc.get("_source", {})
+
+            # Extract title - try common field names
+            title = (
+                source.get("title")
+                or source.get("name")
+                or source.get("subject")
+                or source.get("filename")
+                or f"Document {es_doc.get('_id', 'Unknown')}"
+            )
+
+            # Extract content - try common field names
+            content = (
+                source.get("content")
+                or source.get("text")
+                or source.get("body")
+                or source.get("message")
+                or json.dumps(source)  # Fallback to full source as JSON
+            )
+
+            # Generate URL pointing back to Elasticsearch
+            url = f"elasticsearch://{es_doc.get('_index')}/{es_doc.get('_id')}"
+
+            # Convert content to markdown if it's HTML
+            if content and isinstance(content, str):
+                content = await self.document_converter.convert_to_markdown(content)
+
+            return DocumentCreate(
+                title=title[:500],  # Limit title length
+                content=content,
+                url=url,
+                document_type=DocumentType.ELASTICSEARCH,
+                metadata={
+                    "elasticsearch_index": es_doc.get("_index"),
+                    "elasticsearch_id": es_doc.get("_id"),
+                    "elasticsearch_score": es_doc.get("_score"),
+                    "source_fields": list(source.keys()),
+                    **{
+                        k: v
+                        for k, v in source.items()
+                        if k not in ["content", "text", "body", "message"]
+                        and isinstance(v, str | int | float | bool)
+                    },
+                },
+            )
+        except Exception as e:
+            logger.error(f"Error converting Elasticsearch document: {e}")
+            return None

surfsense_backend/app/db.py

@@ -50,6 +50,7 @@ class DocumentType(str, Enum):
     GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR"
     AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR"
     LUMA_CONNECTOR = "LUMA_CONNECTOR"
+    ELASTICSEARCH = "ELASTICSEARCH"
 
 
 class SearchSourceConnectorType(str, Enum):
@@ -68,6 +69,7 @@ class SearchSourceConnectorType(str, Enum):
     GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR"
     AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR"
     LUMA_CONNECTOR = "LUMA_CONNECTOR"
+    ELASTICSEARCH = "ELASTICSEARCH"
 
 
 class ChatType(str, Enum):

surfsense_backend/app/routes/__init__.py

@@ -5,6 +5,7 @@ from .airtable_add_connector_route import (
 )
 from .chats_routes import router as chats_router
 from .documents_routes import router as documents_router
+from .elasticsearch_add_connector_route import router as elasticsearch_router
 from .google_calendar_add_connector_route import (
     router as google_calendar_add_connector_router,
 )
@@ -24,6 +25,7 @@ router.include_router(search_spaces_router)
 router.include_router(documents_router)
 router.include_router(podcasts_router)
 router.include_router(chats_router)
+router.include_router(elasticsearch_router)
 router.include_router(search_source_connectors_router)
 router.include_router(google_calendar_add_connector_router)
 router.include_router(google_gmail_add_connector_router)

surfsense_backend/app/routes/elasticsearch_add_connector_route.py

@@ -0,0 +1,526 @@
+import hashlib
+import logging
+from datetime import UTC, datetime
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import BaseModel
+from sqlalchemy import select, update
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import config
+from app.connectors.elasticsearch_connector import ElasticsearchConnector
+from app.db import (
+    Chunk,
+    Document,
+    SearchSourceConnector,
+    SearchSourceConnectorType,
+    get_async_session,
+)
+from app.schemas.users import User
+from app.tasks.connector_indexers.elasticsearch_indexer import ElasticsearchIndexer
+from app.users import current_active_user
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/elasticsearch", tags=["elasticsearch"])
+
+
+class ElasticsearchConnectorConfig(BaseModel):
+    hostname: str
+    port: int = 9200
+    username: str | None = None
+    password: str | None = None
+    api_key: str | None = None
+    ssl_enabled: bool = True
+    indices: list[str] | None = None
+    query: str = "*"
+    search_fields: list[str] | None = None
+    max_documents: int = 1000
+
+
+@router.post("/add-connector")
+async def add_elasticsearch_connector(
+    connector_data: ElasticsearchConnectorConfig,
+    current_user: User = Depends(current_active_user),
+    session: AsyncSession = Depends(get_async_session),
+) -> dict[str, Any]:
+    """Add a new Elasticsearch connector for the current user"""
+
+    # Test the connection before saving
+    elasticsearch_connector = ElasticsearchConnector(connector_data)
+
+    try:
+        connection_test = await elasticsearch_connector.test_connection()
+        if not connection_test.get("success"):
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"Failed to connect to Elasticsearch: {connection_test.get('error')}",
+            )
+    except Exception as e:
+        logger.error(f"Error testing Elasticsearch connection: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Failed to test Elasticsearch connection: {e!s}",
+        ) from e
+    finally:
+        await elasticsearch_connector.disconnect()
+
+    try:
+        # Check if connector already exists for this user
+        existing_connector_result = await session.execute(
+            select(SearchSourceConnector).filter(
+                SearchSourceConnector.user_id == current_user.id,
+                SearchSourceConnector.connector_type
+                == SearchSourceConnectorType.ELASTICSEARCH,
+            )
+        )
+        existing_connector = existing_connector_result.scalars().first()
+
+        if existing_connector:
+            # Update existing connector
+            existing_connector.config = connector_data.model_dump()
+            existing_connector.name = (
+                f"Elasticsearch - {connector_data.hostname}:{connector_data.port}"
+            )
+            existing_connector.is_indexable = True
+            logger.info(
+                f"Updated existing Elasticsearch connector for user {current_user.id}"
+            )
+        else:
+            # Create new connector
+            new_connector = SearchSourceConnector(
+                name=f"Elasticsearch - {connector_data.hostname}:{connector_data.port}",
+                connector_type=SearchSourceConnectorType.ELASTICSEARCH,
+                is_indexable=True,
+                config=connector_data.model_dump(),
+                user_id=current_user.id,
+            )
+            session.add(new_connector)
+            logger.info(
+                f"Created new Elasticsearch connector for user {current_user.id}"
+            )
+
+        await session.commit()
+
+        # Get the connector (either existing or new) to return
+        if existing_connector:
+            connector = existing_connector
+        else:
+            await session.refresh(new_connector)
+            connector = new_connector
+
+        logger.info(
+            f"Successfully saved Elasticsearch connector for user {current_user.id}"
+        )
+
+        return {
+            "success": True,
+            "message": "Elasticsearch connector added successfully",
+            "connector": {
+                "id": connector.id,
+                "name": connector.name,
+                "connector_type": connector.connector_type.value,
+                "is_indexable": connector.is_indexable,
+                "created_at": connector.created_at.isoformat()
+                if connector.created_at
+                else None,
+                "last_indexed_at": connector.last_indexed_at.isoformat()
+                if connector.last_indexed_at
+                else None,
+            },
+        }
+
+    except IntegrityError as e:
+        await session.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"Integrity error: A connector with this configuration already exists. {e!s}",
+        ) from e
+    except Exception as e:
+        logger.error(f"Failed to create Elasticsearch connector: {e!s}")
+        await session.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to create Elasticsearch connector: {e!s}",
+        ) from e
+
+
+@router.get("/connectors")
+async def get_elasticsearch_connectors(
+    current_user: User = Depends(current_active_user),
+    session: AsyncSession = Depends(get_async_session),
+) -> dict[str, Any]:
+    """Get all Elasticsearch connectors for the current user"""
+
+    try:
+        result = await session.execute(
+            select(SearchSourceConnector).filter(
+                SearchSourceConnector.user_id == current_user.id,
+                SearchSourceConnector.connector_type
+                == SearchSourceConnectorType.ELASTICSEARCH,
+            )
+        )
+        connectors = result.scalars().all()
+
+        connector_list = []
+        for connector in connectors:
+            connector_list.append(
+                {
+                    "id": connector.id,
+                    "name": connector.name,
+                    "connector_type": connector.connector_type.value,
+                    "is_indexable": connector.is_indexable,
+                    "created_at": connector.created_at.isoformat()
+                    if connector.created_at
+                    else None,
+                    "last_indexed_at": connector.last_indexed_at.isoformat()
+                    if connector.last_indexed_at
+                    else None,
+                    "config": {
+                        "hostname": connector.config.get("hostname"),
+                        "port": connector.config.get("port"),
+                        "indices": connector.config.get("indices"),
+                        # Don't expose sensitive credentials
+                    },
+                }
+            )
+
+        return {"success": True, "connectors": connector_list}
+
+    except Exception as e:
+        logger.error(f"Error fetching Elasticsearch connectors: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to fetch Elasticsearch connectors",
+        ) from e
+
+
+@router.put("/connectors/{connector_id}")
+async def update_elasticsearch_connector(
+    connector_id: int,
+    connector_data: ElasticsearchConnectorConfig,
+    current_user: User = Depends(current_active_user),
+    session: AsyncSession = Depends(get_async_session),
+) -> dict[str, Any]:
+    """Update an existing Elasticsearch connector"""
+
+    try:
+        # Get the connector
+        result = await session.execute(
+            select(SearchSourceConnector).filter(
+                SearchSourceConnector.id == connector_id,
+                SearchSourceConnector.user_id == current_user.id,
+                SearchSourceConnector.connector_type
+                == SearchSourceConnectorType.ELASTICSEARCH,
+            )
+        )
+        connector = result.scalar_one_or_none()
+
+        if not connector:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail="Elasticsearch connector not found",
+            )
+
+        # Test the new connection
+        elasticsearch_connector = ElasticsearchConnector(connector_data)
+
+        try:
+            connection_test = await elasticsearch_connector.test_connection()
+            if not connection_test.get("success"):
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=f"Failed to connect to Elasticsearch: {connection_test.get('error')}",
+                )
+        finally:
+            await elasticsearch_connector.disconnect()
+
+        # Update the connector
+        connector.config = connector_data.model_dump()
+        connector.name = (
+            f"Elasticsearch - {connector_data.hostname}:{connector_data.port}"
+        )
+
+        await session.commit()
+        await session.refresh(connector)
+
+        return {
+            "success": True,
+            "message": "Elasticsearch connector updated successfully",
+            "connector": {
+                "id": connector.id,
+                "name": connector.name,
+                "connector_type": connector.connector_type.value,
+                "is_indexable": connector.is_indexable,
+                "created_at": connector.created_at.isoformat()
+                if connector.created_at
+                else None,
+                "last_indexed_at": connector.last_indexed_at.isoformat()
+                if connector.last_indexed_at
+                else None,
+            },
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error updating Elasticsearch connector: {e}")
+        await session.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to update Elasticsearch connector",
+        ) from e
+
+
+@router.delete("/connectors/{connector_id}")
+async def delete_elasticsearch_connector(
+    connector_id: int,
+    current_user: User = Depends(current_active_user),
+    session: AsyncSession = Depends(get_async_session),
+) -> dict[str, Any]:
+    """Delete an Elasticsearch connector"""
+
+    try:
+        # Get the connector
+        result = await session.execute(
+            select(SearchSourceConnector).filter(
+                SearchSourceConnector.id == connector_id,
+                SearchSourceConnector.user_id == current_user.id,
+                SearchSourceConnector.connector_type
+                == SearchSourceConnectorType.ELASTICSEARCH,
+            )
+        )
+        connector = result.scalar_one_or_none()
+
+        if not connector:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail="Elasticsearch connector not found",
+            )
+
+        # Delete the connector
+        await session.delete(connector)
+        await session.commit()
+
+        return {
+            "success": True,
+            "message": "Elasticsearch connector deleted successfully",
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error deleting Elasticsearch connector: {e}")
+        await session.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Failed to delete Elasticsearch connector",
+        ) from e
+
+
+@router.get("/test-connection")
+async def test_elasticsearch_connection(
+    hostname: str,
+    port: int = 9200,
+    username: str | None = None,
+    password: str | None = None,
+    api_key: str | None = None,
+    ssl_enabled: bool = True,
+    current_user: User = Depends(current_active_user),
+) -> dict[str, Any]:
+    """Test Elasticsearch connection with provided credentials"""
+
+    config_data = ElasticsearchConnectorConfig(
+        hostname=hostname,
+        port=port,
+        username=username,
+        password=password,
+        api_key=api_key,
+        ssl_enabled=ssl_enabled,
+    )
+
+    elasticsearch_connector = ElasticsearchConnector(config_data)
+
+    try:
+        result = await elasticsearch_connector.test_connection()
+        return result
+    except Exception as e:
+        logger.error(f"Error testing Elasticsearch connection: {e}")
+        return {"success": False, "error": str(e)}
+    finally:
+        await elasticsearch_connector.disconnect()
+
+
+@router.get("/indices")
+async def get_elasticsearch_indices(
+    hostname: str,
+    port: int = 9200,
+    username: str | None = None,
+    password: str | None = None,
+    api_key: str | None = None,
+    ssl_enabled: bool = True,
+    current_user: User = Depends(current_active_user),
+) -> dict[str, Any]:
+    """Get list of available Elasticsearch indices"""
+
+    config_data = ElasticsearchConnectorConfig(
+        hostname=hostname,
+        port=port,
+        username=username,
+        password=password,
+        api_key=api_key,
+        ssl_enabled=ssl_enabled,
+    )
+
+    elasticsearch_connector = ElasticsearchConnector(config_data)
+
+    try:
+        if not await elasticsearch_connector.connect():
+            return {"success": False, "error": "Failed to connect to Elasticsearch"}
+
+        indices = await elasticsearch_connector.get_indices()
+        return {"success": True, "indices": indices}
+    except Exception as e:
+        logger.error(f"Error fetching Elasticsearch indices: {e}")
+        return {"success": False, "error": str(e)}
+    finally:
+        await elasticsearch_connector.disconnect()
+
+
+@router.post("/index-connector/{connector_id}")
+async def index_elasticsearch_connector(
+    connector_id: int,
+    search_space_id: int,
+    current_user: User = Depends(current_active_user),
+    session: AsyncSession = Depends(get_async_session),
+) -> dict[str, Any]:
+    """
+    Index documents from an Elasticsearch connector into a search space
+    """
+    try:
+        # Get the connector directly from database
+        stmt = select(SearchSourceConnector).where(
+            SearchSourceConnector.id == connector_id,
+            SearchSourceConnector.user_id == current_user.id,
+        )
+        result = await session.execute(stmt)
+        connector = result.scalar_one_or_none()
+
+        if not connector:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail=f"Connector {connector_id} not found",
+            )
+
+        if connector.connector_type != SearchSourceConnectorType.ELASTICSEARCH:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail="Connector is not an Elasticsearch connector",
+            )
+
+        # Initialize the indexer
+        indexer = ElasticsearchIndexer(connector.config)
+
+        # Test connection first
+        logger.info(f"Testing Elasticsearch connection for connector {connector_id}")
+        connection_test = await indexer.test_connection()
+        if not connection_test.get("success"):
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"Connection test failed: {connection_test.get('error')}",
+            )
+
+        # Index documents
+        logger.info(f"Starting indexing for Elasticsearch connector {connector_id}")
+        indexed_count = 0
+        error_count = 0
+
+        async for document in indexer.get_documents():
+            try:
+                # Create content hash
+                content_hash = hashlib.sha256(document.content.encode()).hexdigest()
+
+                # Check if document already exists
+                existing_doc_stmt = select(Document).where(
+                    Document.content_hash == content_hash,
+                    Document.search_space_id == search_space_id,
+                )
+                existing_doc_result = await session.execute(existing_doc_stmt)
+                if existing_doc_result.scalar_one_or_none():
+                    logger.info(f"Document already exists, skipping: {document.title}")
+                    continue
+
+                # Generate embedding for document
+                embedding = await config.embedding_model_instance.encode(
+                    document.content
+                )
+
+                # Create document
+                db_document = Document(
+                    title=document.title,
+                    content=document.content,
+                    content_hash=content_hash,
+                    document_type=document.document_type,
+                    document_metadata=document.metadata,
+                    search_space_id=search_space_id,
+                    embedding=embedding,
+                )
+                session.add(db_document)
+                await session.flush()  # Get the document ID
+
+                # Create a single chunk with the entire document content
+                chunk_embedding = await config.embedding_model_instance.encode(
+                    document.content
+                )
+                db_chunk = Chunk(
+                    content=document.content,
+                    embedding=chunk_embedding,
+                    document_id=db_document.id,
+                )
+                session.add(db_chunk)
+
+                indexed_count += 1
+
+                if indexed_count % 10 == 0:
+                    logger.info(f"Indexed {indexed_count} Elasticsearch documents")
+                    await session.commit()  # Periodic commit
+
+            except Exception as e:
+                logger.error(f"Error indexing document: {e}")
+                error_count += 1
+
+        await session.commit()
+
+        # Update last_indexed_at timestamp
+        update_stmt = (
+            update(SearchSourceConnector)
+            .where(SearchSourceConnector.id == connector_id)
+            .values(last_indexed_at=datetime.now(UTC))
+        )
+        await session.execute(update_stmt)
+        await session.commit()
+
+        logger.info(
+            f"Elasticsearch indexing completed: {indexed_count} documents indexed, {error_count} errors"
+        )
+
+        return {
+            "success": True,
+            "indexed_documents": indexed_count,
+            "errors": error_count,
+            "connector_id": connector_id,
+            "search_space_id": search_space_id,
+            "message": f"Successfully indexed {indexed_count} documents from Elasticsearch",
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error during Elasticsearch indexing: {e}")
+        await session.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Error during indexing: {e!s}",
+        ) from e

surfsense_backend/app/schemas/elasticsearch.py

@@ -0,0 +1,55 @@
+from pydantic import BaseModel, Field
+
+
+class ElasticsearchConnectorConfig(BaseModel):
+    """Configuration for Elasticsearch connector"""
+
+    hostname: str = Field(..., description="Elasticsearch hostname")
+    port: int = Field(default=9200, description="Elasticsearch port")
+    username: str | None = Field(
+        default=None, description="Username for authentication"
+    )
+    password: str | None = Field(
+        default=None, description="Password for authentication"
+    )
+    api_key: str | None = Field(default=None, description="API key for authentication")
+    ssl_enabled: bool = Field(default=True, description="Whether to use SSL/TLS")
+    indices: list[str] | None = Field(
+        default=None, description="Specific indices to search (optional)"
+    )
+    query: str = Field(default="*", description="Default search query")
+    search_fields: list[str] | None = Field(
+        default=None, description="Specific fields to search in"
+    )
+    max_documents: int = Field(
+        default=1000, description="Maximum number of documents to retrieve"
+    )
+
+
+class ElasticsearchTestConnectionRequest(BaseModel):
+    """Request model for testing Elasticsearch connection"""
+
+    hostname: str
+    port: int = 9200
+    username: str | None = None
+    password: str | None = None
+    api_key: str | None = None
+    ssl_enabled: bool = True
+
+
+class ElasticsearchTestConnectionResponse(BaseModel):
+    """Response model for Elasticsearch connection test"""
+
+    success: bool
+    cluster_name: str | None = None
+    version: str | None = None
+    indices_count: int | None = None
+    error: str | None = None
+
+
+class ElasticsearchIndicesResponse(BaseModel):
+    """Response model for Elasticsearch indices list"""
+
+    success: bool
+    indices: list[str] | None = None
+    error: str | None = None

surfsense_backend/app/services/connector_service.py

@@ -2013,3 +2013,104 @@ class ConnectorService:
         }
 
         return result_object, luma_chunks
+
+    async def search_elasticsearch(
+        self,
+        user_query: str,
+        user_id: str,
+        search_space_id: int,
+        top_k: int = 20,
+        search_mode: SearchMode = SearchMode.CHUNKS,
+    ) -> tuple:
+        """
+        Search for Elasticsearch documents and return both the source information and langchain documents
+
+        Args:
+            user_query: The user's query
+            user_id: The user's ID
+            search_space_id: The search space ID to search in
+            top_k: Maximum number of results to return
+            search_mode: Search mode (CHUNKS or DOCUMENTS)
+
+        Returns:
+            tuple: (sources_info, langchain_documents)
+        """
+        if search_mode == SearchMode.CHUNKS:
+            elasticsearch_chunks = await self.chunk_retriever.hybrid_search(
+                query_text=user_query,
+                top_k=top_k,
+                user_id=user_id,
+                search_space_id=search_space_id,
+                document_type="ELASTICSEARCH",
+            )
+        elif search_mode == SearchMode.DOCUMENTS:
+            elasticsearch_chunks = await self.document_retriever.hybrid_search(
+                query_text=user_query,
+                top_k=top_k,
+                user_id=user_id,
+                search_space_id=search_space_id,
+                document_type="ELASTICSEARCH",
+            )
+            # Transform document retriever results to match expected format
+            elasticsearch_chunks = self._transform_document_results(
+                elasticsearch_chunks
+            )
+
+        # Early return if no results
+        if not elasticsearch_chunks:
+            return {
+                "id": 12,
+                "name": "Elasticsearch",
+                "type": "ELASTICSEARCH",
+                "sources": [],
+            }, []
+
+        # Process each chunk and create sources directly without deduplication
+        sources_list = []
+        async with self.counter_lock:
+            for _i, chunk in enumerate(elasticsearch_chunks):
+                # Extract document metadata
+                document = chunk.get("document", {})
+                metadata = document.get("metadata", {})
+
+                # Extract Elasticsearch-specific metadata
+                es_index = metadata.get("elasticsearch_index", "unknown")
+                es_id = metadata.get("elasticsearch_id", "")
+                es_score = metadata.get("elasticsearch_score", 0)
+
+                # Create title with Elasticsearch context
+                title = document.get("title", "Elasticsearch Document")
+                if es_index:
+                    title = f"[{es_index}] {title}"
+
+                # Create description from content
+                description = chunk.get("content", "")[:100]
+                if len(description) == 100:
+                    description += "..."
+
+                # Add score information to description
+                if es_score:
+                    description = f"Score: {es_score:.2f} | {description}"
+
+                source = {
+                    "id": chunk.get("chunk_id", self.source_id_counter),
+                    "title": title,
+                    "description": description,
+                    "url": metadata.get("url", f"elasticsearch://{es_index}/{es_id}"),
+                    "elasticsearch_index": es_index,
+                    "elasticsearch_id": es_id,
+                    "elasticsearch_score": es_score,
+                }
+
+                self.source_id_counter += 1
+                sources_list.append(source)
+
+        # Create result object
+        result_object = {
+            "id": 12,
+            "name": "Elasticsearch",
+            "type": "ELASTICSEARCH",
+            "sources": sources_list,
+        }
+
+        return result_object, elasticsearch_chunks

surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py

@@ -0,0 +1,60 @@
+import asyncio
+import logging
+from collections.abc import AsyncGenerator
+from typing import Any
+
+from app.connectors.elasticsearch_connector import ElasticsearchConnector
+from app.schemas.documents import DocumentCreate
+from app.tasks.connector_indexers.base import BaseConnectorIndexer
+
+logger = logging.getLogger(__name__)
+
+
+class ElasticsearchIndexer(BaseConnectorIndexer):
+    """Indexer for Elasticsearch data sources"""
+
+    def __init__(self, connector_config: dict[str, Any]):
+        super().__init__(connector_config)
+        self.connector = ElasticsearchConnector(self.config)
+
+    async def get_documents(self) -> AsyncGenerator[DocumentCreate, None]:
+        """Fetch documents from Elasticsearch based on configuration"""
+        try:
+            # Connect to Elasticsearch
+            if not await self.connector.connect():
+                logger.error("Failed to connect to Elasticsearch")
+                return
+
+            # Get configuration parameters
+            query = self.connector_config.get("query", "*")
+            indices = self.connector_config.get("indices", None)
+            max_docs = self.connector_config.get("max_documents", 1000)
+            fields = self.connector_config.get("search_fields", None)
+
+            logger.info(f"Searching Elasticsearch with query: {query}")
+
+            # Search documents
+            doc_count = 0
+            async for document in self.connector.search_documents(
+                query=query, indices=indices, size=max_docs, fields=fields
+            ):
+                if doc_count >= max_docs:
+                    break
+
+                yield document
+                doc_count += 1
+
+                # Add small delay to prevent overwhelming Elasticsearch
+                if doc_count % 10 == 0:
+                    await asyncio.sleep(0.1)
+
+            logger.info(f"Fetched {doc_count} documents from Elasticsearch")
+
+        except Exception as e:
+            logger.error(f"Error fetching documents from Elasticsearch: {e}")
+        finally:
+            await self.connector.disconnect()
+
+    async def test_connection(self) -> dict[str, Any]:
+        """Test the Elasticsearch connection"""
+        return await self.connector.test_connection()