feat: added elasticsearch connector

This commit is contained in:
Anish Sarkar 2025-10-12 09:39:04 +05:30
parent 402039f02f
commit 55d752e3c8
27 changed files with 4331 additions and 2499 deletions

View file

@ -75,6 +75,7 @@ Open source and easy to deploy locally.
- Airtable
- Google Calendar
- Luma
- Elasticsearch
- and more to come.....
## 📄 **Supported File Extensions**

View file

@ -0,0 +1,56 @@
"""Add ElasticSearch connector enums
Revision ID: 26
Revises: 25
Create Date: 2025-10-12 12:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers
revision: str = "26"
down_revision: str | None = "25"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# Add enum values
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_type t
JOIN pg_enum e ON t.oid = e.enumtypid
WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'ELASTICSEARCH_CONNECTOR'
) THEN
ALTER TYPE searchsourceconnectortype ADD VALUE 'ELASTICSEARCH_CONNECTOR';
END IF;
END
$$;
"""
)
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_type t
JOIN pg_enum e ON t.oid = e.enumtypid
WHERE t.typname = 'documenttype' AND e.enumlabel = 'ELASTICSEARCH_CONNECTOR'
) THEN
ALTER TYPE documenttype ADD VALUE 'ELASTICSEARCH_CONNECTOR';
END IF;
END
$$;
"""
)
def downgrade() -> None:
"""Remove 'ELASTICSEARCH_CONNECTOR' from enum types."""
pass

View file

@ -488,6 +488,25 @@ async def fetch_documents_by_ids(
)
url = metadata.get("url", "")
elif doc_type == "ELASTICSEARCH_CONNECTOR":
# Prefer explicit title in metadata/source, otherwise fallback to doc.title
es_title = (
metadata.get("title")
or metadata.get("es_title")
or doc.title
or f"Elasticsearch: {metadata.get('elasticsearch_index', '')}"
)
title = es_title
description = metadata.get("description") or (
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
)
# If a link or index info is stored, surface it
url = metadata.get("url", "") or metadata.get(
"elasticsearch_index", ""
)
else: # FILE and other types
title = doc.title
description = (
@ -512,6 +531,7 @@ async def fetch_documents_by_ids(
"SLACK_CONNECTOR": "Slack (Selected)",
"NOTION_CONNECTOR": "Notion (Selected)",
"GITHUB_CONNECTOR": "GitHub (Selected)",
"ELASTICSEARCH_CONNECTOR": "Elasticsearch (Selected)",
"YOUTUBE_VIDEO": "YouTube Videos (Selected)",
"DISCORD_CONNECTOR": "Discord (Selected)",
"JIRA_CONNECTOR": "Jira Issues (Selected)",
@ -1266,6 +1286,33 @@ async def fetch_relevant_documents(
}
)
elif connector == "ELASTICSEARCH_CONNECTOR":
(
source_object,
elasticsearch_chunks,
) = await connector_service.search_elasticsearch(
user_query=reformulated_query,
user_id=user_id,
search_space_id=search_space_id,
top_k=top_k,
search_mode=search_mode,
)
# Add to sources and raw documents
if source_object:
all_sources.append(source_object)
all_raw_documents.extend(elasticsearch_chunks)
# Stream found document count
if streaming_service and writer:
writer(
{
"yield_value": streaming_service.format_terminal_info_delta(
f"🔎 Found {len(elasticsearch_chunks)} Elasticsearch chunks related to your query"
)
}
)
except Exception as e:
logging.error("Error in search_airtable: %s", traceback.format_exc())
error_message = f"Error searching connector {connector}: {e!s}"

View file

@ -51,6 +51,7 @@ def get_connector_emoji(connector_name: str) -> str:
"GOOGLE_CALENDAR_CONNECTOR": "📅",
"AIRTABLE_CONNECTOR": "🗃️",
"LUMA_CONNECTOR": "",
"ELASTICSEARCH_CONNECTOR": "🔎",
}
return connector_emojis.get(connector_name, "🔎")
@ -74,6 +75,7 @@ def get_connector_friendly_name(connector_name: str) -> str:
"LINKUP_API": "Linkup Search",
"AIRTABLE_CONNECTOR": "Airtable",
"LUMA_CONNECTOR": "Luma",
"ELASTICSEARCH_CONNECTOR": "Elasticsearch",
}
return connector_friendly_names.get(connector_name, connector_name)

View file

@ -0,0 +1,254 @@
"""
Elasticsearch connector for SurfSense
"""
import logging
from typing import Any
from elasticsearch import AsyncElasticsearch
from elasticsearch.exceptions import (
AuthenticationException,
ConnectionError,
NotFoundError,
)
logger = logging.getLogger(__name__)
class ElasticsearchConnector:
"""
Connector for Elasticsearch instances
"""
def __init__(
self,
url: str,
api_key: str | None = None,
username: str | None = None,
password: str | None = None,
verify_certs: bool = True,
ca_certs: str | None = None,
):
"""
Initialize Elasticsearch connector
Args:
url: Full Elasticsearch URL (e.g., https://host:port or cloud endpoint)
api_key: API key for authentication (preferred method)
username: Username for basic authentication
password: Password for basic authentication
verify_certs: Whether to verify SSL certificates
ca_certs: Path to CA certificates file
"""
self.url = url
self.api_key = api_key
self.username = username
self.password = password
self.verify_certs = verify_certs
self.ca_certs = ca_certs
# Build connection configuration
self.es_config = self._build_config()
# Initialize Elasticsearch client
try:
self.client = AsyncElasticsearch(**self.es_config)
except Exception as e:
logger.error(f"Failed to initialize Elasticsearch client: {e}")
raise
def _build_config(self) -> dict[str, Any]:
"""Build Elasticsearch client configuration"""
config = {
"hosts": [self.url],
"verify_certs": self.verify_certs,
"request_timeout": 30,
"max_retries": 3,
"retry_on_timeout": True,
}
# Authentication - API key takes precedence
if self.api_key:
config["api_key"] = self.api_key
elif self.username and self.password:
config["basic_auth"] = (self.username, self.password)
# SSL configuration
if self.ca_certs:
config["ca_certs"] = self.ca_certs
return config
async def search(
self,
index: str | list[str],
query: dict[str, Any],
size: int = 100,
from_: int = 0,
fields: list[str] | None = None,
sort: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""
Search documents in Elasticsearch
Args:
index: Elasticsearch index name or list of indices
query: Elasticsearch query DSL
size: Number of results to return
from_: Starting offset for pagination
fields: List of fields to include in response
sort: Sort configuration
Returns:
Elasticsearch search response
"""
try:
search_body: dict[str, Any] = {
"query": query,
"size": size,
"from": from_,
}
if fields:
search_body["_source"] = fields
if sort:
search_body["sort"] = sort
response = await self.client.search(index=index, body=search_body)
logger.info(
f"Successfully searched index '{index}', found {response['hits']['total']['value']} results"
)
return response
except NotFoundError:
logger.error(f"Index '{index}' not found")
raise
except AuthenticationException:
logger.error("Authentication failed")
raise
except ConnectionError:
logger.error("Failed to connect to Elasticsearch")
raise
except Exception as e:
logger.error(f"Search failed: {e}")
raise
async def get_indices(self) -> list[str]:
"""
Get list of available indices
Returns:
List of index names
"""
try:
indices = await self.client.indices.get_alias(index="*")
return list(indices.keys())
except Exception as e:
logger.error(f"Failed to get indices: {e}")
raise
async def get_mapping(self, index: str) -> dict[str, Any]:
"""
Get mapping for an index
Args:
index: Index name
Returns:
Index mapping
"""
try:
mapping = await self.client.indices.get_mapping(index=index)
return mapping[index]["mappings"] if index in mapping else {}
except Exception as e:
logger.error(f"Failed to get mapping for index '{index}': {e}")
raise
async def scroll_search(
self,
index: str | list[str],
query: dict[str, Any],
size: int = 1000,
scroll_timeout: str = "5m",
fields: list[str] | None = None,
):
"""
Perform a scroll search for large result sets
Args:
index: Elasticsearch index name or list of indices
query: Elasticsearch query DSL
size: Number of results per scroll
scroll_timeout: Scroll timeout
fields: List of fields to include in response
Yields:
Document hits from Elasticsearch
"""
try:
search_body: dict[str, Any] = {
"query": query,
"size": size,
}
if fields:
search_body["_source"] = fields
# Initial search
response = await self.client.search(
index=index, body=search_body, scroll=scroll_timeout
)
scroll_id = response.get("_scroll_id")
hits = response["hits"]["hits"]
while hits:
for hit in hits:
yield hit
# Continue scrolling
if scroll_id:
response = await self.client.scroll(
scroll_id=scroll_id, scroll=scroll_timeout
)
scroll_id = response.get("_scroll_id")
hits = response["hits"]["hits"]
# Clear scroll
if scroll_id:
await self.client.clear_scroll(scroll_id=scroll_id)
except Exception as e:
logger.error(f"Scroll search failed: {e}")
raise
async def count_documents(
self, index: str | list[str], query: dict[str, Any] | None = None
) -> int:
"""
Count documents in an index
Args:
index: Index name or list of indices
query: Optional query to filter documents
Returns:
Number of documents
"""
try:
if query:
response = await self.client.count(index=index, body={"query": query})
else:
response = await self.client.count(index=index)
return response["count"]
except Exception as e:
logger.error(f"Failed to count documents in index '{index}': {e}")
raise
async def close(self):
"""Close the Elasticsearch client connection"""
if hasattr(self, "client"):
await self.client.close()

View file

@ -50,6 +50,7 @@ class DocumentType(str, Enum):
GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR"
AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR"
LUMA_CONNECTOR = "LUMA_CONNECTOR"
ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR"
class SearchSourceConnectorType(str, Enum):
@ -68,6 +69,7 @@ class SearchSourceConnectorType(str, Enum):
GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR"
AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR"
LUMA_CONNECTOR = "LUMA_CONNECTOR"
ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR"
class ChatType(str, Enum):

View file

@ -40,6 +40,7 @@ from app.tasks.connector_indexers import (
index_clickup_tasks,
index_confluence_pages,
index_discord_messages,
index_elasticsearch_documents,
index_github_repos,
index_google_calendar_events,
index_google_gmail_messages,
@ -363,6 +364,7 @@ async def index_connector_content(
- JIRA_CONNECTOR: Indexes issues and comments from Jira
- DISCORD_CONNECTOR: Indexes messages from all accessible Discord channels
- LUMA_CONNECTOR: Indexes events from Luma
- ELASTICSEARCH_CONNECTOR: Indexes documents from Elasticsearch
Args:
connector_id: ID of the connector to use
@ -589,6 +591,24 @@ async def index_connector_content(
)
response_message = "Luma indexing started in the background."
elif (
connector.connector_type
== SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR
):
# Run indexing in background
logger.info(
f"Triggering Elasticsearch indexing for connector {connector_id} into search space {search_space_id}"
)
background_tasks.add_task(
run_elasticsearch_indexing_with_new_session,
connector_id,
search_space_id,
str(user.id),
indexing_from,
indexing_to,
)
response_message = "Elasticsearch indexing started in the background."
else:
raise HTTPException(
status_code=400,
@ -1358,3 +1378,61 @@ async def run_luma_indexing(
)
except Exception as e:
logger.error(f"Error in background Luma indexing task: {e!s}")
async def run_elasticsearch_indexing_with_new_session(
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str,
):
"""Wrapper to run Elasticsearch indexing with its own database session."""
logger.info(
f"Background task started: Indexing Elasticsearch connector {connector_id} into space {search_space_id}"
)
async with async_session_maker() as session:
await run_elasticsearch_indexing(
session, connector_id, search_space_id, user_id, start_date, end_date
)
logger.info(
f"Background task finished: Indexing Elasticsearch connector {connector_id}"
)
async def run_elasticsearch_indexing(
session: AsyncSession,
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str,
):
"""Runs the Elasticsearch indexing task and updates the timestamp."""
try:
indexed_count, error_message = await index_elasticsearch_documents(
session,
connector_id,
search_space_id,
user_id,
start_date,
end_date,
update_last_indexed=False,
)
if error_message:
logger.error(
f"Elasticsearch indexing failed for connector {connector_id}: {error_message}"
)
else:
logger.info(
f"Elasticsearch indexing successful for connector {connector_id}. Indexed {indexed_count} documents."
)
# Update the last indexed timestamp only on success
await update_connector_last_indexed(session, connector_id)
await session.commit()
except Exception as e:
await session.rollback()
logger.error(
f"Critical error in run_elasticsearch_indexing for connector {connector_id}: {e}",
exc_info=True,
)

View file

@ -2028,3 +2028,117 @@ class ConnectorService:
}
return result_object, luma_chunks
async def search_elasticsearch(
self,
user_query: str,
user_id: str,
search_space_id: int,
top_k: int = 20,
search_mode: SearchMode = SearchMode.CHUNKS,
) -> tuple:
"""
Search for Elasticsearch documents and return both the source information and langchain documents
Args:
user_query: The user's query
user_id: The user's ID
search_space_id: The search space ID to search in
top_k: Maximum number of results to return
search_mode: Search mode (CHUNKS or DOCUMENTS)
Returns:
tuple: (sources_info, langchain_documents)
"""
if search_mode == SearchMode.CHUNKS:
elasticsearch_chunks = await self.chunk_retriever.hybrid_search(
query_text=user_query,
top_k=top_k,
user_id=user_id,
search_space_id=search_space_id,
document_type="ELASTICSEARCH_CONNECTOR",
)
elif search_mode == SearchMode.DOCUMENTS:
elasticsearch_chunks = await self.document_retriever.hybrid_search(
query_text=user_query,
top_k=top_k,
user_id=user_id,
search_space_id=search_space_id,
document_type="ELASTICSEARCH_CONNECTOR",
)
# Transform document retriever results to match expected format
elasticsearch_chunks = self._transform_document_results(
elasticsearch_chunks
)
# Early return if no results
if not elasticsearch_chunks:
return {
"id": 34,
"name": "Elasticsearch",
"type": "ELASTICSEARCH_CONNECTOR",
"sources": [],
}, []
# Process each chunk and create sources directly without deduplication
sources_list = []
async with self.counter_lock:
for _i, chunk in enumerate(elasticsearch_chunks):
# Extract document metadata
document = chunk.get("document", {})
metadata = document.get("metadata", {})
# Extract Elasticsearch-specific metadata
es_id = metadata.get("elasticsearch_id", "")
es_index = metadata.get("elasticsearch_index", "")
es_score = metadata.get("elasticsearch_score", "")
# Create a more descriptive title for Elasticsearch documents
title = document.get("title", "Elasticsearch Document")
if es_index:
title = f"{title} (Index: {es_index})"
# Create a more descriptive description for Elasticsearch documents
description = chunk.get("content", "")[:150]
if len(description) == 150:
description += "..."
# Add Elasticsearch info to description
info_parts = []
if es_id:
info_parts.append(f"ID: {es_id}")
if es_score:
info_parts.append(f"Score: {es_score}")
if info_parts:
if description:
description = f"{description} | {' | '.join(info_parts)}"
else:
description = " | ".join(info_parts)
# For URL, we could construct a URL to view the document if we have the Elasticsearch UI URL
url = ""
# Could be extended to include Kibana or other UI URLs if configured
source = {
"id": chunk.get("chunk_id", self.source_id_counter),
"title": title,
"description": description,
"url": url,
"elasticsearch_id": es_id,
"elasticsearch_index": es_index,
"elasticsearch_score": es_score,
}
self.source_id_counter += 1
sources_list.append(source)
# Create result object
result_object = {
"id": 34, # Assign a unique ID for the Elasticsearch connector
"name": "Elasticsearch",
"type": "ELASTICSEARCH_CONNECTOR",
"sources": sources_list,
}
return result_object, elasticsearch_chunks

View file

@ -17,6 +17,7 @@ Available indexers:
- Google Gmail: Index messages from Google Gmail
- Google Calendar: Index events from Google Calendar
- Luma: Index events from Luma
- Elasticsearch: Index documents from Elasticsearch instances
"""
# Communication platforms
@ -27,6 +28,7 @@ from .confluence_indexer import index_confluence_pages
from .discord_indexer import index_discord_messages
# Development platforms
from .elasticsearch_indexer import index_elasticsearch_documents
from .github_indexer import index_github_repos
from .google_calendar_indexer import index_google_calendar_events
from .google_gmail_indexer import index_google_gmail_messages
@ -46,6 +48,7 @@ __all__ = [ # noqa: RUF022
"index_confluence_pages",
"index_discord_messages",
# Development platforms
"index_elasticsearch_documents",
"index_github_repos",
# Calendar and scheduling
"index_google_calendar_events",

View file

@ -0,0 +1,354 @@
"""
Elasticsearch indexer for SurfSense
"""
import hashlib
import json
import logging
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.connectors.elasticsearch_connector import ElasticsearchConnector
from app.db import Document, DocumentType, SearchSourceConnector
logger = logging.getLogger(__name__)
class _ChunkingService:
def __init__(self, chunk_size: int = 1000, overlap: int = 200) -> None:
self.chunk_size = max(100, chunk_size)
self.overlap = max(0, min(overlap, self.chunk_size - 1))
def chunk_text(self, text: str) -> list[str]:
if not text:
return []
text = text.strip()
if len(text) <= self.chunk_size:
return [text]
chunks: list[str] = []
step = self.chunk_size - self.overlap
pos = 0
while pos < len(text):
end = pos + self.chunk_size
chunks.append(text[pos:end].strip())
pos += step
return chunks
class _DocumentService:
def __init__(self, session):
self.session = session
async def get_document_by_hash(self, content_hash: str):
from sqlalchemy.future import select
from app.db import Document
if not content_hash:
return None
result = await self.session.execute(
select(Document).where(Document.content_hash == content_hash)
)
return result.scalars().first()
async def create_chunks_for_document(self, document_id: int, chunks: list[str]):
from app.db import Chunk
for chunk_text in chunks:
self.session.add(Chunk(content=chunk_text, document_id=document_id))
await self.session.flush()
async def index_elasticsearch_documents(
session: AsyncSession,
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str,
update_last_indexed: bool = True,
) -> tuple[int, str | None]:
"""
Index documents from Elasticsearch into SurfSense
Args:
session: Database session
connector_id: Elasticsearch connector ID
search_space_id: Search space ID
user_id: User ID
start_date: Start date for indexing (not used for Elasticsearch, kept for compatibility)
end_date: End date for indexing (not used for Elasticsearch, kept for compatibility)
update_last_indexed: Whether to update the last indexed timestamp
Returns:
Tuple of (number of documents processed, error message if any)
"""
es_connector = None
try:
# Get the connector configuration
result = await session.execute(
select(SearchSourceConnector).filter(
SearchSourceConnector.id == connector_id
)
)
connector = result.scalars().first()
if not connector:
error_msg = f"Elasticsearch connector with ID {connector_id} not found"
logger.error(error_msg)
return 0, error_msg
# Get connector configuration
config = connector.config
# Validate required fields - now only URL and INDEX are required
# Authentication can be either API key OR username/password
if "ELASTICSEARCH_URL" not in config:
error_msg = "Missing required field in connector config: ELASTICSEARCH_URL"
logger.error(error_msg)
return 0, error_msg
if "ELASTICSEARCH_INDEX" not in config:
error_msg = (
"Missing required field in connector config: ELASTICSEARCH_INDEX"
)
logger.error(error_msg)
return 0, error_msg
# Check authentication - must have either API key or username+password
has_api_key = (
"ELASTICSEARCH_API_KEY" in config and config["ELASTICSEARCH_API_KEY"]
)
has_basic_auth = (
"ELASTICSEARCH_USERNAME" in config
and config["ELASTICSEARCH_USERNAME"]
and "ELASTICSEARCH_PASSWORD" in config
and config["ELASTICSEARCH_PASSWORD"]
)
if not has_api_key and not has_basic_auth:
error_msg = "Missing authentication: provide either ELASTICSEARCH_API_KEY or ELASTICSEARCH_USERNAME + ELASTICSEARCH_PASSWORD"
logger.error(error_msg)
return 0, error_msg
# Initialize document service
document_service = _DocumentService(session)
chunking_service = _ChunkingService()
# Initialize Elasticsearch connector
es_connector = ElasticsearchConnector(
url=config["ELASTICSEARCH_URL"],
api_key=config.get("ELASTICSEARCH_API_KEY"),
username=config.get("ELASTICSEARCH_USERNAME"),
password=config.get("ELASTICSEARCH_PASSWORD"),
verify_certs=config.get("ELASTICSEARCH_VERIFY_CERTS", True),
ca_certs=config.get("ELASTICSEARCH_CA_CERTS"),
)
# Build query based on configuration
query = _build_elasticsearch_query(config)
# Get the index name(s) - can be a string or list
index_name = config["ELASTICSEARCH_INDEX"]
# Get max documents to index
max_documents = config.get("ELASTICSEARCH_MAX_DOCUMENTS", 1000)
logger.info(
f"Starting Elasticsearch indexing for index '{index_name}' with max {max_documents} documents"
)
documents_processed = 0
try:
# Use scroll search for large result sets
async for hit in es_connector.scroll_search(
index=index_name,
query=query,
size=min(max_documents, 100), # Scroll in batches
fields=config.get("ELASTICSEARCH_FIELDS"),
):
if documents_processed >= max_documents:
break
try:
# Extract document data
doc_id = hit["_id"]
source = hit.get("_source", {})
# Build document title
title_field = config.get("ELASTICSEARCH_TITLE_FIELD")
if not title_field:
for candidate in ("title", "name", "subject"):
if candidate in source:
title_field = candidate
break
title = (
str(source.get(title_field, doc_id))
if title_field is not None
else str(doc_id)
)
# Build document content
content = _build_document_content(source, config)
if not content.strip():
logger.warning(f"Skipping document {doc_id} - no content found")
continue
# Create content hash
content_hash = hashlib.sha256(content.encode()).hexdigest()
# Build metadata
metadata = {
"elasticsearch_id": doc_id,
"elasticsearch_index": hit.get("_index", index_name),
"elasticsearch_score": hit.get("_score"),
"indexed_at": datetime.now().isoformat(),
"source": "ELASTICSEARCH_CONNECTOR",
}
# Add any additional metadata fields specified in config
if "ELASTICSEARCH_METADATA_FIELDS" in config:
for field in config["ELASTICSEARCH_METADATA_FIELDS"]:
if field in source:
metadata[f"es_{field}"] = source[field]
# Check if document already exists
existing_doc = await document_service.get_document_by_hash(
content_hash
)
if existing_doc:
logger.debug(f"Document {doc_id} already exists, skipping")
continue
# Create document
document = Document(
title=title,
content=content,
content_hash=content_hash,
document_type=DocumentType.ELASTICSEARCH_CONNECTOR,
document_metadata=metadata,
search_space_id=search_space_id,
)
# Add document to session
session.add(document)
await session.flush() # Get the document ID
# Create chunks
chunks = chunking_service.chunk_text(content)
await document_service.create_chunks_for_document(
document.id, chunks
)
documents_processed += 1
if documents_processed % 10 == 0:
logger.info(
f"Processed {documents_processed} Elasticsearch documents"
)
await session.commit()
except Exception as e:
logger.error(
f"Error processing Elasticsearch document {hit.get('_id', 'unknown')}: {e}"
)
continue
# Final commit
await session.commit()
logger.info(
f"Successfully indexed {documents_processed} documents from Elasticsearch"
)
# Update last indexed timestamp if requested
if update_last_indexed and documents_processed > 0:
connector.last_indexed_at = datetime.now()
await session.commit()
if update_last_indexed and documents_processed > 0:
# store ISO-8601 UTC timestamp with 'Z' suffix, e.g. 2025-10-09T22:04:53.599658Z
connector.last_indexed_at = (
datetime.now(UTC).isoformat().replace("+00:00", "Z")
)
await session.commit()
return documents_processed, None
finally:
# Clean up Elasticsearch connection
if es_connector:
await es_connector.close()
except Exception as e:
error_msg = f"Error indexing Elasticsearch documents: {e}"
logger.error(error_msg, exc_info=True)
await session.rollback()
if es_connector:
await es_connector.close()
return 0, error_msg
def _build_elasticsearch_query(config: dict[str, Any]) -> dict[str, Any]:
"""
Build Elasticsearch query from connector configuration
Args:
config: Connector configuration
Returns:
Elasticsearch query DSL
"""
# Check if custom query is provided
if config.get("ELASTICSEARCH_QUERY"):
try:
if isinstance(config["ELASTICSEARCH_QUERY"], str):
return json.loads(config["ELASTICSEARCH_QUERY"])
else:
return config["ELASTICSEARCH_QUERY"]
except (json.JSONDecodeError, TypeError) as e:
logger.warning(f"Invalid custom query, using match_all: {e}")
# Default to match all documents
return {"match_all": {}}
def _build_document_content(source: dict[str, Any], config: dict[str, Any]) -> str:
"""
Build document content from Elasticsearch document source
Args:
source: Elasticsearch document source
config: Connector configuration
Returns:
Formatted document content
"""
content_parts = []
# Get content fields from config
content_fields = config.get("ELASTICSEARCH_CONTENT_FIELDS", [])
if content_fields:
# Use specified content fields
for field in content_fields:
if field in source:
field_value = source[field]
if isinstance(field_value, str | int | float):
content_parts.append(f"{field}: {field_value}")
if isinstance(field_value, str | int | float):
content_parts.append(f"{field}: {json.dumps(field_value)}")
else:
# Use all fields if no specific content fields specified
for key, value in source.items():
if isinstance(value, str | int | float):
content_parts.append(f"{key}: {value}")
elif isinstance(value, list | dict):
content_parts.append(f"{key}: {json.dumps(value)}")
return "\n".join(content_parts)

View file

@ -43,6 +43,7 @@ dependencies = [
"youtube-transcript-api>=1.0.3",
"litellm>=1.77.5",
"langchain-litellm>=0.2.3",
"elasticsearch>=9.1.1",
]
[dependency-groups]

5003
surfsense_backend/uv.lock generated

File diff suppressed because it is too large Load diff

View file

@ -271,6 +271,17 @@ export default function EditConnectorPage() {
placeholder="API Key..."
/>
)}
{/* == Elasticsearch == */}
{connector.connector_type === "ELASTICSEARCH_CONNECTOR" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="ELASTICSEARCH_API_KEY"
fieldLabel="Elasticsearch API Key"
fieldDescription="Update your Elasticsearch API Key if needed."
placeholder="Your Elasticsearch API Key"
/>
)}
</CardContent>
<CardFooter className="border-t pt-6">
<Button type="submit" disabled={isSaving} className="w-full sm:w-auto">

View file

@ -54,6 +54,7 @@ const getConnectorTypeDisplay = (type: string): string => {
GOOGLE_GMAIL_CONNECTOR: "Google Gmail Connector",
AIRTABLE_CONNECTOR: "Airtable Connector",
LUMA_CONNECTOR: "Luma Connector",
ELASTICSEARCH_CONNECTOR: "Elasticsearch Connector",
// Add other connector types here as needed
};
return typeMap[type] || type;
@ -233,7 +234,9 @@ export default function EditConnectorPage() {
? "GitHub Personal Access Token (PAT)"
: connector?.connector_type === "LINKUP_API"
? "Linkup API Key"
: "API Key"}
: connector?.connector_type === "ELASTICSEARCH_CONNECTOR"
? "Elasticsearch API Key"
: "API Key"}
</FormLabel>
<FormControl>
<Input
@ -247,7 +250,9 @@ export default function EditConnectorPage() {
? "Enter new GitHub PAT (optional)"
: connector?.connector_type === "LINKUP_API"
? "Enter new Linkup API Key (optional)"
: "Enter new API key (optional)"
: connector?.connector_type === "ELASTICSEARCH_CONNECTOR"
? "Enter new Elasticsearch API Key (optional)"
: "Enter new API key (optional)"
}
{...field}
/>
@ -261,7 +266,9 @@ export default function EditConnectorPage() {
? "Enter a new GitHub PAT or leave blank to keep your existing token."
: connector?.connector_type === "LINKUP_API"
? "Enter a new Linkup API Key or leave blank to keep your existing key."
: "Enter a new API key or leave blank to keep your existing key."}
: connector?.connector_type === "ELASTICSEARCH_CONNECTOR"
? "Enter a new Elasticsearch API Key or leave blank to keep your existing key."
: "Enter a new API key or leave blank to keep your existing key."}
</FormDescription>
<FormMessage />
</FormItem>

View file

@ -0,0 +1,751 @@
"use client";
import { zodResolver } from "@hookform/resolvers/zod";
import * as RadioGroup from "@radix-ui/react-radio-group";
import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
import { motion } from "motion/react";
import { useParams, useRouter, useSearchParams } from "next/navigation";
import { useId, useState } from "react";
import { useForm } from "react-hook-form";
import { toast } from "sonner";
import * as z from "zod";
import {
Accordion,
AccordionContent,
AccordionItem,
AccordionTrigger,
} from "@/components/ui/accordion";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
Card,
CardContent,
CardDescription,
CardFooter,
CardHeader,
CardTitle,
} from "@/components/ui/card";
import {
Form,
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Separator } from "@/components/ui/separator";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { EnumConnectorName } from "@/contracts/enums/connector";
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
// Define the form schema with Zod
const elasticsearchConnectorFormSchema = z
.object({
name: z.string().min(3, {
message: "Connector name must be at least 3 characters.",
}),
endpoint_url: z.string().url({ message: "Please enter a valid Elasticsearch endpoint URL." }),
auth_method: z.enum(["basic", "api_key"]).default("api_key"),
username: z.string().optional(),
password: z.string().optional(),
ELASTICSEARCH_API_KEY: z.string().optional(),
indices: z.string().optional(),
query: z.string().default("*"),
search_fields: z.string().optional(),
max_documents: z.number().min(1).max(10000).optional(),
})
.refine(
(data) => {
if (data.auth_method === "basic") {
return Boolean(data.username?.trim() && data.password?.trim());
}
if (data.auth_method === "api_key") {
return Boolean(data.ELASTICSEARCH_API_KEY?.trim());
}
return true;
},
{
message: "Authentication credentials are required for the selected method.",
path: ["auth_method"],
}
);
// Define the type for the form values
type ElasticsearchConnectorFormValues = z.infer<typeof elasticsearchConnectorFormSchema>;
export default function ElasticsearchConnectorPage() {
const router = useRouter();
const params = useParams();
const searchParams = useSearchParams();
// match pattern used in other connector pages: prefer route param, fallback to query param
const searchSpaceId = (params.search_space_id ?? searchParams?.get("search_space_id")) as string;
const [isSubmitting, setIsSubmitting] = useState(false);
const authBasicId = useId();
const authApiKeyId = useId();
const { createConnector } = useSearchSourceConnectors();
// Initialize the form
const form = useForm<ElasticsearchConnectorFormValues>({
resolver: zodResolver(elasticsearchConnectorFormSchema),
defaultValues: {
name: "Elasticsearch Connector",
endpoint_url: "",
auth_method: "api_key",
username: "",
password: "",
ELASTICSEARCH_API_KEY: "",
indices: "",
query: "*",
search_fields: "",
max_documents: undefined,
},
});
const stringToArray = (str: string): string[] => {
const items = str
.split(",")
.map((item) => item.trim())
.filter((item) => item.length > 0);
return Array.from(new Set(items));
};
// Handle form submission
const onSubmit = async (values: ElasticsearchConnectorFormValues) => {
setIsSubmitting(true);
if (!searchSpaceId) {
toast.error(
"Missing search_space_id (route or ?search_space_id=). Provide it in the URL or pick a search space."
);
setIsSubmitting(false);
return;
}
const searchSpaceIdNum = Number(searchSpaceId);
if (!Number.isInteger(searchSpaceIdNum) || searchSpaceIdNum <= 0) {
toast.error("Invalid search_space_id. It must be a positive integer.");
setIsSubmitting(false);
return;
}
try {
// Send full URL to backend (backend expects ELASTICSEARCH_URL)
const config: Record<string, string | number | boolean | string[]> = {
ELASTICSEARCH_URL: values.endpoint_url,
// default to verifying certs; expose fields for CA/verify if UI added later
ELASTICSEARCH_VERIFY_CERTS: true,
};
if (values.auth_method === "basic") {
if (values.username) config.ELASTICSEARCH_USERNAME = values.username;
if (values.password) config.ELASTICSEARCH_PASSWORD = values.password;
} else if (values.auth_method === "api_key") {
if (values.ELASTICSEARCH_API_KEY)
config.ELASTICSEARCH_API_KEY = values.ELASTICSEARCH_API_KEY;
}
if (values.indices?.trim()) {
const indicesArr = stringToArray(values.indices);
config.ELASTICSEARCH_INDEX = indicesArr.length === 1 ? indicesArr[0] : indicesArr;
}
if (values.query && values.query !== "*") {
config.ELASTICSEARCH_QUERY = values.query;
}
if (values.search_fields?.trim()) {
config.ELASTICSEARCH_FIELDS = stringToArray(values.search_fields);
const fields = stringToArray(values.search_fields);
config.ELASTICSEARCH_FIELDS = fields;
config.ELASTICSEARCH_CONTENT_FIELDS = fields;
if (fields.includes("title")) {
config.ELASTICSEARCH_TITLE_FIELD = "title";
}
}
if (values.max_documents !== undefined && values.max_documents > 0) {
config.ELASTICSEARCH_MAX_DOCUMENTS = values.max_documents;
}
const connectorPayload = {
name: values.name,
connector_type: EnumConnectorName.ELASTICSEARCH_CONNECTOR,
is_indexable: true,
search_space_id: searchSpaceIdNum,
config,
};
// Use existing hook method
await createConnector(connectorPayload, searchSpaceIdNum);
toast.success("Elasticsearch connector created successfully!");
router.push(`/dashboard/${searchSpaceId}/connectors`);
} catch (error) {
console.error("Error creating connector:", error);
toast.error(error instanceof Error ? error.message : "Failed to create connector");
} finally {
setIsSubmitting(false);
}
};
return (
<div className="container mx-auto py-8 max-w-3xl">
<Button
variant="ghost"
className="mb-6"
onClick={() => router.push(`/dashboard/${searchSpaceId}/connectors/add`)}
>
<ArrowLeft className="mr-2 h-4 w-4" />
Back to Connectors
</Button>
{/* Header */}
<div className="mb-8">
<div className="flex items-center gap-4">
<div className="flex h-12 w-12 items-center justify-center rounded-lg">
{getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6")}
</div>
<div>
<h1 className="text-3xl font-bold tracking-tight">Connect Elasticsearch</h1>
<p className="text-muted-foreground">
Connect to your Elasticsearch cluster to search and index documents.
</p>
</div>
</div>
</div>
<motion.div
initial={{ opacity: 0, y: 20 }}
animate={{ opacity: 1, y: 0 }}
transition={{ duration: 0.5 }}
>
<Tabs defaultValue="connect" className="w-full">
<TabsList className="grid w-full grid-cols-2 mb-6">
<TabsTrigger value="connect">Connect</TabsTrigger>
<TabsTrigger value="documentation">Documentation</TabsTrigger>
</TabsList>
<TabsContent value="connect">
<Card className="border-2 border-border">
<CardHeader>
<CardTitle className="text-2xl font-bold">Connect Elasticsearch Cluster</CardTitle>
<CardDescription>
Connect to your Elasticsearch instance to search and index documents for enhanced
search capabilities.
</CardDescription>
</CardHeader>
<CardContent>
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-6">
{/* Connector Name */}
<FormField
control={form.control}
name="name"
render={({ field }) => (
<FormItem>
<FormLabel>Connector Name</FormLabel>
<FormControl>
<Input placeholder="My Elasticsearch Connector" {...field} />
</FormControl>
<FormDescription>
A friendly name to identify this connector.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
{/* Connection Details */}
<div className="space-y-4">
<h3 className="text-lg font-medium">Connection Details</h3>
<FormField
control={form.control}
name="endpoint_url"
render={({ field }) => (
<FormItem>
<FormLabel>Elasticsearch Endpoint URL</FormLabel>
<FormControl>
<Input
type="url"
autoComplete="off"
placeholder="https://your-cluster.es.region.aws.com:443"
{...field}
/>
</FormControl>
<FormDescription>
Enter the complete Elasticsearch endpoint URL. We'll automatically
extract the hostname, port, and SSL settings.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
{/* Show parsed URL details */}
{form.watch("endpoint_url") && (
<div className="rounded-lg border bg-muted/50 p-3">
<h4 className="text-sm font-medium mb-2">Parsed Connection Details:</h4>
<div className="text-sm text-muted-foreground space-y-1">
{(() => {
try {
const url = new URL(form.watch("endpoint_url"));
return (
<>
<div>
<strong>Hostname:</strong> {url.hostname}
</div>
<div>
<strong>Port:</strong>{" "}
{url.port || (url.protocol === "https:" ? "443" : "80")}
</div>
<div>
<strong>SSL/TLS:</strong>{" "}
{url.protocol === "https:" ? "Enabled" : "Disabled"}
</div>
</>
);
} catch {
return <div className="text-destructive">Invalid URL format</div>;
}
})()}
</div>
</div>
)}
</div>
{/* Authentication */}
<div className="space-y-4">
<h3 className="text-lg font-medium">Authentication</h3>
<FormField
control={form.control}
name="auth_method"
render={({ field }) => (
<FormItem className="space-y-3">
<FormControl>
<RadioGroup.Root
onValueChange={(value) => {
field.onChange(value);
// Clear auth fields when method changes
if (value !== "basic") {
form.setValue("username", "");
form.setValue("password", "");
}
if (value !== "api_key") {
form.setValue("ELASTICSEARCH_API_KEY", "");
}
}}
value={field.value}
className="flex flex-col space-y-2"
>
<div className="flex items-center space-x-2">
<RadioGroup.Item
value="api_key"
id={authApiKeyId}
className="aspect-square h-4 w-4 rounded-full border border-primary text-primary ring-offset-background focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground"
>
<RadioGroup.Indicator className="flex items-center justify-center">
<div className="h-2.5 w-2.5 rounded-full bg-current" />
</RadioGroup.Indicator>
</RadioGroup.Item>
<Label htmlFor={authApiKeyId}>API Key</Label>
</div>
<div className="flex items-center space-x-2">
<RadioGroup.Item
value="basic"
id={authBasicId}
className="aspect-square h-4 w-4 rounded-full border border-primary text-primary ring-offset-background focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground"
>
<RadioGroup.Indicator className="flex items-center justify-center">
<div className="h-2.5 w-2.5 rounded-full bg-current" />
</RadioGroup.Indicator>
</RadioGroup.Item>
<Label htmlFor={authBasicId}>Username & Password</Label>
</div>
</RadioGroup.Root>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
{/* Basic Auth Fields */}
{form.watch("auth_method") === "basic" && (
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
<FormField
control={form.control}
name="username"
render={({ field }) => (
<FormItem>
<FormLabel>Username</FormLabel>
<FormControl>
<Input placeholder="elastic" autoComplete="username" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="password"
render={({ field }) => (
<FormItem>
<FormLabel>Password</FormLabel>
<FormControl>
<Input
type="password"
placeholder="Password"
autoComplete="current-password"
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
</div>
)}
{/* API Key Field */}
{form.watch("auth_method") === "api_key" && (
<FormField
control={form.control}
name="ELASTICSEARCH_API_KEY"
render={({ field }) => (
<FormItem>
<FormLabel>API Key</FormLabel>
<FormControl>
<Input
type="password"
placeholder="Your API Key Here"
autoComplete="off"
{...field}
/>
</FormControl>
<FormDescription>
Enter your Elasticsearch API key (base64 encoded). This will be
stored securely.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
</div>
{/* Advanced Configuration */}
<Accordion type="single" collapsible className="w-full">
<AccordionItem value="advanced">
<AccordionTrigger>Advanced Configuration</AccordionTrigger>
<AccordionContent className="space-y-4">
{/* Index Selection */}
<FormField
control={form.control}
name="indices"
render={({ field }) => (
<FormItem>
<FormLabel>
Index Selection{" "}
<span className="text-muted-foreground">(Optional)</span>
</FormLabel>
<FormControl>
<Input placeholder="logs-*, documents-*, app-logs" {...field} />
</FormControl>
<FormDescription>
Comma-separated indices to search (e.g., "logs-*, documents-*").
Leave empty for all indices. Supports wildcards.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
{/* Show parsed indices as badges */}
{form.watch("indices")?.trim() && (
<div className="rounded-lg border bg-muted/50 p-3">
<h4 className="text-sm font-medium mb-2">Selected Indices:</h4>
<div className="flex flex-wrap gap-2">
{stringToArray(form.watch("indices")).map((index) => (
<Badge key={index} variant="secondary" className="text-xs">
{index}
</Badge>
))}
</div>
</div>
)}
<FormField
control={form.control}
name="query"
render={({ field }) => (
<FormItem>
<FormLabel>
Default Search Query{" "}
<span className="text-muted-foreground">(Optional)</span>
</FormLabel>
<FormControl>
<Input placeholder="*" {...field} />
</FormControl>
<FormDescription>
Default Elasticsearch query to use for searches. Use "*" to match
all documents.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="search_fields"
render={({ field }) => (
<FormItem>
<FormLabel>
Search Fields{" "}
<span className="text-muted-foreground">(Optional)</span>
</FormLabel>
<FormControl>
<Input placeholder="title, content, description" {...field} />
</FormControl>
<FormDescription>
Comma-separated list of specific fields to search in (e.g.,
"title, content, description"). Leave empty to search all fields.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
{/* Show parsed search fields as badges */}
{form.watch("search_fields")?.trim() && (
<div className="rounded-lg border bg-muted/50 p-3">
<h4 className="text-sm font-medium mb-2">Search Fields:</h4>
<div className="flex flex-wrap gap-2">
{stringToArray(form.watch("search_fields")).map((field) => (
<Badge key={field} variant="outline" className="text-xs">
{field}
</Badge>
))}
</div>
</div>
)}
<FormField
control={form.control}
name="max_documents"
render={({ field }) => (
<FormItem>
<FormLabel>
Maximum Documents{" "}
<span className="text-muted-foreground">(Optional)</span>
</FormLabel>
<FormControl>
<Input
type="number"
placeholder="1000"
min="1"
max="10000"
{...field}
onChange={(e) =>
field.onChange(
e.target.value === ""
? undefined
: parseInt(e.target.value, 10)
)
}
/>
</FormControl>
<FormDescription>
Maximum number of documents to retrieve per search (1-10,000).
Leave empty to use Elasticsearch's default limit.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<Alert className="bg-muted">
<Info className="h-4 w-4" />
<AlertTitle>Index Selection Tips</AlertTitle>
<AlertDescription className="mt-2">
<ul className="list-disc pl-4 space-y-1 text-sm">
<li>Use wildcards like "logs-*" to match multiple indices</li>
<li>Separate multiple indices with commas</li>
<li>Leave empty to search all accessible indices</li>
<li>Choosing specific indices improves search performance</li>
</ul>
</AlertDescription>
</Alert>
</AccordionContent>
</AccordionItem>
</Accordion>
<Separator />
<div className="flex justify-end">
<Button type="submit" disabled={isSubmitting} className="w-full sm:w-auto">
{isSubmitting ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
Connecting...
</>
) : (
<>
<Check className="mr-2 h-4 w-4" />
Connect Elasticsearch
</>
)}
</Button>
</div>
</form>
</Form>
</CardContent>
<CardFooter className="flex flex-col items-start border-t bg-muted/50 px-6 py-4">
<h4 className="text-sm font-medium">
What you get with Elasticsearch integration:
</h4>
<ul className="mt-2 list-disc pl-5 text-sm text-muted-foreground">
<li>Search across your indexed documents and logs</li>
<li>Access structured and unstructured data from your cluster</li>
<li>Leverage existing Elasticsearch indices for enhanced search</li>
<li>Real-time search capabilities with powerful query features</li>
<li>Integration with your existing Elasticsearch infrastructure</li>
</ul>
</CardFooter>
</Card>
</TabsContent>
<TabsContent value="documentation">
<Card className="border-2 border-border">
<CardHeader>
<CardTitle className="text-2xl font-bold">
Elasticsearch Connector Documentation
</CardTitle>
<CardDescription>
Learn how to set up and use the Elasticsearch connector to search your data.
</CardDescription>
</CardHeader>
<CardContent className="space-y-6">
<div>
<h3 className="text-xl font-semibold mb-2">How it works</h3>
<p className="text-muted-foreground">
The Elasticsearch connector allows you to search and retrieve documents from
your Elasticsearch cluster. Configure connection details, select specific
indices, and set search parameters to make your existing data searchable within
SurfSense.
</p>
</div>
<Accordion type="single" collapsible className="w-full">
<AccordionItem value="connection">
<AccordionTrigger className="text-lg font-medium">
Connection Setup
</AccordionTrigger>
<AccordionContent className="space-y-4">
<ol className="list-decimal pl-5 space-y-3">
<li>
<strong>Endpoint URL:</strong> Enter the complete Elasticsearch endpoint
URL (e.g., https://your-cluster.es.region.aws.com:443). We'll
automatically extract hostname, port, and SSL settings.
</li>
<li>
<strong>Authentication:</strong> Choose the appropriate method:
<ul className="list-disc pl-5 mt-1">
<li>
<strong>API Key:</strong> Base64 encoded API key (recommended for
security)
</li>
<li>
<strong>Username/Password:</strong> Basic authentication credentials
</li>
</ul>
</li>
</ol>
</AccordionContent>
</AccordionItem>
<AccordionItem value="advanced">
<AccordionTrigger className="text-lg font-medium">
Advanced Configuration
</AccordionTrigger>
<AccordionContent className="space-y-4">
<p className="text-muted-foreground">
Fine-tune your Elasticsearch connector with these optional settings:
</p>
<ul className="list-disc pl-5 space-y-2">
<li>
<strong>Index Selection:</strong> Specify which indices to search using
comma-separated patterns (e.g., "logs-*, documents-*")
</li>
<li>
<strong>Search Fields:</strong> Limit searches to specific fields (e.g.,
"title, content") for better relevance
</li>
<li>
<strong>Default Query:</strong> Set a default Elasticsearch query pattern
</li>
<li>
<strong>Max Documents:</strong> Limit the number of documents returned per
search (1-10,000)
</li>
</ul>
</AccordionContent>
</AccordionItem>
<AccordionItem value="troubleshooting">
<AccordionTrigger className="text-lg font-medium">
Troubleshooting
</AccordionTrigger>
<AccordionContent className="space-y-4">
<div className="space-y-4">
<div>
<h4 className="font-medium mb-2">Common Connection Issues:</h4>
<ul className="list-disc pl-5 space-y-2 text-muted-foreground">
<li>
<strong>Connection Refused:</strong> Check hostname and port. Ensure
Elasticsearch is running.
</li>
<li>
<strong>Authentication Failed:</strong> Verify credentials. For API
keys, ensure they have proper permissions.
</li>
<li>
<strong>SSL Errors:</strong> Try disabling SSL for local development
or check certificate validity.
</li>
<li>
<strong>No Indices Found:</strong> Ensure your credentials have
permission to list and read indices.
</li>
</ul>
</div>
<Alert className="bg-muted">
<Info className="h-4 w-4" />
<AlertTitle>Security Note</AlertTitle>
<AlertDescription>
For production environments, use API keys with minimal required
permissions: cluster monitoring and read access to specific indices.
</AlertDescription>
</Alert>
</div>
</AccordionContent>
</AccordionItem>
</Accordion>
</CardContent>
</Card>
</TabsContent>
</Tabs>
</motion.div>
</div>
);
}

View file

@ -52,6 +52,13 @@ const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.LINKUP_API, "h-6 w-6"),
status: "available",
},
{
id: "elasticsearch-connector",
title: "Elasticsearch",
description: "Connect to Elasticsearch to index and search documents, logs and metrics.",
icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"),
status: "available",
},
],
},
{

View file

@ -88,6 +88,7 @@ export function DashboardBreadcrumb() {
"serper-api": "Serper API",
"linkup-api": "LinkUp API",
"luma-connector": "Luma",
"elasticsearch-connector": "Elasticsearch",
};
const connectorLabel = connectorLabels[connectorType] || connectorType;

View file

@ -44,5 +44,6 @@ export const editConnectorSchema = z.object({
GOOGLE_CALENDAR_REFRESH_TOKEN: z.string().optional(),
GOOGLE_CALENDAR_CALENDAR_IDS: z.string().optional(),
LUMA_API_KEY: z.string().optional(),
ELASTICSEARCH_API_KEY: z.string().optional(),
});
export type EditConnectorFormValues = z.infer<typeof editConnectorSchema>;

View file

@ -13,6 +13,7 @@ const INTEGRATIONS: Integration[] = [
name: "LinkUp",
icon: "https://framerusercontent.com/images/7zeIm6t3f1HaSltkw8upEvsD80.png?scale-down-to=512",
},
{ name: "Elasticsearch", icon: "https://cdn.simpleicons.org/elastic/00A9E5" },
// Communication
{ name: "Slack", icon: "https://cdn.simpleicons.org/slack/4A154B" },

View file

@ -14,4 +14,5 @@ export enum EnumConnectorName {
GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR",
AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR",
LUMA_CONNECTOR = "LUMA_CONNECTOR",
ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR",
}

View file

@ -1,6 +1,7 @@
import {
IconBook,
IconBrandDiscord,
IconBrandElastic,
IconBrandGithub,
IconBrandNotion,
IconBrandSlack,
@ -52,6 +53,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas
return <IconChecklist {...iconProps} />;
case EnumConnectorName.LUMA_CONNECTOR:
return <IconSparkles {...iconProps} />;
case EnumConnectorName.ELASTICSEARCH_CONNECTOR:
return <IconBrandElastic {...iconProps} />;
// Additional cases for non-enum connector types
case "YOUTUBE_VIDEO":
return <IconBrandYoutube {...iconProps} />;

View file

@ -57,6 +57,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
JIRA_EMAIL: "",
JIRA_API_TOKEN: "",
LUMA_API_KEY: "",
ELASTICSEARCH_API_KEY: "",
},
});
@ -84,6 +85,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
JIRA_EMAIL: config.JIRA_EMAIL || "",
JIRA_API_TOKEN: config.JIRA_API_TOKEN || "",
LUMA_API_KEY: config.LUMA_API_KEY || "",
ELASTICSEARCH_API_KEY: config.ELASTICSEARCH_API_KEY || "",
});
if (currentConnector.connector_type === "GITHUB_CONNECTOR") {
const savedRepos = config.repo_full_names || [];
@ -319,6 +321,16 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
newConfig = { LUMA_API_KEY: formData.LUMA_API_KEY };
}
break;
case "ELASTICSEARCH_CONNECTOR":
if (formData.ELASTICSEARCH_API_KEY !== originalConfig.ELASTICSEARCH_API_KEY) {
if (!formData.ELASTICSEARCH_API_KEY) {
toast.error("Elasticsearch API Key cannot be empty.");
setIsSaving(false);
return;
}
newConfig = { ELASTICSEARCH_API_KEY: formData.ELASTICSEARCH_API_KEY };
}
break;
}
if (newConfig !== null) {
@ -383,6 +395,11 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
editForm.setValue("JIRA_API_TOKEN", newlySavedConfig.JIRA_API_TOKEN || "");
} else if (connector.connector_type === "LUMA_CONNECTOR") {
editForm.setValue("LUMA_API_KEY", newlySavedConfig.LUMA_API_KEY || "");
} else if (connector.connector_type === "ELASTICSEARCH_CONNECTOR") {
editForm.setValue(
"ELASTICSEARCH_API_KEY",
newlySavedConfig.ELASTICSEARCH_API_KEY || ""
);
}
}
if (connector.connector_type === "GITHUB_CONNECTOR") {

View file

@ -35,7 +35,8 @@ export type DocumentType =
| "CLICKUP_CONNECTOR"
| "GOOGLE_CALENDAR_CONNECTOR"
| "GOOGLE_GMAIL_CONNECTOR"
| "LUMA_CONNECTOR";
| "LUMA_CONNECTOR"
| "ELASTICSEARCH_CONNECTOR";
export function useDocumentByChunk() {
const [document, setDocument] = useState<DocumentWithChunks | null>(null);

View file

@ -29,7 +29,8 @@ export type DocumentType =
| "GOOGLE_CALENDAR_CONNECTOR"
| "GOOGLE_GMAIL_CONNECTOR"
| "AIRTABLE_CONNECTOR"
| "LUMA_CONNECTOR";
| "LUMA_CONNECTOR"
| "ELASTICSEARCH_CONNECTOR";
export interface UseDocumentsOptions {
page?: number;

View file

@ -16,6 +16,7 @@ export const getConnectorTypeDisplay = (type: string): string => {
GOOGLE_GMAIL_CONNECTOR: "Google Gmail",
AIRTABLE_CONNECTOR: "Airtable",
LUMA_CONNECTOR: "Luma",
ELASTICSEARCH_CONNECTOR: "Elasticsearch",
};
return typeMap[type] || type;
};

View file

@ -32,6 +32,7 @@
"@radix-ui/react-dropdown-menu": "^2.1.15",
"@radix-ui/react-label": "^2.1.7",
"@radix-ui/react-popover": "^1.1.14",
"@radix-ui/react-radio-group": "^1.3.8",
"@radix-ui/react-scroll-area": "^1.2.9",
"@radix-ui/react-select": "^2.2.5",
"@radix-ui/react-separator": "^1.1.7",

View file

@ -47,6 +47,9 @@ importers:
'@radix-ui/react-popover':
specifier: ^1.1.14
version: 1.1.14(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
'@radix-ui/react-radio-group':
specifier: ^1.3.8
version: 1.3.8(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
'@radix-ui/react-scroll-area':
specifier: ^1.2.9
version: 1.2.9(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
@ -118,7 +121,7 @@ importers:
version: 15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
fumadocs-mdx:
specifier: ^11.7.1
version: 11.7.1(acorn@8.14.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0)
version: 11.7.1(acorn@8.15.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0)
fumadocs-ui:
specifier: ^15.6.6
version: 15.6.6(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(tailwindcss@4.1.11)
@ -1735,6 +1738,19 @@ packages:
'@types/react-dom':
optional: true
'@radix-ui/react-presence@1.1.5':
resolution: {integrity: sha512-/jfEwNDdQVBCNvjkGit4h6pMOzq8bHkopq458dPt2lMjx+eBQUohZNG9A7DtO/O5ukSbxuaNGXMjHicgwy6rQQ==}
peerDependencies:
'@types/react': '*'
'@types/react-dom': '*'
react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
peerDependenciesMeta:
'@types/react':
optional: true
'@types/react-dom':
optional: true
'@radix-ui/react-primitive@1.0.0':
resolution: {integrity: sha512-EyXe6mnRlHZ8b6f4ilTDrXmkLShICIuOTTj0GX4w1rp+wSxf3+TD05u1UOITC8VsJ2a9nwHvdXtOXEOl0Cw/zQ==}
peerDependencies:
@ -1793,6 +1809,19 @@ packages:
'@types/react-dom':
optional: true
'@radix-ui/react-radio-group@1.3.8':
resolution: {integrity: sha512-VBKYIYImA5zsxACdisNQ3BjCBfmbGH3kQlnFVqlWU4tXwjy7cGX8ta80BcrO+WJXIn5iBylEH3K6ZTlee//lgQ==}
peerDependencies:
'@types/react': '*'
'@types/react-dom': '*'
react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
peerDependenciesMeta:
'@types/react':
optional: true
'@types/react-dom':
optional: true
'@radix-ui/react-roving-focus@1.1.10':
resolution: {integrity: sha512-dT9aOXUen9JSsxnMPv/0VqySQf5eDQ6LCk5Sw28kamz8wSOW2bJdlX2Bg5VUIIcV+6XlHpWTIuTPCf/UNIyq8Q==}
peerDependencies:
@ -1806,6 +1835,19 @@ packages:
'@types/react-dom':
optional: true
'@radix-ui/react-roving-focus@1.1.11':
resolution: {integrity: sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA==}
peerDependencies:
'@types/react': '*'
'@types/react-dom': '*'
react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
peerDependenciesMeta:
'@types/react':
optional: true
'@types/react-dom':
optional: true
'@radix-ui/react-scroll-area@1.2.9':
resolution: {integrity: sha512-YSjEfBXnhUELsO2VzjdtYYD4CfQjvao+lhhrX5XsHD7/cyUNzljF1FHEbgTPN7LH2MClfwRMIsYlqTYpKTTe2A==}
peerDependencies:
@ -6554,7 +6596,7 @@ snapshots:
'@marijn/find-cluster-break@1.0.2': {}
'@mdx-js/mdx@3.1.0(acorn@8.14.0)':
'@mdx-js/mdx@3.1.0(acorn@8.15.0)':
dependencies:
'@types/estree': 1.0.8
'@types/estree-jsx': 1.0.5
@ -6568,7 +6610,7 @@ snapshots:
hast-util-to-jsx-runtime: 2.3.6
markdown-extensions: 2.0.0
recma-build-jsx: 1.0.0
recma-jsx: 1.0.0(acorn@8.14.0)
recma-jsx: 1.0.0(acorn@8.15.0)
recma-stringify: 1.0.0
rehype-recma: 1.0.0
remark-mdx: 3.1.0
@ -7260,6 +7302,16 @@ snapshots:
'@types/react': 19.1.8
'@types/react-dom': 19.1.6(@types/react@19.1.8)
'@radix-ui/react-presence@1.1.5(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
dependencies:
'@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.1.8)(react@19.1.0)
react: 19.1.0
react-dom: 19.1.0(react@19.1.0)
optionalDependencies:
'@types/react': 19.1.8
'@types/react-dom': 19.1.6(@types/react@19.1.8)
'@radix-ui/react-primitive@1.0.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
dependencies:
'@babel/runtime': 7.26.9
@ -7305,6 +7357,24 @@ snapshots:
'@types/react': 19.1.8
'@types/react-dom': 19.1.6(@types/react@19.1.8)
'@radix-ui/react-radio-group@1.3.8(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
dependencies:
'@radix-ui/primitive': 1.1.3
'@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-context': 1.1.2(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-direction': 1.1.1(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-presence': 1.1.5(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
'@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
'@radix-ui/react-roving-focus': 1.1.11(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
'@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-use-previous': 1.1.1(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-use-size': 1.1.1(@types/react@19.1.8)(react@19.1.0)
react: 19.1.0
react-dom: 19.1.0(react@19.1.0)
optionalDependencies:
'@types/react': 19.1.8
'@types/react-dom': 19.1.6(@types/react@19.1.8)
'@radix-ui/react-roving-focus@1.1.10(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
dependencies:
'@radix-ui/primitive': 1.1.2
@ -7322,6 +7392,23 @@ snapshots:
'@types/react': 19.1.8
'@types/react-dom': 19.1.6(@types/react@19.1.8)
'@radix-ui/react-roving-focus@1.1.11(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
dependencies:
'@radix-ui/primitive': 1.1.3
'@radix-ui/react-collection': 1.1.7(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
'@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-context': 1.1.2(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-direction': 1.1.1(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-id': 1.1.1(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
'@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.1.8)(react@19.1.0)
'@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.1.8)(react@19.1.0)
react: 19.1.0
react-dom: 19.1.0(react@19.1.0)
optionalDependencies:
'@types/react': 19.1.8
'@types/react-dom': 19.1.6(@types/react@19.1.8)
'@radix-ui/react-scroll-area@1.2.9(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
dependencies:
'@radix-ui/number': 1.1.1
@ -9073,9 +9160,9 @@ snapshots:
transitivePeerDependencies:
- supports-color
fumadocs-mdx@11.7.1(acorn@8.14.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0):
fumadocs-mdx@11.7.1(acorn@8.15.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0):
dependencies:
'@mdx-js/mdx': 3.1.0(acorn@8.14.0)
'@mdx-js/mdx': 3.1.0(acorn@8.15.0)
'@standard-schema/spec': 1.0.0
chokidar: 4.0.3
esbuild: 0.25.8
@ -11137,9 +11224,9 @@ snapshots:
estree-util-build-jsx: 3.0.1
vfile: 6.0.3
recma-jsx@1.0.0(acorn@8.14.0):
recma-jsx@1.0.0(acorn@8.15.0):
dependencies:
acorn-jsx: 5.3.2(acorn@8.14.0)
acorn-jsx: 5.3.2(acorn@8.15.0)
estree-util-to-js: 2.0.0
recma-parse: 1.0.0
recma-stringify: 1.0.0