mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-17 18:35:19 +02:00
feat: added elasticsearch connector
This commit is contained in:
parent
402039f02f
commit
55d752e3c8
27 changed files with 4331 additions and 2499 deletions
|
|
@ -75,6 +75,7 @@ Open source and easy to deploy locally.
|
|||
- Airtable
|
||||
- Google Calendar
|
||||
- Luma
|
||||
- Elasticsearch
|
||||
- and more to come.....
|
||||
|
||||
## 📄 **Supported File Extensions**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
"""Add ElasticSearch connector enums
|
||||
|
||||
Revision ID: 26
|
||||
Revises: 25
|
||||
Create Date: 2025-10-12 12:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers
|
||||
revision: str = "26"
|
||||
down_revision: str | None = "25"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Add enum values
|
||||
op.execute(
|
||||
"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_type t
|
||||
JOIN pg_enum e ON t.oid = e.enumtypid
|
||||
WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'ELASTICSEARCH_CONNECTOR'
|
||||
) THEN
|
||||
ALTER TYPE searchsourceconnectortype ADD VALUE 'ELASTICSEARCH_CONNECTOR';
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_type t
|
||||
JOIN pg_enum e ON t.oid = e.enumtypid
|
||||
WHERE t.typname = 'documenttype' AND e.enumlabel = 'ELASTICSEARCH_CONNECTOR'
|
||||
) THEN
|
||||
ALTER TYPE documenttype ADD VALUE 'ELASTICSEARCH_CONNECTOR';
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove 'ELASTICSEARCH_CONNECTOR' from enum types."""
|
||||
pass
|
||||
|
|
@ -488,6 +488,25 @@ async def fetch_documents_by_ids(
|
|||
)
|
||||
url = metadata.get("url", "")
|
||||
|
||||
elif doc_type == "ELASTICSEARCH_CONNECTOR":
|
||||
# Prefer explicit title in metadata/source, otherwise fallback to doc.title
|
||||
es_title = (
|
||||
metadata.get("title")
|
||||
or metadata.get("es_title")
|
||||
or doc.title
|
||||
or f"Elasticsearch: {metadata.get('elasticsearch_index', '')}"
|
||||
)
|
||||
title = es_title
|
||||
description = metadata.get("description") or (
|
||||
doc.content[:100] + "..."
|
||||
if len(doc.content) > 100
|
||||
else doc.content
|
||||
)
|
||||
# If a link or index info is stored, surface it
|
||||
url = metadata.get("url", "") or metadata.get(
|
||||
"elasticsearch_index", ""
|
||||
)
|
||||
|
||||
else: # FILE and other types
|
||||
title = doc.title
|
||||
description = (
|
||||
|
|
@ -512,6 +531,7 @@ async def fetch_documents_by_ids(
|
|||
"SLACK_CONNECTOR": "Slack (Selected)",
|
||||
"NOTION_CONNECTOR": "Notion (Selected)",
|
||||
"GITHUB_CONNECTOR": "GitHub (Selected)",
|
||||
"ELASTICSEARCH_CONNECTOR": "Elasticsearch (Selected)",
|
||||
"YOUTUBE_VIDEO": "YouTube Videos (Selected)",
|
||||
"DISCORD_CONNECTOR": "Discord (Selected)",
|
||||
"JIRA_CONNECTOR": "Jira Issues (Selected)",
|
||||
|
|
@ -1266,6 +1286,33 @@ async def fetch_relevant_documents(
|
|||
}
|
||||
)
|
||||
|
||||
elif connector == "ELASTICSEARCH_CONNECTOR":
|
||||
(
|
||||
source_object,
|
||||
elasticsearch_chunks,
|
||||
) = await connector_service.search_elasticsearch(
|
||||
user_query=reformulated_query,
|
||||
user_id=user_id,
|
||||
search_space_id=search_space_id,
|
||||
top_k=top_k,
|
||||
search_mode=search_mode,
|
||||
)
|
||||
|
||||
# Add to sources and raw documents
|
||||
if source_object:
|
||||
all_sources.append(source_object)
|
||||
all_raw_documents.extend(elasticsearch_chunks)
|
||||
|
||||
# Stream found document count
|
||||
if streaming_service and writer:
|
||||
writer(
|
||||
{
|
||||
"yield_value": streaming_service.format_terminal_info_delta(
|
||||
f"🔎 Found {len(elasticsearch_chunks)} Elasticsearch chunks related to your query"
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logging.error("Error in search_airtable: %s", traceback.format_exc())
|
||||
error_message = f"Error searching connector {connector}: {e!s}"
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ def get_connector_emoji(connector_name: str) -> str:
|
|||
"GOOGLE_CALENDAR_CONNECTOR": "📅",
|
||||
"AIRTABLE_CONNECTOR": "🗃️",
|
||||
"LUMA_CONNECTOR": "✨",
|
||||
"ELASTICSEARCH_CONNECTOR": "🔎",
|
||||
}
|
||||
return connector_emojis.get(connector_name, "🔎")
|
||||
|
||||
|
|
@ -74,6 +75,7 @@ def get_connector_friendly_name(connector_name: str) -> str:
|
|||
"LINKUP_API": "Linkup Search",
|
||||
"AIRTABLE_CONNECTOR": "Airtable",
|
||||
"LUMA_CONNECTOR": "Luma",
|
||||
"ELASTICSEARCH_CONNECTOR": "Elasticsearch",
|
||||
}
|
||||
return connector_friendly_names.get(connector_name, connector_name)
|
||||
|
||||
|
|
|
|||
254
surfsense_backend/app/connectors/elasticsearch_connector.py
Normal file
254
surfsense_backend/app/connectors/elasticsearch_connector.py
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
"""
|
||||
Elasticsearch connector for SurfSense
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from elasticsearch import AsyncElasticsearch
|
||||
from elasticsearch.exceptions import (
|
||||
AuthenticationException,
|
||||
ConnectionError,
|
||||
NotFoundError,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ElasticsearchConnector:
|
||||
"""
|
||||
Connector for Elasticsearch instances
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
api_key: str | None = None,
|
||||
username: str | None = None,
|
||||
password: str | None = None,
|
||||
verify_certs: bool = True,
|
||||
ca_certs: str | None = None,
|
||||
):
|
||||
"""
|
||||
Initialize Elasticsearch connector
|
||||
|
||||
Args:
|
||||
url: Full Elasticsearch URL (e.g., https://host:port or cloud endpoint)
|
||||
api_key: API key for authentication (preferred method)
|
||||
username: Username for basic authentication
|
||||
password: Password for basic authentication
|
||||
verify_certs: Whether to verify SSL certificates
|
||||
ca_certs: Path to CA certificates file
|
||||
"""
|
||||
self.url = url
|
||||
self.api_key = api_key
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.verify_certs = verify_certs
|
||||
self.ca_certs = ca_certs
|
||||
|
||||
# Build connection configuration
|
||||
self.es_config = self._build_config()
|
||||
|
||||
# Initialize Elasticsearch client
|
||||
try:
|
||||
self.client = AsyncElasticsearch(**self.es_config)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize Elasticsearch client: {e}")
|
||||
raise
|
||||
|
||||
def _build_config(self) -> dict[str, Any]:
|
||||
"""Build Elasticsearch client configuration"""
|
||||
config = {
|
||||
"hosts": [self.url],
|
||||
"verify_certs": self.verify_certs,
|
||||
"request_timeout": 30,
|
||||
"max_retries": 3,
|
||||
"retry_on_timeout": True,
|
||||
}
|
||||
|
||||
# Authentication - API key takes precedence
|
||||
if self.api_key:
|
||||
config["api_key"] = self.api_key
|
||||
elif self.username and self.password:
|
||||
config["basic_auth"] = (self.username, self.password)
|
||||
|
||||
# SSL configuration
|
||||
if self.ca_certs:
|
||||
config["ca_certs"] = self.ca_certs
|
||||
|
||||
return config
|
||||
|
||||
async def search(
|
||||
self,
|
||||
index: str | list[str],
|
||||
query: dict[str, Any],
|
||||
size: int = 100,
|
||||
from_: int = 0,
|
||||
fields: list[str] | None = None,
|
||||
sort: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Search documents in Elasticsearch
|
||||
|
||||
Args:
|
||||
index: Elasticsearch index name or list of indices
|
||||
query: Elasticsearch query DSL
|
||||
size: Number of results to return
|
||||
from_: Starting offset for pagination
|
||||
fields: List of fields to include in response
|
||||
sort: Sort configuration
|
||||
|
||||
Returns:
|
||||
Elasticsearch search response
|
||||
"""
|
||||
try:
|
||||
search_body: dict[str, Any] = {
|
||||
"query": query,
|
||||
"size": size,
|
||||
"from": from_,
|
||||
}
|
||||
|
||||
if fields:
|
||||
search_body["_source"] = fields
|
||||
|
||||
if sort:
|
||||
search_body["sort"] = sort
|
||||
|
||||
response = await self.client.search(index=index, body=search_body)
|
||||
|
||||
logger.info(
|
||||
f"Successfully searched index '{index}', found {response['hits']['total']['value']} results"
|
||||
)
|
||||
return response
|
||||
|
||||
except NotFoundError:
|
||||
logger.error(f"Index '{index}' not found")
|
||||
raise
|
||||
except AuthenticationException:
|
||||
logger.error("Authentication failed")
|
||||
raise
|
||||
except ConnectionError:
|
||||
logger.error("Failed to connect to Elasticsearch")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Search failed: {e}")
|
||||
raise
|
||||
|
||||
async def get_indices(self) -> list[str]:
|
||||
"""
|
||||
Get list of available indices
|
||||
|
||||
Returns:
|
||||
List of index names
|
||||
"""
|
||||
try:
|
||||
indices = await self.client.indices.get_alias(index="*")
|
||||
return list(indices.keys())
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get indices: {e}")
|
||||
raise
|
||||
|
||||
async def get_mapping(self, index: str) -> dict[str, Any]:
|
||||
"""
|
||||
Get mapping for an index
|
||||
|
||||
Args:
|
||||
index: Index name
|
||||
|
||||
Returns:
|
||||
Index mapping
|
||||
"""
|
||||
try:
|
||||
mapping = await self.client.indices.get_mapping(index=index)
|
||||
return mapping[index]["mappings"] if index in mapping else {}
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get mapping for index '{index}': {e}")
|
||||
raise
|
||||
|
||||
async def scroll_search(
|
||||
self,
|
||||
index: str | list[str],
|
||||
query: dict[str, Any],
|
||||
size: int = 1000,
|
||||
scroll_timeout: str = "5m",
|
||||
fields: list[str] | None = None,
|
||||
):
|
||||
"""
|
||||
Perform a scroll search for large result sets
|
||||
|
||||
Args:
|
||||
index: Elasticsearch index name or list of indices
|
||||
query: Elasticsearch query DSL
|
||||
size: Number of results per scroll
|
||||
scroll_timeout: Scroll timeout
|
||||
fields: List of fields to include in response
|
||||
|
||||
Yields:
|
||||
Document hits from Elasticsearch
|
||||
"""
|
||||
try:
|
||||
search_body: dict[str, Any] = {
|
||||
"query": query,
|
||||
"size": size,
|
||||
}
|
||||
|
||||
if fields:
|
||||
search_body["_source"] = fields
|
||||
|
||||
# Initial search
|
||||
response = await self.client.search(
|
||||
index=index, body=search_body, scroll=scroll_timeout
|
||||
)
|
||||
|
||||
scroll_id = response.get("_scroll_id")
|
||||
hits = response["hits"]["hits"]
|
||||
|
||||
while hits:
|
||||
for hit in hits:
|
||||
yield hit
|
||||
|
||||
# Continue scrolling
|
||||
if scroll_id:
|
||||
response = await self.client.scroll(
|
||||
scroll_id=scroll_id, scroll=scroll_timeout
|
||||
)
|
||||
scroll_id = response.get("_scroll_id")
|
||||
hits = response["hits"]["hits"]
|
||||
|
||||
# Clear scroll
|
||||
if scroll_id:
|
||||
await self.client.clear_scroll(scroll_id=scroll_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Scroll search failed: {e}")
|
||||
raise
|
||||
|
||||
async def count_documents(
|
||||
self, index: str | list[str], query: dict[str, Any] | None = None
|
||||
) -> int:
|
||||
"""
|
||||
Count documents in an index
|
||||
|
||||
Args:
|
||||
index: Index name or list of indices
|
||||
query: Optional query to filter documents
|
||||
|
||||
Returns:
|
||||
Number of documents
|
||||
"""
|
||||
try:
|
||||
if query:
|
||||
response = await self.client.count(index=index, body={"query": query})
|
||||
else:
|
||||
response = await self.client.count(index=index)
|
||||
|
||||
return response["count"]
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to count documents in index '{index}': {e}")
|
||||
raise
|
||||
|
||||
async def close(self):
|
||||
"""Close the Elasticsearch client connection"""
|
||||
if hasattr(self, "client"):
|
||||
await self.client.close()
|
||||
|
|
@ -50,6 +50,7 @@ class DocumentType(str, Enum):
|
|||
GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR"
|
||||
AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR"
|
||||
LUMA_CONNECTOR = "LUMA_CONNECTOR"
|
||||
ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR"
|
||||
|
||||
|
||||
class SearchSourceConnectorType(str, Enum):
|
||||
|
|
@ -68,6 +69,7 @@ class SearchSourceConnectorType(str, Enum):
|
|||
GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR"
|
||||
AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR"
|
||||
LUMA_CONNECTOR = "LUMA_CONNECTOR"
|
||||
ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR"
|
||||
|
||||
|
||||
class ChatType(str, Enum):
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from app.tasks.connector_indexers import (
|
|||
index_clickup_tasks,
|
||||
index_confluence_pages,
|
||||
index_discord_messages,
|
||||
index_elasticsearch_documents,
|
||||
index_github_repos,
|
||||
index_google_calendar_events,
|
||||
index_google_gmail_messages,
|
||||
|
|
@ -363,6 +364,7 @@ async def index_connector_content(
|
|||
- JIRA_CONNECTOR: Indexes issues and comments from Jira
|
||||
- DISCORD_CONNECTOR: Indexes messages from all accessible Discord channels
|
||||
- LUMA_CONNECTOR: Indexes events from Luma
|
||||
- ELASTICSEARCH_CONNECTOR: Indexes documents from Elasticsearch
|
||||
|
||||
Args:
|
||||
connector_id: ID of the connector to use
|
||||
|
|
@ -589,6 +591,24 @@ async def index_connector_content(
|
|||
)
|
||||
response_message = "Luma indexing started in the background."
|
||||
|
||||
elif (
|
||||
connector.connector_type
|
||||
== SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR
|
||||
):
|
||||
# Run indexing in background
|
||||
logger.info(
|
||||
f"Triggering Elasticsearch indexing for connector {connector_id} into search space {search_space_id}"
|
||||
)
|
||||
background_tasks.add_task(
|
||||
run_elasticsearch_indexing_with_new_session,
|
||||
connector_id,
|
||||
search_space_id,
|
||||
str(user.id),
|
||||
indexing_from,
|
||||
indexing_to,
|
||||
)
|
||||
response_message = "Elasticsearch indexing started in the background."
|
||||
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
|
|
@ -1358,3 +1378,61 @@ async def run_luma_indexing(
|
|||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in background Luma indexing task: {e!s}")
|
||||
|
||||
|
||||
async def run_elasticsearch_indexing_with_new_session(
|
||||
connector_id: int,
|
||||
search_space_id: int,
|
||||
user_id: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
):
|
||||
"""Wrapper to run Elasticsearch indexing with its own database session."""
|
||||
logger.info(
|
||||
f"Background task started: Indexing Elasticsearch connector {connector_id} into space {search_space_id}"
|
||||
)
|
||||
async with async_session_maker() as session:
|
||||
await run_elasticsearch_indexing(
|
||||
session, connector_id, search_space_id, user_id, start_date, end_date
|
||||
)
|
||||
logger.info(
|
||||
f"Background task finished: Indexing Elasticsearch connector {connector_id}"
|
||||
)
|
||||
|
||||
|
||||
async def run_elasticsearch_indexing(
|
||||
session: AsyncSession,
|
||||
connector_id: int,
|
||||
search_space_id: int,
|
||||
user_id: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
):
|
||||
"""Runs the Elasticsearch indexing task and updates the timestamp."""
|
||||
try:
|
||||
indexed_count, error_message = await index_elasticsearch_documents(
|
||||
session,
|
||||
connector_id,
|
||||
search_space_id,
|
||||
user_id,
|
||||
start_date,
|
||||
end_date,
|
||||
update_last_indexed=False,
|
||||
)
|
||||
if error_message:
|
||||
logger.error(
|
||||
f"Elasticsearch indexing failed for connector {connector_id}: {error_message}"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Elasticsearch indexing successful for connector {connector_id}. Indexed {indexed_count} documents."
|
||||
)
|
||||
# Update the last indexed timestamp only on success
|
||||
await update_connector_last_indexed(session, connector_id)
|
||||
await session.commit()
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(
|
||||
f"Critical error in run_elasticsearch_indexing for connector {connector_id}: {e}",
|
||||
exc_info=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2028,3 +2028,117 @@ class ConnectorService:
|
|||
}
|
||||
|
||||
return result_object, luma_chunks
|
||||
|
||||
async def search_elasticsearch(
|
||||
self,
|
||||
user_query: str,
|
||||
user_id: str,
|
||||
search_space_id: int,
|
||||
top_k: int = 20,
|
||||
search_mode: SearchMode = SearchMode.CHUNKS,
|
||||
) -> tuple:
|
||||
"""
|
||||
Search for Elasticsearch documents and return both the source information and langchain documents
|
||||
|
||||
Args:
|
||||
user_query: The user's query
|
||||
user_id: The user's ID
|
||||
search_space_id: The search space ID to search in
|
||||
top_k: Maximum number of results to return
|
||||
search_mode: Search mode (CHUNKS or DOCUMENTS)
|
||||
|
||||
Returns:
|
||||
tuple: (sources_info, langchain_documents)
|
||||
"""
|
||||
if search_mode == SearchMode.CHUNKS:
|
||||
elasticsearch_chunks = await self.chunk_retriever.hybrid_search(
|
||||
query_text=user_query,
|
||||
top_k=top_k,
|
||||
user_id=user_id,
|
||||
search_space_id=search_space_id,
|
||||
document_type="ELASTICSEARCH_CONNECTOR",
|
||||
)
|
||||
elif search_mode == SearchMode.DOCUMENTS:
|
||||
elasticsearch_chunks = await self.document_retriever.hybrid_search(
|
||||
query_text=user_query,
|
||||
top_k=top_k,
|
||||
user_id=user_id,
|
||||
search_space_id=search_space_id,
|
||||
document_type="ELASTICSEARCH_CONNECTOR",
|
||||
)
|
||||
# Transform document retriever results to match expected format
|
||||
elasticsearch_chunks = self._transform_document_results(
|
||||
elasticsearch_chunks
|
||||
)
|
||||
|
||||
# Early return if no results
|
||||
if not elasticsearch_chunks:
|
||||
return {
|
||||
"id": 34,
|
||||
"name": "Elasticsearch",
|
||||
"type": "ELASTICSEARCH_CONNECTOR",
|
||||
"sources": [],
|
||||
}, []
|
||||
|
||||
# Process each chunk and create sources directly without deduplication
|
||||
sources_list = []
|
||||
async with self.counter_lock:
|
||||
for _i, chunk in enumerate(elasticsearch_chunks):
|
||||
# Extract document metadata
|
||||
document = chunk.get("document", {})
|
||||
metadata = document.get("metadata", {})
|
||||
|
||||
# Extract Elasticsearch-specific metadata
|
||||
es_id = metadata.get("elasticsearch_id", "")
|
||||
es_index = metadata.get("elasticsearch_index", "")
|
||||
es_score = metadata.get("elasticsearch_score", "")
|
||||
|
||||
# Create a more descriptive title for Elasticsearch documents
|
||||
title = document.get("title", "Elasticsearch Document")
|
||||
if es_index:
|
||||
title = f"{title} (Index: {es_index})"
|
||||
|
||||
# Create a more descriptive description for Elasticsearch documents
|
||||
description = chunk.get("content", "")[:150]
|
||||
if len(description) == 150:
|
||||
description += "..."
|
||||
|
||||
# Add Elasticsearch info to description
|
||||
info_parts = []
|
||||
if es_id:
|
||||
info_parts.append(f"ID: {es_id}")
|
||||
if es_score:
|
||||
info_parts.append(f"Score: {es_score}")
|
||||
|
||||
if info_parts:
|
||||
if description:
|
||||
description = f"{description} | {' | '.join(info_parts)}"
|
||||
else:
|
||||
description = " | ".join(info_parts)
|
||||
|
||||
# For URL, we could construct a URL to view the document if we have the Elasticsearch UI URL
|
||||
url = ""
|
||||
# Could be extended to include Kibana or other UI URLs if configured
|
||||
|
||||
source = {
|
||||
"id": chunk.get("chunk_id", self.source_id_counter),
|
||||
"title": title,
|
||||
"description": description,
|
||||
"url": url,
|
||||
"elasticsearch_id": es_id,
|
||||
"elasticsearch_index": es_index,
|
||||
"elasticsearch_score": es_score,
|
||||
}
|
||||
|
||||
self.source_id_counter += 1
|
||||
sources_list.append(source)
|
||||
|
||||
# Create result object
|
||||
result_object = {
|
||||
"id": 34, # Assign a unique ID for the Elasticsearch connector
|
||||
"name": "Elasticsearch",
|
||||
"type": "ELASTICSEARCH_CONNECTOR",
|
||||
"sources": sources_list,
|
||||
}
|
||||
|
||||
return result_object, elasticsearch_chunks
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ Available indexers:
|
|||
- Google Gmail: Index messages from Google Gmail
|
||||
- Google Calendar: Index events from Google Calendar
|
||||
- Luma: Index events from Luma
|
||||
- Elasticsearch: Index documents from Elasticsearch instances
|
||||
"""
|
||||
|
||||
# Communication platforms
|
||||
|
|
@ -27,6 +28,7 @@ from .confluence_indexer import index_confluence_pages
|
|||
from .discord_indexer import index_discord_messages
|
||||
|
||||
# Development platforms
|
||||
from .elasticsearch_indexer import index_elasticsearch_documents
|
||||
from .github_indexer import index_github_repos
|
||||
from .google_calendar_indexer import index_google_calendar_events
|
||||
from .google_gmail_indexer import index_google_gmail_messages
|
||||
|
|
@ -46,6 +48,7 @@ __all__ = [ # noqa: RUF022
|
|||
"index_confluence_pages",
|
||||
"index_discord_messages",
|
||||
# Development platforms
|
||||
"index_elasticsearch_documents",
|
||||
"index_github_repos",
|
||||
# Calendar and scheduling
|
||||
"index_google_calendar_events",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,354 @@
|
|||
"""
|
||||
Elasticsearch indexer for SurfSense
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from app.connectors.elasticsearch_connector import ElasticsearchConnector
|
||||
from app.db import Document, DocumentType, SearchSourceConnector
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _ChunkingService:
|
||||
def __init__(self, chunk_size: int = 1000, overlap: int = 200) -> None:
|
||||
self.chunk_size = max(100, chunk_size)
|
||||
self.overlap = max(0, min(overlap, self.chunk_size - 1))
|
||||
|
||||
def chunk_text(self, text: str) -> list[str]:
|
||||
if not text:
|
||||
return []
|
||||
text = text.strip()
|
||||
if len(text) <= self.chunk_size:
|
||||
return [text]
|
||||
chunks: list[str] = []
|
||||
step = self.chunk_size - self.overlap
|
||||
pos = 0
|
||||
while pos < len(text):
|
||||
end = pos + self.chunk_size
|
||||
chunks.append(text[pos:end].strip())
|
||||
pos += step
|
||||
return chunks
|
||||
|
||||
|
||||
class _DocumentService:
|
||||
def __init__(self, session):
|
||||
self.session = session
|
||||
|
||||
async def get_document_by_hash(self, content_hash: str):
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from app.db import Document
|
||||
|
||||
if not content_hash:
|
||||
return None
|
||||
result = await self.session.execute(
|
||||
select(Document).where(Document.content_hash == content_hash)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
async def create_chunks_for_document(self, document_id: int, chunks: list[str]):
|
||||
from app.db import Chunk
|
||||
|
||||
for chunk_text in chunks:
|
||||
self.session.add(Chunk(content=chunk_text, document_id=document_id))
|
||||
await self.session.flush()
|
||||
|
||||
|
||||
async def index_elasticsearch_documents(
|
||||
session: AsyncSession,
|
||||
connector_id: int,
|
||||
search_space_id: int,
|
||||
user_id: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
update_last_indexed: bool = True,
|
||||
) -> tuple[int, str | None]:
|
||||
"""
|
||||
Index documents from Elasticsearch into SurfSense
|
||||
|
||||
Args:
|
||||
session: Database session
|
||||
connector_id: Elasticsearch connector ID
|
||||
search_space_id: Search space ID
|
||||
user_id: User ID
|
||||
start_date: Start date for indexing (not used for Elasticsearch, kept for compatibility)
|
||||
end_date: End date for indexing (not used for Elasticsearch, kept for compatibility)
|
||||
update_last_indexed: Whether to update the last indexed timestamp
|
||||
|
||||
Returns:
|
||||
Tuple of (number of documents processed, error message if any)
|
||||
"""
|
||||
es_connector = None
|
||||
try:
|
||||
# Get the connector configuration
|
||||
result = await session.execute(
|
||||
select(SearchSourceConnector).filter(
|
||||
SearchSourceConnector.id == connector_id
|
||||
)
|
||||
)
|
||||
connector = result.scalars().first()
|
||||
|
||||
if not connector:
|
||||
error_msg = f"Elasticsearch connector with ID {connector_id} not found"
|
||||
logger.error(error_msg)
|
||||
return 0, error_msg
|
||||
|
||||
# Get connector configuration
|
||||
config = connector.config
|
||||
|
||||
# Validate required fields - now only URL and INDEX are required
|
||||
# Authentication can be either API key OR username/password
|
||||
if "ELASTICSEARCH_URL" not in config:
|
||||
error_msg = "Missing required field in connector config: ELASTICSEARCH_URL"
|
||||
logger.error(error_msg)
|
||||
return 0, error_msg
|
||||
|
||||
if "ELASTICSEARCH_INDEX" not in config:
|
||||
error_msg = (
|
||||
"Missing required field in connector config: ELASTICSEARCH_INDEX"
|
||||
)
|
||||
logger.error(error_msg)
|
||||
return 0, error_msg
|
||||
|
||||
# Check authentication - must have either API key or username+password
|
||||
has_api_key = (
|
||||
"ELASTICSEARCH_API_KEY" in config and config["ELASTICSEARCH_API_KEY"]
|
||||
)
|
||||
has_basic_auth = (
|
||||
"ELASTICSEARCH_USERNAME" in config
|
||||
and config["ELASTICSEARCH_USERNAME"]
|
||||
and "ELASTICSEARCH_PASSWORD" in config
|
||||
and config["ELASTICSEARCH_PASSWORD"]
|
||||
)
|
||||
|
||||
if not has_api_key and not has_basic_auth:
|
||||
error_msg = "Missing authentication: provide either ELASTICSEARCH_API_KEY or ELASTICSEARCH_USERNAME + ELASTICSEARCH_PASSWORD"
|
||||
logger.error(error_msg)
|
||||
return 0, error_msg
|
||||
|
||||
# Initialize document service
|
||||
document_service = _DocumentService(session)
|
||||
chunking_service = _ChunkingService()
|
||||
|
||||
# Initialize Elasticsearch connector
|
||||
es_connector = ElasticsearchConnector(
|
||||
url=config["ELASTICSEARCH_URL"],
|
||||
api_key=config.get("ELASTICSEARCH_API_KEY"),
|
||||
username=config.get("ELASTICSEARCH_USERNAME"),
|
||||
password=config.get("ELASTICSEARCH_PASSWORD"),
|
||||
verify_certs=config.get("ELASTICSEARCH_VERIFY_CERTS", True),
|
||||
ca_certs=config.get("ELASTICSEARCH_CA_CERTS"),
|
||||
)
|
||||
|
||||
# Build query based on configuration
|
||||
query = _build_elasticsearch_query(config)
|
||||
|
||||
# Get the index name(s) - can be a string or list
|
||||
index_name = config["ELASTICSEARCH_INDEX"]
|
||||
|
||||
# Get max documents to index
|
||||
max_documents = config.get("ELASTICSEARCH_MAX_DOCUMENTS", 1000)
|
||||
|
||||
logger.info(
|
||||
f"Starting Elasticsearch indexing for index '{index_name}' with max {max_documents} documents"
|
||||
)
|
||||
|
||||
documents_processed = 0
|
||||
|
||||
try:
|
||||
# Use scroll search for large result sets
|
||||
async for hit in es_connector.scroll_search(
|
||||
index=index_name,
|
||||
query=query,
|
||||
size=min(max_documents, 100), # Scroll in batches
|
||||
fields=config.get("ELASTICSEARCH_FIELDS"),
|
||||
):
|
||||
if documents_processed >= max_documents:
|
||||
break
|
||||
|
||||
try:
|
||||
# Extract document data
|
||||
doc_id = hit["_id"]
|
||||
source = hit.get("_source", {})
|
||||
|
||||
# Build document title
|
||||
title_field = config.get("ELASTICSEARCH_TITLE_FIELD")
|
||||
if not title_field:
|
||||
for candidate in ("title", "name", "subject"):
|
||||
if candidate in source:
|
||||
title_field = candidate
|
||||
break
|
||||
title = (
|
||||
str(source.get(title_field, doc_id))
|
||||
if title_field is not None
|
||||
else str(doc_id)
|
||||
)
|
||||
|
||||
# Build document content
|
||||
content = _build_document_content(source, config)
|
||||
|
||||
if not content.strip():
|
||||
logger.warning(f"Skipping document {doc_id} - no content found")
|
||||
continue
|
||||
|
||||
# Create content hash
|
||||
content_hash = hashlib.sha256(content.encode()).hexdigest()
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
"elasticsearch_id": doc_id,
|
||||
"elasticsearch_index": hit.get("_index", index_name),
|
||||
"elasticsearch_score": hit.get("_score"),
|
||||
"indexed_at": datetime.now().isoformat(),
|
||||
"source": "ELASTICSEARCH_CONNECTOR",
|
||||
}
|
||||
|
||||
# Add any additional metadata fields specified in config
|
||||
if "ELASTICSEARCH_METADATA_FIELDS" in config:
|
||||
for field in config["ELASTICSEARCH_METADATA_FIELDS"]:
|
||||
if field in source:
|
||||
metadata[f"es_{field}"] = source[field]
|
||||
|
||||
# Check if document already exists
|
||||
existing_doc = await document_service.get_document_by_hash(
|
||||
content_hash
|
||||
)
|
||||
|
||||
if existing_doc:
|
||||
logger.debug(f"Document {doc_id} already exists, skipping")
|
||||
continue
|
||||
|
||||
# Create document
|
||||
document = Document(
|
||||
title=title,
|
||||
content=content,
|
||||
content_hash=content_hash,
|
||||
document_type=DocumentType.ELASTICSEARCH_CONNECTOR,
|
||||
document_metadata=metadata,
|
||||
search_space_id=search_space_id,
|
||||
)
|
||||
|
||||
# Add document to session
|
||||
session.add(document)
|
||||
await session.flush() # Get the document ID
|
||||
|
||||
# Create chunks
|
||||
chunks = chunking_service.chunk_text(content)
|
||||
await document_service.create_chunks_for_document(
|
||||
document.id, chunks
|
||||
)
|
||||
|
||||
documents_processed += 1
|
||||
|
||||
if documents_processed % 10 == 0:
|
||||
logger.info(
|
||||
f"Processed {documents_processed} Elasticsearch documents"
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error processing Elasticsearch document {hit.get('_id', 'unknown')}: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
# Final commit
|
||||
await session.commit()
|
||||
|
||||
logger.info(
|
||||
f"Successfully indexed {documents_processed} documents from Elasticsearch"
|
||||
)
|
||||
|
||||
# Update last indexed timestamp if requested
|
||||
if update_last_indexed and documents_processed > 0:
|
||||
connector.last_indexed_at = datetime.now()
|
||||
await session.commit()
|
||||
if update_last_indexed and documents_processed > 0:
|
||||
# store ISO-8601 UTC timestamp with 'Z' suffix, e.g. 2025-10-09T22:04:53.599658Z
|
||||
connector.last_indexed_at = (
|
||||
datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
return documents_processed, None
|
||||
|
||||
finally:
|
||||
# Clean up Elasticsearch connection
|
||||
if es_connector:
|
||||
await es_connector.close()
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error indexing Elasticsearch documents: {e}"
|
||||
logger.error(error_msg, exc_info=True)
|
||||
await session.rollback()
|
||||
if es_connector:
|
||||
await es_connector.close()
|
||||
return 0, error_msg
|
||||
|
||||
|
||||
def _build_elasticsearch_query(config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Build Elasticsearch query from connector configuration
|
||||
|
||||
Args:
|
||||
config: Connector configuration
|
||||
|
||||
Returns:
|
||||
Elasticsearch query DSL
|
||||
"""
|
||||
# Check if custom query is provided
|
||||
if config.get("ELASTICSEARCH_QUERY"):
|
||||
try:
|
||||
if isinstance(config["ELASTICSEARCH_QUERY"], str):
|
||||
return json.loads(config["ELASTICSEARCH_QUERY"])
|
||||
else:
|
||||
return config["ELASTICSEARCH_QUERY"]
|
||||
except (json.JSONDecodeError, TypeError) as e:
|
||||
logger.warning(f"Invalid custom query, using match_all: {e}")
|
||||
|
||||
# Default to match all documents
|
||||
return {"match_all": {}}
|
||||
|
||||
|
||||
def _build_document_content(source: dict[str, Any], config: dict[str, Any]) -> str:
|
||||
"""
|
||||
Build document content from Elasticsearch document source
|
||||
|
||||
Args:
|
||||
source: Elasticsearch document source
|
||||
config: Connector configuration
|
||||
|
||||
Returns:
|
||||
Formatted document content
|
||||
"""
|
||||
content_parts = []
|
||||
|
||||
# Get content fields from config
|
||||
content_fields = config.get("ELASTICSEARCH_CONTENT_FIELDS", [])
|
||||
|
||||
if content_fields:
|
||||
# Use specified content fields
|
||||
for field in content_fields:
|
||||
if field in source:
|
||||
field_value = source[field]
|
||||
if isinstance(field_value, str | int | float):
|
||||
content_parts.append(f"{field}: {field_value}")
|
||||
if isinstance(field_value, str | int | float):
|
||||
content_parts.append(f"{field}: {json.dumps(field_value)}")
|
||||
else:
|
||||
# Use all fields if no specific content fields specified
|
||||
for key, value in source.items():
|
||||
if isinstance(value, str | int | float):
|
||||
content_parts.append(f"{key}: {value}")
|
||||
elif isinstance(value, list | dict):
|
||||
content_parts.append(f"{key}: {json.dumps(value)}")
|
||||
|
||||
return "\n".join(content_parts)
|
||||
|
|
@ -43,6 +43,7 @@ dependencies = [
|
|||
"youtube-transcript-api>=1.0.3",
|
||||
"litellm>=1.77.5",
|
||||
"langchain-litellm>=0.2.3",
|
||||
"elasticsearch>=9.1.1",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
|
|
|
|||
5003
surfsense_backend/uv.lock
generated
5003
surfsense_backend/uv.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -271,6 +271,17 @@ export default function EditConnectorPage() {
|
|||
placeholder="API Key..."
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* == Elasticsearch == */}
|
||||
{connector.connector_type === "ELASTICSEARCH_CONNECTOR" && (
|
||||
<EditSimpleTokenForm
|
||||
control={editForm.control}
|
||||
fieldName="ELASTICSEARCH_API_KEY"
|
||||
fieldLabel="Elasticsearch API Key"
|
||||
fieldDescription="Update your Elasticsearch API Key if needed."
|
||||
placeholder="Your Elasticsearch API Key"
|
||||
/>
|
||||
)}
|
||||
</CardContent>
|
||||
<CardFooter className="border-t pt-6">
|
||||
<Button type="submit" disabled={isSaving} className="w-full sm:w-auto">
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ const getConnectorTypeDisplay = (type: string): string => {
|
|||
GOOGLE_GMAIL_CONNECTOR: "Google Gmail Connector",
|
||||
AIRTABLE_CONNECTOR: "Airtable Connector",
|
||||
LUMA_CONNECTOR: "Luma Connector",
|
||||
ELASTICSEARCH_CONNECTOR: "Elasticsearch Connector",
|
||||
// Add other connector types here as needed
|
||||
};
|
||||
return typeMap[type] || type;
|
||||
|
|
@ -233,7 +234,9 @@ export default function EditConnectorPage() {
|
|||
? "GitHub Personal Access Token (PAT)"
|
||||
: connector?.connector_type === "LINKUP_API"
|
||||
? "Linkup API Key"
|
||||
: "API Key"}
|
||||
: connector?.connector_type === "ELASTICSEARCH_CONNECTOR"
|
||||
? "Elasticsearch API Key"
|
||||
: "API Key"}
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
|
|
@ -247,7 +250,9 @@ export default function EditConnectorPage() {
|
|||
? "Enter new GitHub PAT (optional)"
|
||||
: connector?.connector_type === "LINKUP_API"
|
||||
? "Enter new Linkup API Key (optional)"
|
||||
: "Enter new API key (optional)"
|
||||
: connector?.connector_type === "ELASTICSEARCH_CONNECTOR"
|
||||
? "Enter new Elasticsearch API Key (optional)"
|
||||
: "Enter new API key (optional)"
|
||||
}
|
||||
{...field}
|
||||
/>
|
||||
|
|
@ -261,7 +266,9 @@ export default function EditConnectorPage() {
|
|||
? "Enter a new GitHub PAT or leave blank to keep your existing token."
|
||||
: connector?.connector_type === "LINKUP_API"
|
||||
? "Enter a new Linkup API Key or leave blank to keep your existing key."
|
||||
: "Enter a new API key or leave blank to keep your existing key."}
|
||||
: connector?.connector_type === "ELASTICSEARCH_CONNECTOR"
|
||||
? "Enter a new Elasticsearch API Key or leave blank to keep your existing key."
|
||||
: "Enter a new API key or leave blank to keep your existing key."}
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,751 @@
|
|||
"use client";
|
||||
|
||||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import * as RadioGroup from "@radix-ui/react-radio-group";
|
||||
import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
|
||||
import { motion } from "motion/react";
|
||||
import { useParams, useRouter, useSearchParams } from "next/navigation";
|
||||
import { useId, useState } from "react";
|
||||
import { useForm } from "react-hook-form";
|
||||
import { toast } from "sonner";
|
||||
import * as z from "zod";
|
||||
|
||||
import {
|
||||
Accordion,
|
||||
AccordionContent,
|
||||
AccordionItem,
|
||||
AccordionTrigger,
|
||||
} from "@/components/ui/accordion";
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardFooter,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import {
|
||||
Form,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
} from "@/components/ui/form";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Separator } from "@/components/ui/separator";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
|
||||
import { EnumConnectorName } from "@/contracts/enums/connector";
|
||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||
import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
|
||||
|
||||
// Define the form schema with Zod
|
||||
const elasticsearchConnectorFormSchema = z
|
||||
.object({
|
||||
name: z.string().min(3, {
|
||||
message: "Connector name must be at least 3 characters.",
|
||||
}),
|
||||
endpoint_url: z.string().url({ message: "Please enter a valid Elasticsearch endpoint URL." }),
|
||||
auth_method: z.enum(["basic", "api_key"]).default("api_key"),
|
||||
username: z.string().optional(),
|
||||
password: z.string().optional(),
|
||||
ELASTICSEARCH_API_KEY: z.string().optional(),
|
||||
indices: z.string().optional(),
|
||||
query: z.string().default("*"),
|
||||
search_fields: z.string().optional(),
|
||||
max_documents: z.number().min(1).max(10000).optional(),
|
||||
})
|
||||
.refine(
|
||||
(data) => {
|
||||
if (data.auth_method === "basic") {
|
||||
return Boolean(data.username?.trim() && data.password?.trim());
|
||||
}
|
||||
if (data.auth_method === "api_key") {
|
||||
return Boolean(data.ELASTICSEARCH_API_KEY?.trim());
|
||||
}
|
||||
return true;
|
||||
},
|
||||
{
|
||||
message: "Authentication credentials are required for the selected method.",
|
||||
path: ["auth_method"],
|
||||
}
|
||||
);
|
||||
|
||||
// Define the type for the form values
|
||||
type ElasticsearchConnectorFormValues = z.infer<typeof elasticsearchConnectorFormSchema>;
|
||||
|
||||
export default function ElasticsearchConnectorPage() {
|
||||
const router = useRouter();
|
||||
const params = useParams();
|
||||
const searchParams = useSearchParams();
|
||||
// match pattern used in other connector pages: prefer route param, fallback to query param
|
||||
const searchSpaceId = (params.search_space_id ?? searchParams?.get("search_space_id")) as string;
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
|
||||
const authBasicId = useId();
|
||||
const authApiKeyId = useId();
|
||||
|
||||
const { createConnector } = useSearchSourceConnectors();
|
||||
|
||||
// Initialize the form
|
||||
const form = useForm<ElasticsearchConnectorFormValues>({
|
||||
resolver: zodResolver(elasticsearchConnectorFormSchema),
|
||||
defaultValues: {
|
||||
name: "Elasticsearch Connector",
|
||||
endpoint_url: "",
|
||||
auth_method: "api_key",
|
||||
username: "",
|
||||
password: "",
|
||||
ELASTICSEARCH_API_KEY: "",
|
||||
indices: "",
|
||||
query: "*",
|
||||
search_fields: "",
|
||||
max_documents: undefined,
|
||||
},
|
||||
});
|
||||
|
||||
const stringToArray = (str: string): string[] => {
|
||||
const items = str
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0);
|
||||
return Array.from(new Set(items));
|
||||
};
|
||||
|
||||
// Handle form submission
|
||||
const onSubmit = async (values: ElasticsearchConnectorFormValues) => {
|
||||
setIsSubmitting(true);
|
||||
if (!searchSpaceId) {
|
||||
toast.error(
|
||||
"Missing search_space_id (route or ?search_space_id=). Provide it in the URL or pick a search space."
|
||||
);
|
||||
setIsSubmitting(false);
|
||||
return;
|
||||
}
|
||||
const searchSpaceIdNum = Number(searchSpaceId);
|
||||
if (!Number.isInteger(searchSpaceIdNum) || searchSpaceIdNum <= 0) {
|
||||
toast.error("Invalid search_space_id. It must be a positive integer.");
|
||||
setIsSubmitting(false);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// Send full URL to backend (backend expects ELASTICSEARCH_URL)
|
||||
const config: Record<string, string | number | boolean | string[]> = {
|
||||
ELASTICSEARCH_URL: values.endpoint_url,
|
||||
// default to verifying certs; expose fields for CA/verify if UI added later
|
||||
ELASTICSEARCH_VERIFY_CERTS: true,
|
||||
};
|
||||
|
||||
if (values.auth_method === "basic") {
|
||||
if (values.username) config.ELASTICSEARCH_USERNAME = values.username;
|
||||
if (values.password) config.ELASTICSEARCH_PASSWORD = values.password;
|
||||
} else if (values.auth_method === "api_key") {
|
||||
if (values.ELASTICSEARCH_API_KEY)
|
||||
config.ELASTICSEARCH_API_KEY = values.ELASTICSEARCH_API_KEY;
|
||||
}
|
||||
|
||||
if (values.indices?.trim()) {
|
||||
const indicesArr = stringToArray(values.indices);
|
||||
config.ELASTICSEARCH_INDEX = indicesArr.length === 1 ? indicesArr[0] : indicesArr;
|
||||
}
|
||||
|
||||
if (values.query && values.query !== "*") {
|
||||
config.ELASTICSEARCH_QUERY = values.query;
|
||||
}
|
||||
|
||||
if (values.search_fields?.trim()) {
|
||||
config.ELASTICSEARCH_FIELDS = stringToArray(values.search_fields);
|
||||
const fields = stringToArray(values.search_fields);
|
||||
config.ELASTICSEARCH_FIELDS = fields;
|
||||
config.ELASTICSEARCH_CONTENT_FIELDS = fields;
|
||||
if (fields.includes("title")) {
|
||||
config.ELASTICSEARCH_TITLE_FIELD = "title";
|
||||
}
|
||||
}
|
||||
|
||||
if (values.max_documents !== undefined && values.max_documents > 0) {
|
||||
config.ELASTICSEARCH_MAX_DOCUMENTS = values.max_documents;
|
||||
}
|
||||
|
||||
const connectorPayload = {
|
||||
name: values.name,
|
||||
connector_type: EnumConnectorName.ELASTICSEARCH_CONNECTOR,
|
||||
is_indexable: true,
|
||||
search_space_id: searchSpaceIdNum,
|
||||
config,
|
||||
};
|
||||
|
||||
// Use existing hook method
|
||||
await createConnector(connectorPayload, searchSpaceIdNum);
|
||||
|
||||
toast.success("Elasticsearch connector created successfully!");
|
||||
router.push(`/dashboard/${searchSpaceId}/connectors`);
|
||||
} catch (error) {
|
||||
console.error("Error creating connector:", error);
|
||||
toast.error(error instanceof Error ? error.message : "Failed to create connector");
|
||||
} finally {
|
||||
setIsSubmitting(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="container mx-auto py-8 max-w-3xl">
|
||||
<Button
|
||||
variant="ghost"
|
||||
className="mb-6"
|
||||
onClick={() => router.push(`/dashboard/${searchSpaceId}/connectors/add`)}
|
||||
>
|
||||
<ArrowLeft className="mr-2 h-4 w-4" />
|
||||
Back to Connectors
|
||||
</Button>
|
||||
|
||||
{/* Header */}
|
||||
<div className="mb-8">
|
||||
<div className="flex items-center gap-4">
|
||||
<div className="flex h-12 w-12 items-center justify-center rounded-lg">
|
||||
{getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6")}
|
||||
</div>
|
||||
<div>
|
||||
<h1 className="text-3xl font-bold tracking-tight">Connect Elasticsearch</h1>
|
||||
<p className="text-muted-foreground">
|
||||
Connect to your Elasticsearch cluster to search and index documents.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<motion.div
|
||||
initial={{ opacity: 0, y: 20 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
transition={{ duration: 0.5 }}
|
||||
>
|
||||
<Tabs defaultValue="connect" className="w-full">
|
||||
<TabsList className="grid w-full grid-cols-2 mb-6">
|
||||
<TabsTrigger value="connect">Connect</TabsTrigger>
|
||||
<TabsTrigger value="documentation">Documentation</TabsTrigger>
|
||||
</TabsList>
|
||||
|
||||
<TabsContent value="connect">
|
||||
<Card className="border-2 border-border">
|
||||
<CardHeader>
|
||||
<CardTitle className="text-2xl font-bold">Connect Elasticsearch Cluster</CardTitle>
|
||||
<CardDescription>
|
||||
Connect to your Elasticsearch instance to search and index documents for enhanced
|
||||
search capabilities.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<Form {...form}>
|
||||
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-6">
|
||||
{/* Connector Name */}
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="name"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Connector Name</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder="My Elasticsearch Connector" {...field} />
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
A friendly name to identify this connector.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
{/* Connection Details */}
|
||||
<div className="space-y-4">
|
||||
<h3 className="text-lg font-medium">Connection Details</h3>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="endpoint_url"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Elasticsearch Endpoint URL</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="url"
|
||||
autoComplete="off"
|
||||
placeholder="https://your-cluster.es.region.aws.com:443"
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Enter the complete Elasticsearch endpoint URL. We'll automatically
|
||||
extract the hostname, port, and SSL settings.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
{/* Show parsed URL details */}
|
||||
{form.watch("endpoint_url") && (
|
||||
<div className="rounded-lg border bg-muted/50 p-3">
|
||||
<h4 className="text-sm font-medium mb-2">Parsed Connection Details:</h4>
|
||||
<div className="text-sm text-muted-foreground space-y-1">
|
||||
{(() => {
|
||||
try {
|
||||
const url = new URL(form.watch("endpoint_url"));
|
||||
return (
|
||||
<>
|
||||
<div>
|
||||
<strong>Hostname:</strong> {url.hostname}
|
||||
</div>
|
||||
<div>
|
||||
<strong>Port:</strong>{" "}
|
||||
{url.port || (url.protocol === "https:" ? "443" : "80")}
|
||||
</div>
|
||||
<div>
|
||||
<strong>SSL/TLS:</strong>{" "}
|
||||
{url.protocol === "https:" ? "Enabled" : "Disabled"}
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
} catch {
|
||||
return <div className="text-destructive">Invalid URL format</div>;
|
||||
}
|
||||
})()}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Authentication */}
|
||||
<div className="space-y-4">
|
||||
<h3 className="text-lg font-medium">Authentication</h3>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="auth_method"
|
||||
render={({ field }) => (
|
||||
<FormItem className="space-y-3">
|
||||
<FormControl>
|
||||
<RadioGroup.Root
|
||||
onValueChange={(value) => {
|
||||
field.onChange(value);
|
||||
// Clear auth fields when method changes
|
||||
if (value !== "basic") {
|
||||
form.setValue("username", "");
|
||||
form.setValue("password", "");
|
||||
}
|
||||
if (value !== "api_key") {
|
||||
form.setValue("ELASTICSEARCH_API_KEY", "");
|
||||
}
|
||||
}}
|
||||
value={field.value}
|
||||
className="flex flex-col space-y-2"
|
||||
>
|
||||
<div className="flex items-center space-x-2">
|
||||
<RadioGroup.Item
|
||||
value="api_key"
|
||||
id={authApiKeyId}
|
||||
className="aspect-square h-4 w-4 rounded-full border border-primary text-primary ring-offset-background focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground"
|
||||
>
|
||||
<RadioGroup.Indicator className="flex items-center justify-center">
|
||||
<div className="h-2.5 w-2.5 rounded-full bg-current" />
|
||||
</RadioGroup.Indicator>
|
||||
</RadioGroup.Item>
|
||||
<Label htmlFor={authApiKeyId}>API Key</Label>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center space-x-2">
|
||||
<RadioGroup.Item
|
||||
value="basic"
|
||||
id={authBasicId}
|
||||
className="aspect-square h-4 w-4 rounded-full border border-primary text-primary ring-offset-background focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground"
|
||||
>
|
||||
<RadioGroup.Indicator className="flex items-center justify-center">
|
||||
<div className="h-2.5 w-2.5 rounded-full bg-current" />
|
||||
</RadioGroup.Indicator>
|
||||
</RadioGroup.Item>
|
||||
<Label htmlFor={authBasicId}>Username & Password</Label>
|
||||
</div>
|
||||
</RadioGroup.Root>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
{/* Basic Auth Fields */}
|
||||
{form.watch("auth_method") === "basic" && (
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="username"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Username</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder="elastic" autoComplete="username" {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="password"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Password</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="password"
|
||||
placeholder="Password"
|
||||
autoComplete="current-password"
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* API Key Field */}
|
||||
{form.watch("auth_method") === "api_key" && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="ELASTICSEARCH_API_KEY"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>API Key</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="password"
|
||||
placeholder="Your API Key Here"
|
||||
autoComplete="off"
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Enter your Elasticsearch API key (base64 encoded). This will be
|
||||
stored securely.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Advanced Configuration */}
|
||||
<Accordion type="single" collapsible className="w-full">
|
||||
<AccordionItem value="advanced">
|
||||
<AccordionTrigger>Advanced Configuration</AccordionTrigger>
|
||||
<AccordionContent className="space-y-4">
|
||||
{/* Index Selection */}
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="indices"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>
|
||||
Index Selection{" "}
|
||||
<span className="text-muted-foreground">(Optional)</span>
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder="logs-*, documents-*, app-logs" {...field} />
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Comma-separated indices to search (e.g., "logs-*, documents-*").
|
||||
Leave empty for all indices. Supports wildcards.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
{/* Show parsed indices as badges */}
|
||||
{form.watch("indices")?.trim() && (
|
||||
<div className="rounded-lg border bg-muted/50 p-3">
|
||||
<h4 className="text-sm font-medium mb-2">Selected Indices:</h4>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{stringToArray(form.watch("indices")).map((index) => (
|
||||
<Badge key={index} variant="secondary" className="text-xs">
|
||||
{index}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="query"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>
|
||||
Default Search Query{" "}
|
||||
<span className="text-muted-foreground">(Optional)</span>
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder="*" {...field} />
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Default Elasticsearch query to use for searches. Use "*" to match
|
||||
all documents.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="search_fields"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>
|
||||
Search Fields{" "}
|
||||
<span className="text-muted-foreground">(Optional)</span>
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder="title, content, description" {...field} />
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Comma-separated list of specific fields to search in (e.g.,
|
||||
"title, content, description"). Leave empty to search all fields.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
{/* Show parsed search fields as badges */}
|
||||
{form.watch("search_fields")?.trim() && (
|
||||
<div className="rounded-lg border bg-muted/50 p-3">
|
||||
<h4 className="text-sm font-medium mb-2">Search Fields:</h4>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{stringToArray(form.watch("search_fields")).map((field) => (
|
||||
<Badge key={field} variant="outline" className="text-xs">
|
||||
{field}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="max_documents"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>
|
||||
Maximum Documents{" "}
|
||||
<span className="text-muted-foreground">(Optional)</span>
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="number"
|
||||
placeholder="1000"
|
||||
min="1"
|
||||
max="10000"
|
||||
{...field}
|
||||
onChange={(e) =>
|
||||
field.onChange(
|
||||
e.target.value === ""
|
||||
? undefined
|
||||
: parseInt(e.target.value, 10)
|
||||
)
|
||||
}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Maximum number of documents to retrieve per search (1-10,000).
|
||||
Leave empty to use Elasticsearch's default limit.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<Alert className="bg-muted">
|
||||
<Info className="h-4 w-4" />
|
||||
<AlertTitle>Index Selection Tips</AlertTitle>
|
||||
<AlertDescription className="mt-2">
|
||||
<ul className="list-disc pl-4 space-y-1 text-sm">
|
||||
<li>Use wildcards like "logs-*" to match multiple indices</li>
|
||||
<li>Separate multiple indices with commas</li>
|
||||
<li>Leave empty to search all accessible indices</li>
|
||||
<li>Choosing specific indices improves search performance</li>
|
||||
</ul>
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
</AccordionContent>
|
||||
</AccordionItem>
|
||||
</Accordion>
|
||||
|
||||
<Separator />
|
||||
|
||||
<div className="flex justify-end">
|
||||
<Button type="submit" disabled={isSubmitting} className="w-full sm:w-auto">
|
||||
{isSubmitting ? (
|
||||
<>
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
Connecting...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Check className="mr-2 h-4 w-4" />
|
||||
Connect Elasticsearch
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
</form>
|
||||
</Form>
|
||||
</CardContent>
|
||||
<CardFooter className="flex flex-col items-start border-t bg-muted/50 px-6 py-4">
|
||||
<h4 className="text-sm font-medium">
|
||||
What you get with Elasticsearch integration:
|
||||
</h4>
|
||||
<ul className="mt-2 list-disc pl-5 text-sm text-muted-foreground">
|
||||
<li>Search across your indexed documents and logs</li>
|
||||
<li>Access structured and unstructured data from your cluster</li>
|
||||
<li>Leverage existing Elasticsearch indices for enhanced search</li>
|
||||
<li>Real-time search capabilities with powerful query features</li>
|
||||
<li>Integration with your existing Elasticsearch infrastructure</li>
|
||||
</ul>
|
||||
</CardFooter>
|
||||
</Card>
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="documentation">
|
||||
<Card className="border-2 border-border">
|
||||
<CardHeader>
|
||||
<CardTitle className="text-2xl font-bold">
|
||||
Elasticsearch Connector Documentation
|
||||
</CardTitle>
|
||||
<CardDescription>
|
||||
Learn how to set up and use the Elasticsearch connector to search your data.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-6">
|
||||
<div>
|
||||
<h3 className="text-xl font-semibold mb-2">How it works</h3>
|
||||
<p className="text-muted-foreground">
|
||||
The Elasticsearch connector allows you to search and retrieve documents from
|
||||
your Elasticsearch cluster. Configure connection details, select specific
|
||||
indices, and set search parameters to make your existing data searchable within
|
||||
SurfSense.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<Accordion type="single" collapsible className="w-full">
|
||||
<AccordionItem value="connection">
|
||||
<AccordionTrigger className="text-lg font-medium">
|
||||
Connection Setup
|
||||
</AccordionTrigger>
|
||||
<AccordionContent className="space-y-4">
|
||||
<ol className="list-decimal pl-5 space-y-3">
|
||||
<li>
|
||||
<strong>Endpoint URL:</strong> Enter the complete Elasticsearch endpoint
|
||||
URL (e.g., https://your-cluster.es.region.aws.com:443). We'll
|
||||
automatically extract hostname, port, and SSL settings.
|
||||
</li>
|
||||
<li>
|
||||
<strong>Authentication:</strong> Choose the appropriate method:
|
||||
<ul className="list-disc pl-5 mt-1">
|
||||
<li>
|
||||
<strong>API Key:</strong> Base64 encoded API key (recommended for
|
||||
security)
|
||||
</li>
|
||||
<li>
|
||||
<strong>Username/Password:</strong> Basic authentication credentials
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ol>
|
||||
</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem value="advanced">
|
||||
<AccordionTrigger className="text-lg font-medium">
|
||||
Advanced Configuration
|
||||
</AccordionTrigger>
|
||||
<AccordionContent className="space-y-4">
|
||||
<p className="text-muted-foreground">
|
||||
Fine-tune your Elasticsearch connector with these optional settings:
|
||||
</p>
|
||||
<ul className="list-disc pl-5 space-y-2">
|
||||
<li>
|
||||
<strong>Index Selection:</strong> Specify which indices to search using
|
||||
comma-separated patterns (e.g., "logs-*, documents-*")
|
||||
</li>
|
||||
<li>
|
||||
<strong>Search Fields:</strong> Limit searches to specific fields (e.g.,
|
||||
"title, content") for better relevance
|
||||
</li>
|
||||
<li>
|
||||
<strong>Default Query:</strong> Set a default Elasticsearch query pattern
|
||||
</li>
|
||||
<li>
|
||||
<strong>Max Documents:</strong> Limit the number of documents returned per
|
||||
search (1-10,000)
|
||||
</li>
|
||||
</ul>
|
||||
</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem value="troubleshooting">
|
||||
<AccordionTrigger className="text-lg font-medium">
|
||||
Troubleshooting
|
||||
</AccordionTrigger>
|
||||
<AccordionContent className="space-y-4">
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h4 className="font-medium mb-2">Common Connection Issues:</h4>
|
||||
<ul className="list-disc pl-5 space-y-2 text-muted-foreground">
|
||||
<li>
|
||||
<strong>Connection Refused:</strong> Check hostname and port. Ensure
|
||||
Elasticsearch is running.
|
||||
</li>
|
||||
<li>
|
||||
<strong>Authentication Failed:</strong> Verify credentials. For API
|
||||
keys, ensure they have proper permissions.
|
||||
</li>
|
||||
<li>
|
||||
<strong>SSL Errors:</strong> Try disabling SSL for local development
|
||||
or check certificate validity.
|
||||
</li>
|
||||
<li>
|
||||
<strong>No Indices Found:</strong> Ensure your credentials have
|
||||
permission to list and read indices.
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<Alert className="bg-muted">
|
||||
<Info className="h-4 w-4" />
|
||||
<AlertTitle>Security Note</AlertTitle>
|
||||
<AlertDescription>
|
||||
For production environments, use API keys with minimal required
|
||||
permissions: cluster monitoring and read access to specific indices.
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
</div>
|
||||
</AccordionContent>
|
||||
</AccordionItem>
|
||||
</Accordion>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</motion.div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -52,6 +52,13 @@ const connectorCategories: ConnectorCategory[] = [
|
|||
icon: getConnectorIcon(EnumConnectorName.LINKUP_API, "h-6 w-6"),
|
||||
status: "available",
|
||||
},
|
||||
{
|
||||
id: "elasticsearch-connector",
|
||||
title: "Elasticsearch",
|
||||
description: "Connect to Elasticsearch to index and search documents, logs and metrics.",
|
||||
icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"),
|
||||
status: "available",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ export function DashboardBreadcrumb() {
|
|||
"serper-api": "Serper API",
|
||||
"linkup-api": "LinkUp API",
|
||||
"luma-connector": "Luma",
|
||||
"elasticsearch-connector": "Elasticsearch",
|
||||
};
|
||||
|
||||
const connectorLabel = connectorLabels[connectorType] || connectorType;
|
||||
|
|
|
|||
|
|
@ -44,5 +44,6 @@ export const editConnectorSchema = z.object({
|
|||
GOOGLE_CALENDAR_REFRESH_TOKEN: z.string().optional(),
|
||||
GOOGLE_CALENDAR_CALENDAR_IDS: z.string().optional(),
|
||||
LUMA_API_KEY: z.string().optional(),
|
||||
ELASTICSEARCH_API_KEY: z.string().optional(),
|
||||
});
|
||||
export type EditConnectorFormValues = z.infer<typeof editConnectorSchema>;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const INTEGRATIONS: Integration[] = [
|
|||
name: "LinkUp",
|
||||
icon: "https://framerusercontent.com/images/7zeIm6t3f1HaSltkw8upEvsD80.png?scale-down-to=512",
|
||||
},
|
||||
{ name: "Elasticsearch", icon: "https://cdn.simpleicons.org/elastic/00A9E5" },
|
||||
|
||||
// Communication
|
||||
{ name: "Slack", icon: "https://cdn.simpleicons.org/slack/4A154B" },
|
||||
|
|
|
|||
|
|
@ -14,4 +14,5 @@ export enum EnumConnectorName {
|
|||
GOOGLE_GMAIL_CONNECTOR = "GOOGLE_GMAIL_CONNECTOR",
|
||||
AIRTABLE_CONNECTOR = "AIRTABLE_CONNECTOR",
|
||||
LUMA_CONNECTOR = "LUMA_CONNECTOR",
|
||||
ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import {
|
||||
IconBook,
|
||||
IconBrandDiscord,
|
||||
IconBrandElastic,
|
||||
IconBrandGithub,
|
||||
IconBrandNotion,
|
||||
IconBrandSlack,
|
||||
|
|
@ -52,6 +53,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas
|
|||
return <IconChecklist {...iconProps} />;
|
||||
case EnumConnectorName.LUMA_CONNECTOR:
|
||||
return <IconSparkles {...iconProps} />;
|
||||
case EnumConnectorName.ELASTICSEARCH_CONNECTOR:
|
||||
return <IconBrandElastic {...iconProps} />;
|
||||
// Additional cases for non-enum connector types
|
||||
case "YOUTUBE_VIDEO":
|
||||
return <IconBrandYoutube {...iconProps} />;
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
|
|||
JIRA_EMAIL: "",
|
||||
JIRA_API_TOKEN: "",
|
||||
LUMA_API_KEY: "",
|
||||
ELASTICSEARCH_API_KEY: "",
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -84,6 +85,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
|
|||
JIRA_EMAIL: config.JIRA_EMAIL || "",
|
||||
JIRA_API_TOKEN: config.JIRA_API_TOKEN || "",
|
||||
LUMA_API_KEY: config.LUMA_API_KEY || "",
|
||||
ELASTICSEARCH_API_KEY: config.ELASTICSEARCH_API_KEY || "",
|
||||
});
|
||||
if (currentConnector.connector_type === "GITHUB_CONNECTOR") {
|
||||
const savedRepos = config.repo_full_names || [];
|
||||
|
|
@ -319,6 +321,16 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
|
|||
newConfig = { LUMA_API_KEY: formData.LUMA_API_KEY };
|
||||
}
|
||||
break;
|
||||
case "ELASTICSEARCH_CONNECTOR":
|
||||
if (formData.ELASTICSEARCH_API_KEY !== originalConfig.ELASTICSEARCH_API_KEY) {
|
||||
if (!formData.ELASTICSEARCH_API_KEY) {
|
||||
toast.error("Elasticsearch API Key cannot be empty.");
|
||||
setIsSaving(false);
|
||||
return;
|
||||
}
|
||||
newConfig = { ELASTICSEARCH_API_KEY: formData.ELASTICSEARCH_API_KEY };
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (newConfig !== null) {
|
||||
|
|
@ -383,6 +395,11 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
|
|||
editForm.setValue("JIRA_API_TOKEN", newlySavedConfig.JIRA_API_TOKEN || "");
|
||||
} else if (connector.connector_type === "LUMA_CONNECTOR") {
|
||||
editForm.setValue("LUMA_API_KEY", newlySavedConfig.LUMA_API_KEY || "");
|
||||
} else if (connector.connector_type === "ELASTICSEARCH_CONNECTOR") {
|
||||
editForm.setValue(
|
||||
"ELASTICSEARCH_API_KEY",
|
||||
newlySavedConfig.ELASTICSEARCH_API_KEY || ""
|
||||
);
|
||||
}
|
||||
}
|
||||
if (connector.connector_type === "GITHUB_CONNECTOR") {
|
||||
|
|
|
|||
|
|
@ -35,7 +35,8 @@ export type DocumentType =
|
|||
| "CLICKUP_CONNECTOR"
|
||||
| "GOOGLE_CALENDAR_CONNECTOR"
|
||||
| "GOOGLE_GMAIL_CONNECTOR"
|
||||
| "LUMA_CONNECTOR";
|
||||
| "LUMA_CONNECTOR"
|
||||
| "ELASTICSEARCH_CONNECTOR";
|
||||
|
||||
export function useDocumentByChunk() {
|
||||
const [document, setDocument] = useState<DocumentWithChunks | null>(null);
|
||||
|
|
|
|||
|
|
@ -29,7 +29,8 @@ export type DocumentType =
|
|||
| "GOOGLE_CALENDAR_CONNECTOR"
|
||||
| "GOOGLE_GMAIL_CONNECTOR"
|
||||
| "AIRTABLE_CONNECTOR"
|
||||
| "LUMA_CONNECTOR";
|
||||
| "LUMA_CONNECTOR"
|
||||
| "ELASTICSEARCH_CONNECTOR";
|
||||
|
||||
export interface UseDocumentsOptions {
|
||||
page?: number;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ export const getConnectorTypeDisplay = (type: string): string => {
|
|||
GOOGLE_GMAIL_CONNECTOR: "Google Gmail",
|
||||
AIRTABLE_CONNECTOR: "Airtable",
|
||||
LUMA_CONNECTOR: "Luma",
|
||||
ELASTICSEARCH_CONNECTOR: "Elasticsearch",
|
||||
};
|
||||
return typeMap[type] || type;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@
|
|||
"@radix-ui/react-dropdown-menu": "^2.1.15",
|
||||
"@radix-ui/react-label": "^2.1.7",
|
||||
"@radix-ui/react-popover": "^1.1.14",
|
||||
"@radix-ui/react-radio-group": "^1.3.8",
|
||||
"@radix-ui/react-scroll-area": "^1.2.9",
|
||||
"@radix-ui/react-select": "^2.2.5",
|
||||
"@radix-ui/react-separator": "^1.1.7",
|
||||
|
|
|
|||
101
surfsense_web/pnpm-lock.yaml
generated
101
surfsense_web/pnpm-lock.yaml
generated
|
|
@ -47,6 +47,9 @@ importers:
|
|||
'@radix-ui/react-popover':
|
||||
specifier: ^1.1.14
|
||||
version: 1.1.14(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
|
||||
'@radix-ui/react-radio-group':
|
||||
specifier: ^1.3.8
|
||||
version: 1.3.8(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
|
||||
'@radix-ui/react-scroll-area':
|
||||
specifier: ^1.2.9
|
||||
version: 1.2.9(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
|
||||
|
|
@ -118,7 +121,7 @@ importers:
|
|||
version: 15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
|
||||
fumadocs-mdx:
|
||||
specifier: ^11.7.1
|
||||
version: 11.7.1(acorn@8.14.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0)
|
||||
version: 11.7.1(acorn@8.15.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0)
|
||||
fumadocs-ui:
|
||||
specifier: ^15.6.6
|
||||
version: 15.6.6(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(tailwindcss@4.1.11)
|
||||
|
|
@ -1735,6 +1738,19 @@ packages:
|
|||
'@types/react-dom':
|
||||
optional: true
|
||||
|
||||
'@radix-ui/react-presence@1.1.5':
|
||||
resolution: {integrity: sha512-/jfEwNDdQVBCNvjkGit4h6pMOzq8bHkopq458dPt2lMjx+eBQUohZNG9A7DtO/O5ukSbxuaNGXMjHicgwy6rQQ==}
|
||||
peerDependencies:
|
||||
'@types/react': '*'
|
||||
'@types/react-dom': '*'
|
||||
react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
|
||||
react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
|
||||
peerDependenciesMeta:
|
||||
'@types/react':
|
||||
optional: true
|
||||
'@types/react-dom':
|
||||
optional: true
|
||||
|
||||
'@radix-ui/react-primitive@1.0.0':
|
||||
resolution: {integrity: sha512-EyXe6mnRlHZ8b6f4ilTDrXmkLShICIuOTTj0GX4w1rp+wSxf3+TD05u1UOITC8VsJ2a9nwHvdXtOXEOl0Cw/zQ==}
|
||||
peerDependencies:
|
||||
|
|
@ -1793,6 +1809,19 @@ packages:
|
|||
'@types/react-dom':
|
||||
optional: true
|
||||
|
||||
'@radix-ui/react-radio-group@1.3.8':
|
||||
resolution: {integrity: sha512-VBKYIYImA5zsxACdisNQ3BjCBfmbGH3kQlnFVqlWU4tXwjy7cGX8ta80BcrO+WJXIn5iBylEH3K6ZTlee//lgQ==}
|
||||
peerDependencies:
|
||||
'@types/react': '*'
|
||||
'@types/react-dom': '*'
|
||||
react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
|
||||
react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
|
||||
peerDependenciesMeta:
|
||||
'@types/react':
|
||||
optional: true
|
||||
'@types/react-dom':
|
||||
optional: true
|
||||
|
||||
'@radix-ui/react-roving-focus@1.1.10':
|
||||
resolution: {integrity: sha512-dT9aOXUen9JSsxnMPv/0VqySQf5eDQ6LCk5Sw28kamz8wSOW2bJdlX2Bg5VUIIcV+6XlHpWTIuTPCf/UNIyq8Q==}
|
||||
peerDependencies:
|
||||
|
|
@ -1806,6 +1835,19 @@ packages:
|
|||
'@types/react-dom':
|
||||
optional: true
|
||||
|
||||
'@radix-ui/react-roving-focus@1.1.11':
|
||||
resolution: {integrity: sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA==}
|
||||
peerDependencies:
|
||||
'@types/react': '*'
|
||||
'@types/react-dom': '*'
|
||||
react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
|
||||
react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
|
||||
peerDependenciesMeta:
|
||||
'@types/react':
|
||||
optional: true
|
||||
'@types/react-dom':
|
||||
optional: true
|
||||
|
||||
'@radix-ui/react-scroll-area@1.2.9':
|
||||
resolution: {integrity: sha512-YSjEfBXnhUELsO2VzjdtYYD4CfQjvao+lhhrX5XsHD7/cyUNzljF1FHEbgTPN7LH2MClfwRMIsYlqTYpKTTe2A==}
|
||||
peerDependencies:
|
||||
|
|
@ -6554,7 +6596,7 @@ snapshots:
|
|||
|
||||
'@marijn/find-cluster-break@1.0.2': {}
|
||||
|
||||
'@mdx-js/mdx@3.1.0(acorn@8.14.0)':
|
||||
'@mdx-js/mdx@3.1.0(acorn@8.15.0)':
|
||||
dependencies:
|
||||
'@types/estree': 1.0.8
|
||||
'@types/estree-jsx': 1.0.5
|
||||
|
|
@ -6568,7 +6610,7 @@ snapshots:
|
|||
hast-util-to-jsx-runtime: 2.3.6
|
||||
markdown-extensions: 2.0.0
|
||||
recma-build-jsx: 1.0.0
|
||||
recma-jsx: 1.0.0(acorn@8.14.0)
|
||||
recma-jsx: 1.0.0(acorn@8.15.0)
|
||||
recma-stringify: 1.0.0
|
||||
rehype-recma: 1.0.0
|
||||
remark-mdx: 3.1.0
|
||||
|
|
@ -7260,6 +7302,16 @@ snapshots:
|
|||
'@types/react': 19.1.8
|
||||
'@types/react-dom': 19.1.6(@types/react@19.1.8)
|
||||
|
||||
'@radix-ui/react-presence@1.1.5(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
|
||||
dependencies:
|
||||
'@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.1.8)(react@19.1.0)
|
||||
react: 19.1.0
|
||||
react-dom: 19.1.0(react@19.1.0)
|
||||
optionalDependencies:
|
||||
'@types/react': 19.1.8
|
||||
'@types/react-dom': 19.1.6(@types/react@19.1.8)
|
||||
|
||||
'@radix-ui/react-primitive@1.0.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
|
||||
dependencies:
|
||||
'@babel/runtime': 7.26.9
|
||||
|
|
@ -7305,6 +7357,24 @@ snapshots:
|
|||
'@types/react': 19.1.8
|
||||
'@types/react-dom': 19.1.6(@types/react@19.1.8)
|
||||
|
||||
'@radix-ui/react-radio-group@1.3.8(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
|
||||
dependencies:
|
||||
'@radix-ui/primitive': 1.1.3
|
||||
'@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-context': 1.1.2(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-direction': 1.1.1(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-presence': 1.1.5(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
|
||||
'@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
|
||||
'@radix-ui/react-roving-focus': 1.1.11(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
|
||||
'@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-use-previous': 1.1.1(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-use-size': 1.1.1(@types/react@19.1.8)(react@19.1.0)
|
||||
react: 19.1.0
|
||||
react-dom: 19.1.0(react@19.1.0)
|
||||
optionalDependencies:
|
||||
'@types/react': 19.1.8
|
||||
'@types/react-dom': 19.1.6(@types/react@19.1.8)
|
||||
|
||||
'@radix-ui/react-roving-focus@1.1.10(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
|
||||
dependencies:
|
||||
'@radix-ui/primitive': 1.1.2
|
||||
|
|
@ -7322,6 +7392,23 @@ snapshots:
|
|||
'@types/react': 19.1.8
|
||||
'@types/react-dom': 19.1.6(@types/react@19.1.8)
|
||||
|
||||
'@radix-ui/react-roving-focus@1.1.11(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
|
||||
dependencies:
|
||||
'@radix-ui/primitive': 1.1.3
|
||||
'@radix-ui/react-collection': 1.1.7(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
|
||||
'@radix-ui/react-compose-refs': 1.1.2(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-context': 1.1.2(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-direction': 1.1.1(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-id': 1.1.1(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
|
||||
'@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.1.8)(react@19.1.0)
|
||||
'@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.1.8)(react@19.1.0)
|
||||
react: 19.1.0
|
||||
react-dom: 19.1.0(react@19.1.0)
|
||||
optionalDependencies:
|
||||
'@types/react': 19.1.8
|
||||
'@types/react-dom': 19.1.6(@types/react@19.1.8)
|
||||
|
||||
'@radix-ui/react-scroll-area@1.2.9(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)':
|
||||
dependencies:
|
||||
'@radix-ui/number': 1.1.1
|
||||
|
|
@ -9073,9 +9160,9 @@ snapshots:
|
|||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
fumadocs-mdx@11.7.1(acorn@8.14.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0):
|
||||
fumadocs-mdx@11.7.1(acorn@8.15.0)(fumadocs-core@15.6.6(@types/react@19.1.8)(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(next@15.4.4(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react@19.1.0):
|
||||
dependencies:
|
||||
'@mdx-js/mdx': 3.1.0(acorn@8.14.0)
|
||||
'@mdx-js/mdx': 3.1.0(acorn@8.15.0)
|
||||
'@standard-schema/spec': 1.0.0
|
||||
chokidar: 4.0.3
|
||||
esbuild: 0.25.8
|
||||
|
|
@ -11137,9 +11224,9 @@ snapshots:
|
|||
estree-util-build-jsx: 3.0.1
|
||||
vfile: 6.0.3
|
||||
|
||||
recma-jsx@1.0.0(acorn@8.14.0):
|
||||
recma-jsx@1.0.0(acorn@8.15.0):
|
||||
dependencies:
|
||||
acorn-jsx: 5.3.2(acorn@8.14.0)
|
||||
acorn-jsx: 5.3.2(acorn@8.15.0)
|
||||
estree-util-to-js: 2.0.0
|
||||
recma-parse: 1.0.0
|
||||
recma-stringify: 1.0.0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue