mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-06-08 20:25:19 +02:00
Merge remote-tracking branch 'upstream/dev' into feat/composio
This commit is contained in:
Anish Sarkar
commit
fae52345f8
65 changed files with 3291 additions and 153 deletions
|
|
@ -45,6 +45,8 @@ services:
|
|||
volumes:
|
||||
- ./surfsense_backend/app:/app/app
|
||||
- shared_temp:/tmp
|
||||
# Uncomment and edit the line below to enable Obsidian vault indexing
|
||||
# - /path/to/your/obsidian/vault:/obsidian-vault:ro
|
||||
env_file:
|
||||
- ./surfsense_backend/.env
|
||||
environment:
|
||||
|
|
|
|||
29
surfsense_backend/alembic/versions/74_no_op.py
Normal file
29
surfsense_backend/alembic/versions/74_no_op.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
"""No-op migration for Composio support
|
||||
|
||||
Revision ID: 74
|
||||
Revises: 73
|
||||
Create Date: 2026-01-21
|
||||
|
||||
NOTE: This migration is a no-op since Composio is not supported yet.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "74"
|
||||
down_revision: str | None = "73"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""No-op upgrade for Composio support."""
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""No-op downgrade for Composio support.
|
||||
|
||||
Note: PostgreSQL does not support removing enum values directly.
|
||||
"""
|
||||
pass
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
"""Add chat_session_state table for live collaboration
|
||||
|
||||
Revision ID: 75
|
||||
Revises: 74
|
||||
|
||||
Creates chat_session_state table to track AI responding state per thread.
|
||||
Enables real-time sync via Electric SQL for shared chat collaboration.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "75"
|
||||
down_revision: str | None = "74"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Create chat_session_state table with Electric SQL replication."""
|
||||
op.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS chat_session_state (
|
||||
id SERIAL PRIMARY KEY,
|
||||
thread_id INTEGER NOT NULL REFERENCES new_chat_threads(id) ON DELETE CASCADE,
|
||||
ai_responding_to_user_id UUID REFERENCES "user"(id) ON DELETE SET NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (thread_id)
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
op.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_chat_session_state_thread_id ON chat_session_state(thread_id)"
|
||||
)
|
||||
|
||||
op.execute("ALTER TABLE chat_session_state REPLICA IDENTITY FULL;")
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_publication_tables
|
||||
WHERE pubname = 'electric_publication_default'
|
||||
AND tablename = 'chat_session_state'
|
||||
) THEN
|
||||
ALTER PUBLICATION electric_publication_default ADD TABLE chat_session_state;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Drop chat_session_state table and remove from Electric SQL replication."""
|
||||
op.execute(
|
||||
"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_publication_tables
|
||||
WHERE pubname = 'electric_publication_default'
|
||||
AND tablename = 'chat_session_state'
|
||||
) THEN
|
||||
ALTER PUBLICATION electric_publication_default DROP TABLE chat_session_state;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
"""
|
||||
)
|
||||
|
||||
op.execute("DROP TABLE IF EXISTS chat_session_state;")
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
"""Add live collaboration tables to Electric SQL publication
|
||||
|
||||
Revision ID: 76
|
||||
Revises: 75
|
||||
|
||||
Enables real-time sync for live collaboration features:
|
||||
- new_chat_messages: Live message sync between users
|
||||
- chat_comments: Live comment updates
|
||||
|
||||
Note: User/member info is fetched via API (membersAtom) for client-side joins,
|
||||
not via Electric SQL, to keep where clauses optimized and reduce complexity.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "76"
|
||||
down_revision: str | None = "75"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add live collaboration tables to Electric SQL replication."""
|
||||
# Set REPLICA IDENTITY FULL for Electric SQL sync
|
||||
op.execute("ALTER TABLE new_chat_messages REPLICA IDENTITY FULL;")
|
||||
op.execute("ALTER TABLE chat_comments REPLICA IDENTITY FULL;")
|
||||
|
||||
# Add new_chat_messages to Electric publication
|
||||
op.execute(
|
||||
"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_publication_tables
|
||||
WHERE pubname = 'electric_publication_default'
|
||||
AND tablename = 'new_chat_messages'
|
||||
) THEN
|
||||
ALTER PUBLICATION electric_publication_default ADD TABLE new_chat_messages;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
"""
|
||||
)
|
||||
|
||||
# Add chat_comments to Electric publication
|
||||
op.execute(
|
||||
"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_publication_tables
|
||||
WHERE pubname = 'electric_publication_default'
|
||||
AND tablename = 'chat_comments'
|
||||
) THEN
|
||||
ALTER PUBLICATION electric_publication_default ADD TABLE chat_comments;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove live collaboration tables from Electric SQL replication."""
|
||||
op.execute(
|
||||
"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_publication_tables
|
||||
WHERE pubname = 'electric_publication_default'
|
||||
AND tablename = 'new_chat_messages'
|
||||
) THEN
|
||||
ALTER PUBLICATION electric_publication_default DROP TABLE new_chat_messages;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
"""
|
||||
)
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_publication_tables
|
||||
WHERE pubname = 'electric_publication_default'
|
||||
AND tablename = 'chat_comments'
|
||||
) THEN
|
||||
ALTER PUBLICATION electric_publication_default DROP TABLE chat_comments;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
"""
|
||||
)
|
||||
|
||||
# Note: Not reverting REPLICA IDENTITY as it doesn't harm normal operations
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
"""Add thread_id to chat_comments for denormalized Electric subscriptions
|
||||
|
||||
This denormalization allows a single Electric SQL subscription per thread
|
||||
instead of one per message, significantly reducing connection overhead.
|
||||
|
||||
Revision ID: 77
|
||||
Revises: 76
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "77"
|
||||
down_revision: str | None = "76"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add thread_id column to chat_comments and backfill from messages."""
|
||||
# Add the column (nullable initially for backfill)
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE chat_comments
|
||||
ADD COLUMN IF NOT EXISTS thread_id INTEGER;
|
||||
"""
|
||||
)
|
||||
|
||||
# Backfill thread_id from the related message
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE chat_comments c
|
||||
SET thread_id = m.thread_id
|
||||
FROM new_chat_messages m
|
||||
WHERE c.message_id = m.id
|
||||
AND c.thread_id IS NULL;
|
||||
"""
|
||||
)
|
||||
|
||||
# Make it NOT NULL after backfill
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE chat_comments
|
||||
ALTER COLUMN thread_id SET NOT NULL;
|
||||
"""
|
||||
)
|
||||
|
||||
# Add FK constraint
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE chat_comments
|
||||
ADD CONSTRAINT fk_chat_comments_thread_id
|
||||
FOREIGN KEY (thread_id) REFERENCES new_chat_threads(id) ON DELETE CASCADE;
|
||||
"""
|
||||
)
|
||||
|
||||
# Add index for efficient Electric subscriptions by thread
|
||||
op.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_chat_comments_thread_id ON chat_comments(thread_id)"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove thread_id column from chat_comments."""
|
||||
op.execute("DROP INDEX IF EXISTS idx_chat_comments_thread_id")
|
||||
op.execute(
|
||||
"ALTER TABLE chat_comments DROP CONSTRAINT IF EXISTS fk_chat_comments_thread_id"
|
||||
)
|
||||
op.execute("ALTER TABLE chat_comments DROP COLUMN IF EXISTS thread_id")
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
"""Add Obsidian connector enums
|
||||
|
||||
Revision ID: 78
|
||||
Revises: 77
|
||||
Create Date: 2026-01-21
|
||||
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "78"
|
||||
down_revision: str | None = "77"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Add OBSIDIAN_CONNECTOR to documenttype enum
|
||||
op.execute("ALTER TYPE documenttype ADD VALUE IF NOT EXISTS 'OBSIDIAN_CONNECTOR'")
|
||||
|
||||
# Add OBSIDIAN_CONNECTOR to searchsourceconnectortype enum
|
||||
op.execute(
|
||||
"ALTER TYPE searchsourceconnectortype ADD VALUE IF NOT EXISTS 'OBSIDIAN_CONNECTOR'"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Note: PostgreSQL doesn't support removing enum values directly.
|
||||
# The values will remain in the enum type but won't be used.
|
||||
pass
|
||||
|
|
@ -1,8 +1,7 @@
|
|||
"""Add Composio connector types to SearchSourceConnectorType and DocumentType enums
|
||||
|
||||
Revision ID: 74
|
||||
Revises: 73
|
||||
Create Date: 2026-01-21
|
||||
Revision ID: 79
|
||||
Revises: 78
|
||||
|
||||
This migration adds the Composio connector enum values to both:
|
||||
- searchsourceconnectortype (for connector type tracking)
|
||||
|
|
@ -23,8 +22,8 @@ from collections.abc import Sequence
|
|||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "74"
|
||||
down_revision: str | None = "73"
|
||||
revision: str = "79"
|
||||
down_revision: str | None = "78"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
|
@ -34,6 +34,12 @@ You have access to the following tools:
|
|||
- Returns: Documentation content with chunk IDs for citations (prefixed with 'doc-', e.g., [citation:doc-123])
|
||||
|
||||
1. search_knowledge_base: Search the user's personal knowledge base for relevant information.
|
||||
- IMPORTANT: When searching for information (meetings, schedules, notes, tasks, etc.), ALWAYS search broadly
|
||||
across ALL sources first by omitting connectors_to_search. The user may store information in various places
|
||||
including calendar apps, note-taking apps (Obsidian, Notion), chat apps (Slack, Discord), and more.
|
||||
- Only narrow to specific connectors if the user explicitly asks (e.g., "check my Slack" or "in my calendar").
|
||||
- Personal notes in Obsidian, Notion, or NOTE often contain schedules, meeting times, reminders, and other
|
||||
important information that may not be in calendars.
|
||||
- Args:
|
||||
- query: The search query - be specific and include key terms
|
||||
- top_k: Number of results to retrieve (default: 10)
|
||||
|
|
@ -157,6 +163,13 @@ You have access to the following tools:
|
|||
stating "Based on your memory..." - integrate the context seamlessly.
|
||||
</tools>
|
||||
<tool_call_examples>
|
||||
- User: "What time is the team meeting today?"
|
||||
- Call: `search_knowledge_base(query="team meeting time today")` (searches ALL sources - calendar, notes, Obsidian, etc.)
|
||||
- DO NOT limit to just calendar - the info might be in notes!
|
||||
|
||||
- User: "When is my gym session?"
|
||||
- Call: `search_knowledge_base(query="gym session time schedule")` (searches ALL sources)
|
||||
|
||||
- User: "How do I install SurfSense?"
|
||||
- Call: `search_surfsense_docs(query="installation setup")`
|
||||
|
||||
|
|
@ -175,6 +188,12 @@ You have access to the following tools:
|
|||
- User: "What did I discuss on Slack last week about the React migration?"
|
||||
- Call: `search_knowledge_base(query="React migration", connectors_to_search=["SLACK_CONNECTOR"], start_date="YYYY-MM-DD", end_date="YYYY-MM-DD")`
|
||||
|
||||
- User: "Check my Obsidian notes for meeting notes"
|
||||
- Call: `search_knowledge_base(query="meeting notes", connectors_to_search=["OBSIDIAN_CONNECTOR"])`
|
||||
|
||||
- User: "What's in my Obsidian vault about project ideas?"
|
||||
- Call: `search_knowledge_base(query="project ideas", connectors_to_search=["OBSIDIAN_CONNECTOR"])`
|
||||
|
||||
- User: "Remember that I prefer TypeScript over JavaScript"
|
||||
- Call: `save_memory(content="User prefers TypeScript over JavaScript for development", category="preference")`
|
||||
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ _ALL_CONNECTORS: list[str] = [
|
|||
"BOOKSTACK_CONNECTOR",
|
||||
"CRAWLED_URL",
|
||||
"CIRCLEBACK",
|
||||
"OBSIDIAN_CONNECTOR",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -508,6 +509,16 @@ async def search_knowledge_base_async(
|
|||
)
|
||||
all_documents.extend(chunks)
|
||||
|
||||
elif connector == "OBSIDIAN_CONNECTOR":
|
||||
_, chunks = await connector_service.search_obsidian(
|
||||
user_query=query,
|
||||
search_space_id=search_space_id,
|
||||
top_k=top_k,
|
||||
start_date=resolved_start_date,
|
||||
end_date=resolved_end_date,
|
||||
)
|
||||
all_documents.extend(chunks)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error searching connector {connector}: {e}")
|
||||
continue
|
||||
|
|
@ -596,6 +607,7 @@ def create_search_knowledge_base_tool(
|
|||
- WEBCRAWLER_CONNECTOR: "Webpages indexed by SurfSense" (personally selected websites)
|
||||
- BOOKSTACK_CONNECTOR: "BookStack pages" (personal documentation)
|
||||
- CIRCLEBACK: "Circleback meeting notes, transcripts, and action items" (personal meeting records)
|
||||
- OBSIDIAN_CONNECTOR: "Obsidian vault notes and markdown files" (personal notes and knowledge management)
|
||||
|
||||
NOTE: `WEBCRAWLER_CONNECTOR` is mapped internally to the canonical document type `CRAWLED_URL`.
|
||||
|
||||
|
|
|
|||
|
|
@ -61,6 +61,21 @@ class Config:
|
|||
"FFmpeg is not installed on the system. Please install it to use the Surfsense Podcaster."
|
||||
)
|
||||
|
||||
# Deployment Mode (self-hosted or cloud)
|
||||
# self-hosted: Full access to local file system connectors (Obsidian, etc.)
|
||||
# cloud: Only cloud-based connectors available
|
||||
DEPLOYMENT_MODE = os.getenv("SURFSENSE_DEPLOYMENT_MODE", "self-hosted")
|
||||
|
||||
@classmethod
|
||||
def is_self_hosted(cls) -> bool:
|
||||
"""Check if running in self-hosted mode."""
|
||||
return cls.DEPLOYMENT_MODE == "self-hosted"
|
||||
|
||||
@classmethod
|
||||
def is_cloud(cls) -> bool:
|
||||
"""Check if running in cloud mode."""
|
||||
return cls.DEPLOYMENT_MODE == "cloud"
|
||||
|
||||
# Database
|
||||
DATABASE_URL = os.getenv("DATABASE_URL")
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ class DocumentType(str, Enum):
|
|||
ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR"
|
||||
BOOKSTACK_CONNECTOR = "BOOKSTACK_CONNECTOR"
|
||||
CIRCLEBACK = "CIRCLEBACK"
|
||||
OBSIDIAN_CONNECTOR = "OBSIDIAN_CONNECTOR"
|
||||
NOTE = "NOTE"
|
||||
COMPOSIO_GOOGLE_DRIVE_CONNECTOR = "COMPOSIO_GOOGLE_DRIVE_CONNECTOR"
|
||||
COMPOSIO_GMAIL_CONNECTOR = "COMPOSIO_GMAIL_CONNECTOR"
|
||||
|
|
@ -83,6 +84,9 @@ class SearchSourceConnectorType(str, Enum):
|
|||
WEBCRAWLER_CONNECTOR = "WEBCRAWLER_CONNECTOR"
|
||||
BOOKSTACK_CONNECTOR = "BOOKSTACK_CONNECTOR"
|
||||
CIRCLEBACK_CONNECTOR = "CIRCLEBACK_CONNECTOR"
|
||||
OBSIDIAN_CONNECTOR = (
|
||||
"OBSIDIAN_CONNECTOR" # Self-hosted only - Local Obsidian vault indexing
|
||||
)
|
||||
MCP_CONNECTOR = "MCP_CONNECTOR" # Model Context Protocol - User-defined API tools
|
||||
COMPOSIO_GOOGLE_DRIVE_CONNECTOR = "COMPOSIO_GOOGLE_DRIVE_CONNECTOR"
|
||||
COMPOSIO_GMAIL_CONNECTOR = "COMPOSIO_GMAIL_CONNECTOR"
|
||||
|
|
@ -419,6 +423,13 @@ class ChatComment(BaseModel, TimestampMixin):
|
|||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
# Denormalized thread_id for efficient Electric SQL subscriptions (one per thread)
|
||||
thread_id = Column(
|
||||
Integer,
|
||||
ForeignKey("new_chat_threads.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
parent_id = Column(
|
||||
Integer,
|
||||
ForeignKey("chat_comments.id", ondelete="CASCADE"),
|
||||
|
|
@ -442,6 +453,7 @@ class ChatComment(BaseModel, TimestampMixin):
|
|||
|
||||
# Relationships
|
||||
message = relationship("NewChatMessage", back_populates="comments")
|
||||
thread = relationship("NewChatThread")
|
||||
author = relationship("User")
|
||||
parent = relationship(
|
||||
"ChatComment", remote_side="ChatComment.id", backref="replies"
|
||||
|
|
@ -478,6 +490,38 @@ class ChatCommentMention(BaseModel, TimestampMixin):
|
|||
mentioned_user = relationship("User")
|
||||
|
||||
|
||||
class ChatSessionState(BaseModel):
|
||||
"""
|
||||
Tracks real-time session state for shared chat collaboration.
|
||||
One record per thread, synced via Electric SQL.
|
||||
"""
|
||||
|
||||
__tablename__ = "chat_session_state"
|
||||
|
||||
thread_id = Column(
|
||||
Integer,
|
||||
ForeignKey("new_chat_threads.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
index=True,
|
||||
)
|
||||
ai_responding_to_user_id = Column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("user.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
updated_at = Column(
|
||||
TIMESTAMP(timezone=True),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
thread = relationship("NewChatThread")
|
||||
ai_responding_to_user = relationship("User")
|
||||
|
||||
|
||||
class MemoryCategory(str, Enum):
|
||||
"""Categories for user memories."""
|
||||
|
||||
|
|
|
|||
|
|
@ -990,7 +990,7 @@ async def handle_new_chat(
|
|||
search_space_id=request.search_space_id,
|
||||
chat_id=request.chat_id,
|
||||
session=session,
|
||||
user_id=str(user.id), # Pass user ID for memory tools
|
||||
user_id=str(user.id), # Pass user ID for memory tools and session state
|
||||
llm_config_id=llm_config_id,
|
||||
attachments=request.attachments,
|
||||
mentioned_document_ids=request.mentioned_document_ids,
|
||||
|
|
|
|||
|
|
@ -901,6 +901,25 @@ async def index_connector_content(
|
|||
)
|
||||
response_message = "Web page indexing started in the background."
|
||||
|
||||
elif connector.connector_type == SearchSourceConnectorType.OBSIDIAN_CONNECTOR:
|
||||
from app.config import config as app_config
|
||||
from app.tasks.celery_tasks.connector_tasks import index_obsidian_vault_task
|
||||
|
||||
# Obsidian connector only available in self-hosted mode
|
||||
if not app_config.is_self_hosted():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Obsidian connector is only available in self-hosted mode",
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Triggering Obsidian vault indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}"
|
||||
)
|
||||
index_obsidian_vault_task.delay(
|
||||
connector_id, search_space_id, str(user.id), indexing_from, indexing_to
|
||||
)
|
||||
response_message = "Obsidian vault indexing started in the background."
|
||||
|
||||
elif (
|
||||
connector.connector_type
|
||||
== SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR
|
||||
|
|
@ -2195,6 +2214,58 @@ async def run_bookstack_indexing(
|
|||
)
|
||||
|
||||
|
||||
# Add new helper functions for Obsidian indexing
|
||||
async def run_obsidian_indexing_with_new_session(
|
||||
connector_id: int,
|
||||
search_space_id: int,
|
||||
user_id: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
):
|
||||
"""Wrapper to run Obsidian indexing with its own database session."""
|
||||
logger.info(
|
||||
f"Background task started: Indexing Obsidian connector {connector_id} into space {search_space_id} from {start_date} to {end_date}"
|
||||
)
|
||||
async with async_session_maker() as session:
|
||||
await run_obsidian_indexing(
|
||||
session, connector_id, search_space_id, user_id, start_date, end_date
|
||||
)
|
||||
logger.info(f"Background task finished: Indexing Obsidian connector {connector_id}")
|
||||
|
||||
|
||||
async def run_obsidian_indexing(
|
||||
session: AsyncSession,
|
||||
connector_id: int,
|
||||
search_space_id: int,
|
||||
user_id: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
):
|
||||
"""
|
||||
Background task to run Obsidian vault indexing.
|
||||
|
||||
Args:
|
||||
session: Database session
|
||||
connector_id: ID of the Obsidian connector
|
||||
search_space_id: ID of the search space
|
||||
user_id: ID of the user
|
||||
start_date: Start date for indexing
|
||||
end_date: End date for indexing
|
||||
"""
|
||||
from app.tasks.connector_indexers import index_obsidian_vault
|
||||
|
||||
await _run_indexing_with_notifications(
|
||||
session=session,
|
||||
connector_id=connector_id,
|
||||
search_space_id=search_space_id,
|
||||
user_id=user_id,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
indexing_function=index_obsidian_vault,
|
||||
update_timestamp_func=_update_connector_timestamp_by_id,
|
||||
)
|
||||
|
||||
|
||||
async def run_composio_indexing_with_new_session(
|
||||
connector_id: int,
|
||||
search_space_id: int,
|
||||
|
|
|
|||
29
surfsense_backend/app/schemas/chat_session_state.py
Normal file
29
surfsense_backend/app/schemas/chat_session_state.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
"""
|
||||
Pydantic schemas for chat session state (live collaboration).
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class RespondingUser(BaseModel):
|
||||
"""The user that the AI is currently responding to."""
|
||||
|
||||
id: UUID
|
||||
display_name: str | None = None
|
||||
email: str
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class ChatSessionStateResponse(BaseModel):
|
||||
"""Current session state for a chat thread."""
|
||||
|
||||
id: int
|
||||
thread_id: int
|
||||
responding_to: RespondingUser | None = None
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
59
surfsense_backend/app/schemas/obsidian_auth_credentials.py
Normal file
59
surfsense_backend/app/schemas/obsidian_auth_credentials.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
"""
|
||||
Obsidian Connector Credentials Schema.
|
||||
|
||||
Obsidian is a local-first note-taking app that stores notes as markdown files.
|
||||
This connector supports indexing from local file system (self-hosted only).
|
||||
"""
|
||||
|
||||
from pydantic import BaseModel, field_validator
|
||||
|
||||
|
||||
class ObsidianAuthCredentialsBase(BaseModel):
|
||||
"""
|
||||
Credentials/configuration for the Obsidian connector.
|
||||
|
||||
Since Obsidian vaults are local directories, this schema primarily
|
||||
holds the vault path and configuration options rather than API tokens.
|
||||
"""
|
||||
|
||||
vault_path: str
|
||||
vault_name: str | None = None
|
||||
exclude_folders: list[str] | None = None
|
||||
include_attachments: bool = False
|
||||
|
||||
@field_validator("vault_path")
|
||||
@classmethod
|
||||
def validate_vault_path(cls, v: str) -> str:
|
||||
"""Ensure vault path is provided and stripped of whitespace."""
|
||||
if not v or not v.strip():
|
||||
raise ValueError("Vault path is required")
|
||||
return v.strip()
|
||||
|
||||
@field_validator("exclude_folders", mode="before")
|
||||
@classmethod
|
||||
def parse_exclude_folders(cls, v):
|
||||
"""Parse exclude_folders from string if needed."""
|
||||
if v is None:
|
||||
return [".trash", ".obsidian", "templates"]
|
||||
if isinstance(v, str):
|
||||
return [f.strip() for f in v.split(",") if f.strip()]
|
||||
return v
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert credentials to dictionary for storage."""
|
||||
return {
|
||||
"vault_path": self.vault_path,
|
||||
"vault_name": self.vault_name,
|
||||
"exclude_folders": self.exclude_folders,
|
||||
"include_attachments": self.include_attachments,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "ObsidianAuthCredentialsBase":
|
||||
"""Create credentials from dictionary."""
|
||||
return cls(
|
||||
vault_path=data.get("vault_path", ""),
|
||||
vault_name=data.get("vault_name"),
|
||||
exclude_folders=data.get("exclude_folders"),
|
||||
include_attachments=data.get("include_attachments", False),
|
||||
)
|
||||
|
|
@ -281,8 +281,10 @@ async def create_comment(
|
|||
detail="You don't have permission to create comments in this search space",
|
||||
)
|
||||
|
||||
thread = message.thread
|
||||
comment = ChatComment(
|
||||
message_id=message_id,
|
||||
thread_id=thread.id, # Denormalized for efficient Electric subscriptions
|
||||
author_id=user.id,
|
||||
content=content,
|
||||
)
|
||||
|
|
@ -299,7 +301,6 @@ async def create_comment(
|
|||
user_names = await get_user_names_for_mentions(session, set(mentions_map.keys()))
|
||||
|
||||
# Create notifications for mentioned users (excluding author)
|
||||
thread = message.thread
|
||||
author_name = user.display_name or user.email
|
||||
content_preview = render_mentions(content, user_names)
|
||||
for mentioned_user_id, mention_id in mentions_map.items():
|
||||
|
|
@ -393,8 +394,10 @@ async def create_reply(
|
|||
detail="You don't have permission to create comments in this search space",
|
||||
)
|
||||
|
||||
thread = parent_comment.message.thread
|
||||
reply = ChatComment(
|
||||
message_id=parent_comment.message_id,
|
||||
thread_id=thread.id, # Denormalized for efficient Electric subscriptions
|
||||
parent_id=comment_id,
|
||||
author_id=user.id,
|
||||
content=content,
|
||||
|
|
@ -412,7 +415,6 @@ async def create_reply(
|
|||
user_names = await get_user_names_for_mentions(session, set(mentions_map.keys()))
|
||||
|
||||
# Create notifications for mentioned users (excluding author)
|
||||
thread = parent_comment.message.thread
|
||||
author_name = user.display_name or user.email
|
||||
content_preview = render_mentions(content, user_names)
|
||||
for mentioned_user_id, mention_id in mentions_map.items():
|
||||
|
|
|
|||
65
surfsense_backend/app/services/chat_session_state_service.py
Normal file
65
surfsense_backend/app/services/chat_session_state_service.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
"""
|
||||
Service layer for chat session state (live collaboration).
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.db import ChatSessionState
|
||||
|
||||
|
||||
async def get_session_state(
|
||||
session: AsyncSession,
|
||||
thread_id: int,
|
||||
) -> ChatSessionState | None:
|
||||
"""Get the current session state for a thread."""
|
||||
result = await session.execute(
|
||||
select(ChatSessionState)
|
||||
.options(selectinload(ChatSessionState.ai_responding_to_user))
|
||||
.filter(ChatSessionState.thread_id == thread_id)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def set_ai_responding(
|
||||
session: AsyncSession,
|
||||
thread_id: int,
|
||||
user_id: UUID,
|
||||
) -> ChatSessionState:
|
||||
"""Mark AI as responding to a specific user. Uses upsert for atomicity."""
|
||||
now = datetime.now(UTC)
|
||||
upsert_query = insert(ChatSessionState).values(
|
||||
thread_id=thread_id,
|
||||
ai_responding_to_user_id=user_id,
|
||||
updated_at=now,
|
||||
)
|
||||
upsert_query = upsert_query.on_conflict_do_update(
|
||||
index_elements=["thread_id"],
|
||||
set_={
|
||||
"ai_responding_to_user_id": user_id,
|
||||
"updated_at": now,
|
||||
},
|
||||
)
|
||||
await session.execute(upsert_query)
|
||||
await session.commit()
|
||||
|
||||
return await get_session_state(session, thread_id)
|
||||
|
||||
|
||||
async def clear_ai_responding(
|
||||
session: AsyncSession,
|
||||
thread_id: int,
|
||||
) -> ChatSessionState | None:
|
||||
"""Clear AI responding state when response is complete."""
|
||||
state = await get_session_state(session, thread_id)
|
||||
if state:
|
||||
state.ai_responding_to_user_id = None
|
||||
state.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(state)
|
||||
return state
|
||||
|
|
@ -2780,3 +2780,94 @@ class ConnectorService:
|
|||
}
|
||||
|
||||
return result_object, circleback_docs
|
||||
|
||||
async def search_obsidian(
|
||||
self,
|
||||
user_query: str,
|
||||
search_space_id: int,
|
||||
top_k: int = 20,
|
||||
start_date: datetime | None = None,
|
||||
end_date: datetime | None = None,
|
||||
) -> tuple:
|
||||
"""
|
||||
Search for Obsidian vault notes and return both the source information and langchain documents.
|
||||
|
||||
Uses combined chunk-level and document-level hybrid search with RRF fusion.
|
||||
|
||||
Args:
|
||||
user_query: The user's query
|
||||
search_space_id: The search space ID to search in
|
||||
top_k: Maximum number of results to return
|
||||
start_date: Optional start date for filtering documents by updated_at
|
||||
end_date: Optional end date for filtering documents by updated_at
|
||||
|
||||
Returns:
|
||||
tuple: (sources_info, langchain_documents)
|
||||
"""
|
||||
obsidian_docs = await self._combined_rrf_search(
|
||||
query_text=user_query,
|
||||
search_space_id=search_space_id,
|
||||
document_type="OBSIDIAN_CONNECTOR",
|
||||
top_k=top_k,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
)
|
||||
|
||||
# Early return if no results
|
||||
if not obsidian_docs:
|
||||
return {
|
||||
"id": 53,
|
||||
"name": "Obsidian Vault",
|
||||
"type": "OBSIDIAN_CONNECTOR",
|
||||
"sources": [],
|
||||
}, []
|
||||
|
||||
def _title_fn(doc_info: dict[str, Any], metadata: dict[str, Any]) -> str:
|
||||
return doc_info.get("title", "Untitled Note")
|
||||
|
||||
def _url_fn(doc_info: dict[str, Any], metadata: dict[str, Any]) -> str:
|
||||
# Obsidian URL format: obsidian://vault_name/path
|
||||
return doc_info.get("url", "")
|
||||
|
||||
def _description_fn(
|
||||
chunk: dict[str, Any], _doc_info: dict[str, Any], metadata: dict[str, Any]
|
||||
) -> str:
|
||||
description = self._chunk_preview(chunk.get("content", ""), limit=200)
|
||||
info_parts = []
|
||||
vault_name = metadata.get("vault_name")
|
||||
tags = metadata.get("tags", [])
|
||||
if vault_name:
|
||||
info_parts.append(f"Vault: {vault_name}")
|
||||
if tags and isinstance(tags, list) and len(tags) > 0:
|
||||
info_parts.append(f"Tags: {', '.join(tags[:3])}")
|
||||
if info_parts:
|
||||
description = (description + " | " + " | ".join(info_parts)).strip(" |")
|
||||
return description
|
||||
|
||||
def _extra_fields_fn(
|
||||
_chunk: dict[str, Any], _doc_info: dict[str, Any], metadata: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"vault_name": metadata.get("vault_name", ""),
|
||||
"file_path": metadata.get("file_path", ""),
|
||||
"tags": metadata.get("tags", []),
|
||||
"outgoing_links": metadata.get("outgoing_links", []),
|
||||
}
|
||||
|
||||
sources_list = self._build_chunk_sources_from_documents(
|
||||
obsidian_docs,
|
||||
title_fn=_title_fn,
|
||||
url_fn=_url_fn,
|
||||
description_fn=_description_fn,
|
||||
extra_fields_fn=_extra_fields_fn,
|
||||
)
|
||||
|
||||
# Create result object
|
||||
result_object = {
|
||||
"id": 53,
|
||||
"name": "Obsidian Vault",
|
||||
"type": "OBSIDIAN_CONNECTOR",
|
||||
"sources": sources_list,
|
||||
}
|
||||
|
||||
return result_object, obsidian_docs
|
||||
|
|
|
|||
|
|
@ -623,6 +623,28 @@ class MentionNotificationHandler(BaseNotificationHandler):
|
|||
def __init__(self):
|
||||
super().__init__("new_mention")
|
||||
|
||||
async def find_notification_by_mention(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
mention_id: int,
|
||||
) -> Notification | None:
|
||||
"""
|
||||
Find an existing notification by mention ID.
|
||||
|
||||
Args:
|
||||
session: Database session
|
||||
mention_id: The mention ID to search for
|
||||
|
||||
Returns:
|
||||
Notification if found, None otherwise
|
||||
"""
|
||||
query = select(Notification).where(
|
||||
Notification.type == self.notification_type,
|
||||
Notification.notification_metadata["mention_id"].astext == str(mention_id),
|
||||
)
|
||||
result = await session.execute(query)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def notify_new_mention(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
|
|
@ -641,11 +663,12 @@ class MentionNotificationHandler(BaseNotificationHandler):
|
|||
) -> Notification:
|
||||
"""
|
||||
Create notification when a user is @mentioned in a comment.
|
||||
Uses mention_id for idempotency to prevent duplicate notifications.
|
||||
|
||||
Args:
|
||||
session: Database session
|
||||
mentioned_user_id: User who was mentioned
|
||||
mention_id: ID of the mention record
|
||||
mention_id: ID of the mention record (used for idempotency)
|
||||
comment_id: ID of the comment containing the mention
|
||||
message_id: ID of the message being commented on
|
||||
thread_id: ID of the chat thread
|
||||
|
|
@ -658,8 +681,16 @@ class MentionNotificationHandler(BaseNotificationHandler):
|
|||
search_space_id: Search space ID
|
||||
|
||||
Returns:
|
||||
Notification: The created notification
|
||||
Notification: The created or existing notification
|
||||
"""
|
||||
# Check if notification already exists for this mention (idempotency)
|
||||
existing = await self.find_notification_by_mention(session, mention_id)
|
||||
if existing:
|
||||
logger.info(
|
||||
f"Notification already exists for mention {mention_id}, returning existing"
|
||||
)
|
||||
return existing
|
||||
|
||||
title = f"{author_name} mentioned you"
|
||||
message = content_preview[:100] + ("..." if len(content_preview) > 100 else "")
|
||||
|
||||
|
|
@ -676,21 +707,37 @@ class MentionNotificationHandler(BaseNotificationHandler):
|
|||
"content_preview": content_preview[:200],
|
||||
}
|
||||
|
||||
notification = Notification(
|
||||
user_id=mentioned_user_id,
|
||||
search_space_id=search_space_id,
|
||||
type=self.notification_type,
|
||||
title=title,
|
||||
message=message,
|
||||
notification_metadata=metadata,
|
||||
)
|
||||
session.add(notification)
|
||||
await session.commit()
|
||||
await session.refresh(notification)
|
||||
logger.info(
|
||||
f"Created new_mention notification {notification.id} for user {mentioned_user_id}"
|
||||
)
|
||||
return notification
|
||||
try:
|
||||
notification = Notification(
|
||||
user_id=mentioned_user_id,
|
||||
search_space_id=search_space_id,
|
||||
type=self.notification_type,
|
||||
title=title,
|
||||
message=message,
|
||||
notification_metadata=metadata,
|
||||
)
|
||||
session.add(notification)
|
||||
await session.commit()
|
||||
await session.refresh(notification)
|
||||
logger.info(
|
||||
f"Created new_mention notification {notification.id} for user {mentioned_user_id}"
|
||||
)
|
||||
return notification
|
||||
except Exception as e:
|
||||
# Handle race condition - if duplicate key error, try to fetch existing
|
||||
await session.rollback()
|
||||
if (
|
||||
"duplicate key" in str(e).lower()
|
||||
or "unique constraint" in str(e).lower()
|
||||
):
|
||||
logger.warning(
|
||||
f"Duplicate notification detected for mention {mention_id}, fetching existing"
|
||||
)
|
||||
existing = await self.find_notification_by_mention(session, mention_id)
|
||||
if existing:
|
||||
return existing
|
||||
# Re-raise if not a duplicate key error or couldn't find existing
|
||||
raise
|
||||
|
||||
|
||||
class NotificationService:
|
||||
|
|
|
|||
|
|
@ -761,6 +761,49 @@ async def _index_bookstack_pages(
|
|||
)
|
||||
|
||||
|
||||
@celery_app.task(name="index_obsidian_vault", bind=True)
|
||||
def index_obsidian_vault_task(
|
||||
self,
|
||||
connector_id: int,
|
||||
search_space_id: int,
|
||||
user_id: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
):
|
||||
"""Celery task to index Obsidian vault notes."""
|
||||
import asyncio
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
try:
|
||||
loop.run_until_complete(
|
||||
_index_obsidian_vault(
|
||||
connector_id, search_space_id, user_id, start_date, end_date
|
||||
)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
async def _index_obsidian_vault(
|
||||
connector_id: int,
|
||||
search_space_id: int,
|
||||
user_id: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
):
|
||||
"""Index Obsidian vault with new session."""
|
||||
from app.routes.search_source_connectors_routes import (
|
||||
run_obsidian_indexing,
|
||||
)
|
||||
|
||||
async with get_celery_session_maker()() as session:
|
||||
await run_obsidian_indexing(
|
||||
session, connector_id, search_space_id, user_id, start_date, end_date
|
||||
)
|
||||
|
||||
|
||||
@celery_app.task(name="index_composio_connector", bind=True)
|
||||
def index_composio_connector_task(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ Supports loading LLM configurations from:
|
|||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from uuid import UUID
|
||||
|
||||
from langchain_core.messages import HumanMessage
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
|
@ -27,6 +28,10 @@ from app.agents.new_chat.llm_config import (
|
|||
)
|
||||
from app.db import Document, SurfsenseDocsDocument
|
||||
from app.schemas.new_chat import ChatAttachment
|
||||
from app.services.chat_session_state_service import (
|
||||
clear_ai_responding,
|
||||
set_ai_responding,
|
||||
)
|
||||
from app.services.connector_service import ConnectorService
|
||||
from app.services.new_streaming_service import VercelStreamingService
|
||||
|
||||
|
|
@ -167,9 +172,8 @@ async def stream_new_chat(
|
|||
search_space_id: The search space ID
|
||||
chat_id: The chat ID (used as LangGraph thread_id for memory)
|
||||
session: The database session
|
||||
user_id: The current user's UUID string (for memory tools)
|
||||
user_id: The current user's UUID string (for memory tools and session state)
|
||||
llm_config_id: The LLM configuration ID (default: -1 for first global config)
|
||||
messages: Optional chat history from frontend (list of ChatMessage)
|
||||
attachments: Optional attachments with extracted content
|
||||
mentioned_document_ids: Optional list of document IDs mentioned with @ in the chat
|
||||
mentioned_surfsense_doc_ids: Optional list of SurfSense doc IDs mentioned with @ in the chat
|
||||
|
|
@ -183,6 +187,9 @@ async def stream_new_chat(
|
|||
current_text_id: str | None = None
|
||||
|
||||
try:
|
||||
# Mark AI as responding to this user for live collaboration
|
||||
if user_id:
|
||||
await set_ai_responding(session, chat_id, UUID(user_id))
|
||||
# Load LLM config - supports both YAML (negative IDs) and database (positive IDs)
|
||||
agent_config: AgentConfig | None = None
|
||||
|
||||
|
|
@ -1147,3 +1154,7 @@ async def stream_new_chat(
|
|||
yield streaming_service.format_finish_step()
|
||||
yield streaming_service.format_finish()
|
||||
yield streaming_service.format_done()
|
||||
|
||||
finally:
|
||||
# Clear AI responding state for live collaboration
|
||||
await clear_ai_responding(session, chat_id)
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ from .luma_indexer import index_luma_events
|
|||
|
||||
# Documentation and knowledge management
|
||||
from .notion_indexer import index_notion_pages
|
||||
from .obsidian_indexer import index_obsidian_vault
|
||||
from .slack_indexer import index_slack_messages
|
||||
from .webcrawler_indexer import index_crawled_urls
|
||||
|
||||
|
|
@ -68,6 +69,7 @@ __all__ = [ # noqa: RUF022
|
|||
"index_linear_issues",
|
||||
# Documentation and knowledge management
|
||||
"index_notion_pages",
|
||||
"index_obsidian_vault",
|
||||
"index_crawled_urls",
|
||||
# Communication platforms
|
||||
"index_slack_messages",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,516 @@
|
|||
"""
|
||||
Obsidian connector indexer.
|
||||
|
||||
Indexes markdown notes from a local Obsidian vault.
|
||||
This connector is only available in self-hosted mode.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import config
|
||||
from app.db import Document, DocumentType, SearchSourceConnectorType
|
||||
from app.services.llm_service import get_user_long_context_llm
|
||||
from app.services.task_logging_service import TaskLoggingService
|
||||
from app.utils.document_converters import (
|
||||
create_document_chunks,
|
||||
generate_content_hash,
|
||||
generate_document_summary,
|
||||
generate_unique_identifier_hash,
|
||||
)
|
||||
|
||||
from .base import (
|
||||
build_document_metadata_string,
|
||||
check_document_by_unique_identifier,
|
||||
get_connector_by_id,
|
||||
get_current_timestamp,
|
||||
logger,
|
||||
update_connector_last_indexed,
|
||||
)
|
||||
|
||||
|
||||
def parse_frontmatter(content: str) -> tuple[dict | None, str]:
|
||||
"""
|
||||
Parse YAML frontmatter from markdown content.
|
||||
|
||||
Args:
|
||||
content: The full markdown content
|
||||
|
||||
Returns:
|
||||
Tuple of (frontmatter dict or None, content without frontmatter)
|
||||
"""
|
||||
if not content.startswith("---"):
|
||||
return None, content
|
||||
|
||||
# Find the closing ---
|
||||
end_match = re.search(r"\n---\n", content[3:])
|
||||
if not end_match:
|
||||
return None, content
|
||||
|
||||
frontmatter_str = content[3 : end_match.start() + 3]
|
||||
remaining_content = content[end_match.end() + 3 :]
|
||||
|
||||
try:
|
||||
frontmatter = yaml.safe_load(frontmatter_str)
|
||||
return frontmatter, remaining_content.strip()
|
||||
except yaml.YAMLError:
|
||||
return None, content
|
||||
|
||||
|
||||
def extract_wiki_links(content: str) -> list[str]:
|
||||
"""
|
||||
Extract [[wiki-style links]] from content.
|
||||
|
||||
Args:
|
||||
content: Markdown content
|
||||
|
||||
Returns:
|
||||
List of linked note names
|
||||
"""
|
||||
# Match [[link]] or [[link|alias]]
|
||||
pattern = r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]"
|
||||
matches = re.findall(pattern, content)
|
||||
return list(set(matches))
|
||||
|
||||
|
||||
def extract_tags(content: str) -> list[str]:
|
||||
"""
|
||||
Extract #tags from content (both inline and frontmatter).
|
||||
|
||||
Args:
|
||||
content: Markdown content
|
||||
|
||||
Returns:
|
||||
List of tags (without # prefix)
|
||||
"""
|
||||
# Match #tag but not ## headers
|
||||
pattern = r"(?<!\S)#([a-zA-Z][a-zA-Z0-9_/-]*)"
|
||||
matches = re.findall(pattern, content)
|
||||
return list(set(matches))
|
||||
|
||||
|
||||
def scan_vault(
|
||||
vault_path: str,
|
||||
exclude_folders: list[str] | None = None,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Scan an Obsidian vault for markdown files.
|
||||
|
||||
Args:
|
||||
vault_path: Path to the Obsidian vault
|
||||
exclude_folders: List of folder names to exclude
|
||||
|
||||
Returns:
|
||||
List of file info dicts with path, name, modified time
|
||||
"""
|
||||
if exclude_folders is None:
|
||||
exclude_folders = [".trash", ".obsidian", "templates"]
|
||||
|
||||
vault = Path(vault_path)
|
||||
if not vault.exists():
|
||||
raise ValueError(f"Vault path does not exist: {vault_path}")
|
||||
|
||||
files = []
|
||||
for md_file in vault.rglob("*.md"):
|
||||
# Check if file is in an excluded folder
|
||||
relative_path = md_file.relative_to(vault)
|
||||
parts = relative_path.parts
|
||||
|
||||
if any(excluded in parts for excluded in exclude_folders):
|
||||
continue
|
||||
|
||||
try:
|
||||
stat = md_file.stat()
|
||||
files.append(
|
||||
{
|
||||
"path": str(md_file),
|
||||
"relative_path": str(relative_path),
|
||||
"name": md_file.stem,
|
||||
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=UTC),
|
||||
"created_at": datetime.fromtimestamp(stat.st_ctime, tz=UTC),
|
||||
"size": stat.st_size,
|
||||
}
|
||||
)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not stat file {md_file}: {e}")
|
||||
|
||||
return files
|
||||
|
||||
|
||||
async def index_obsidian_vault(
|
||||
session: AsyncSession,
|
||||
connector_id: int,
|
||||
search_space_id: int,
|
||||
user_id: str,
|
||||
start_date: str | None = None,
|
||||
end_date: str | None = None,
|
||||
update_last_indexed: bool = True,
|
||||
) -> tuple[int, str | None]:
|
||||
"""
|
||||
Index notes from a local Obsidian vault.
|
||||
|
||||
This indexer is only available in self-hosted mode as it requires
|
||||
direct file system access to the user's Obsidian vault.
|
||||
|
||||
Args:
|
||||
session: Database session
|
||||
connector_id: ID of the Obsidian connector
|
||||
search_space_id: ID of the search space to store documents in
|
||||
user_id: ID of the user
|
||||
start_date: Start date for filtering (YYYY-MM-DD format) - optional
|
||||
end_date: End date for filtering (YYYY-MM-DD format) - optional
|
||||
update_last_indexed: Whether to update the last_indexed_at timestamp
|
||||
|
||||
Returns:
|
||||
Tuple containing (number of documents indexed, error message or None)
|
||||
"""
|
||||
task_logger = TaskLoggingService(session, search_space_id)
|
||||
|
||||
# Check if self-hosted mode
|
||||
if not config.is_self_hosted():
|
||||
return 0, "Obsidian connector is only available in self-hosted mode"
|
||||
|
||||
# Log task start
|
||||
log_entry = await task_logger.log_task_start(
|
||||
task_name="obsidian_vault_indexing",
|
||||
source="connector_indexing_task",
|
||||
message=f"Starting Obsidian vault indexing for connector {connector_id}",
|
||||
metadata={
|
||||
"connector_id": connector_id,
|
||||
"user_id": str(user_id),
|
||||
"start_date": start_date,
|
||||
"end_date": end_date,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
# Get the connector
|
||||
await task_logger.log_task_progress(
|
||||
log_entry,
|
||||
f"Retrieving Obsidian connector {connector_id} from database",
|
||||
{"stage": "connector_retrieval"},
|
||||
)
|
||||
|
||||
connector = await get_connector_by_id(
|
||||
session, connector_id, SearchSourceConnectorType.OBSIDIAN_CONNECTOR
|
||||
)
|
||||
|
||||
if not connector:
|
||||
await task_logger.log_task_failure(
|
||||
log_entry,
|
||||
f"Connector with ID {connector_id} not found or is not an Obsidian connector",
|
||||
"Connector not found",
|
||||
{"error_type": "ConnectorNotFound"},
|
||||
)
|
||||
return (
|
||||
0,
|
||||
f"Connector with ID {connector_id} not found or is not an Obsidian connector",
|
||||
)
|
||||
|
||||
# Get vault path from connector config
|
||||
vault_path = connector.config.get("vault_path")
|
||||
if not vault_path:
|
||||
await task_logger.log_task_failure(
|
||||
log_entry,
|
||||
"Vault path not configured for this connector",
|
||||
"Missing vault path",
|
||||
{"error_type": "MissingVaultPath"},
|
||||
)
|
||||
return 0, "Vault path not configured for this connector"
|
||||
|
||||
# Validate vault path exists
|
||||
if not os.path.exists(vault_path):
|
||||
await task_logger.log_task_failure(
|
||||
log_entry,
|
||||
f"Vault path does not exist: {vault_path}",
|
||||
"Vault path not found",
|
||||
{"error_type": "VaultNotFound", "vault_path": vault_path},
|
||||
)
|
||||
return 0, f"Vault path does not exist: {vault_path}"
|
||||
|
||||
# Get configuration options
|
||||
exclude_folders = connector.config.get(
|
||||
"exclude_folders", [".trash", ".obsidian", "templates"]
|
||||
)
|
||||
vault_name = connector.config.get("vault_name") or os.path.basename(vault_path)
|
||||
|
||||
await task_logger.log_task_progress(
|
||||
log_entry,
|
||||
f"Scanning Obsidian vault: {vault_name}",
|
||||
{"stage": "vault_scan", "vault_path": vault_path},
|
||||
)
|
||||
|
||||
# Scan vault for markdown files
|
||||
try:
|
||||
files = scan_vault(vault_path, exclude_folders)
|
||||
except Exception as e:
|
||||
await task_logger.log_task_failure(
|
||||
log_entry,
|
||||
f"Failed to scan vault: {e}",
|
||||
"Vault scan error",
|
||||
{"error_type": "VaultScanError"},
|
||||
)
|
||||
return 0, f"Failed to scan vault: {e}"
|
||||
|
||||
logger.info(f"Found {len(files)} markdown files in vault")
|
||||
|
||||
await task_logger.log_task_progress(
|
||||
log_entry,
|
||||
f"Found {len(files)} markdown files to process",
|
||||
{"stage": "files_discovered", "file_count": len(files)},
|
||||
)
|
||||
|
||||
# Filter by date if provided (handle "undefined" string from frontend)
|
||||
# Also handle inverted dates (start > end) by skipping filtering
|
||||
start_dt = None
|
||||
end_dt = None
|
||||
|
||||
if start_date and start_date != "undefined":
|
||||
start_dt = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC)
|
||||
|
||||
if end_date and end_date != "undefined":
|
||||
# Make end_date inclusive (end of day)
|
||||
end_dt = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC)
|
||||
end_dt = end_dt.replace(hour=23, minute=59, second=59)
|
||||
|
||||
# Only apply date filtering if dates are valid and in correct order
|
||||
if start_dt and end_dt and start_dt > end_dt:
|
||||
logger.warning(
|
||||
f"start_date ({start_date}) is after end_date ({end_date}), skipping date filter"
|
||||
)
|
||||
else:
|
||||
if start_dt:
|
||||
files = [f for f in files if f["modified_at"] >= start_dt]
|
||||
logger.info(
|
||||
f"After start_date filter ({start_date}): {len(files)} files"
|
||||
)
|
||||
if end_dt:
|
||||
files = [f for f in files if f["modified_at"] <= end_dt]
|
||||
logger.info(f"After end_date filter ({end_date}): {len(files)} files")
|
||||
|
||||
logger.info(f"Processing {len(files)} files after date filtering")
|
||||
|
||||
# Get LLM for summarization
|
||||
long_context_llm = await get_user_long_context_llm(
|
||||
session, user_id, search_space_id
|
||||
)
|
||||
|
||||
indexed_count = 0
|
||||
skipped_count = 0
|
||||
|
||||
for file_info in files:
|
||||
try:
|
||||
file_path = file_info["path"]
|
||||
relative_path = file_info["relative_path"]
|
||||
|
||||
# Read file content
|
||||
try:
|
||||
with open(file_path, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
except UnicodeDecodeError:
|
||||
logger.warning(f"Could not decode file {file_path}, skipping")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
if not content.strip():
|
||||
logger.debug(f"Empty file {file_path}, skipping")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
# Parse frontmatter and extract metadata
|
||||
frontmatter, body_content = parse_frontmatter(content)
|
||||
wiki_links = extract_wiki_links(content)
|
||||
tags = extract_tags(content)
|
||||
|
||||
# Get title from frontmatter or filename
|
||||
title = file_info["name"]
|
||||
if frontmatter:
|
||||
title = frontmatter.get("title", title)
|
||||
# Also extract tags from frontmatter
|
||||
fm_tags = frontmatter.get("tags", [])
|
||||
if isinstance(fm_tags, list):
|
||||
tags = list({*tags, *fm_tags})
|
||||
elif isinstance(fm_tags, str):
|
||||
tags = list({*tags, fm_tags})
|
||||
|
||||
# Generate unique identifier using vault name and relative path
|
||||
unique_identifier = f"{vault_name}:{relative_path}"
|
||||
unique_identifier_hash = generate_unique_identifier_hash(
|
||||
DocumentType.OBSIDIAN_CONNECTOR,
|
||||
unique_identifier,
|
||||
search_space_id,
|
||||
)
|
||||
|
||||
# Check for existing document
|
||||
existing_document = await check_document_by_unique_identifier(
|
||||
session, unique_identifier_hash
|
||||
)
|
||||
|
||||
# Generate content hash
|
||||
content_hash = generate_content_hash(content, search_space_id)
|
||||
|
||||
# Build metadata
|
||||
document_metadata = {
|
||||
"vault_name": vault_name,
|
||||
"file_path": relative_path,
|
||||
"tags": tags,
|
||||
"outgoing_links": wiki_links,
|
||||
"frontmatter": frontmatter,
|
||||
"modified_at": file_info["modified_at"].isoformat(),
|
||||
"created_at": file_info["created_at"].isoformat(),
|
||||
"word_count": len(body_content.split()),
|
||||
}
|
||||
|
||||
# Build document content with metadata
|
||||
metadata_sections = [
|
||||
(
|
||||
"METADATA",
|
||||
[
|
||||
f"Title: {title}",
|
||||
f"Vault: {vault_name}",
|
||||
f"Path: {relative_path}",
|
||||
f"Tags: {', '.join(tags) if tags else 'None'}",
|
||||
f"Links to: {', '.join(wiki_links) if wiki_links else 'None'}",
|
||||
],
|
||||
),
|
||||
("CONTENT", [body_content]),
|
||||
]
|
||||
document_string = build_document_metadata_string(metadata_sections)
|
||||
|
||||
if existing_document:
|
||||
# Check if content has changed
|
||||
if existing_document.content_hash == content_hash:
|
||||
logger.debug(f"Note {title} unchanged, skipping")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
# Update existing document
|
||||
logger.info(f"Updating note: {title}")
|
||||
|
||||
# Generate new summary if content changed
|
||||
if long_context_llm:
|
||||
new_summary, _ = await generate_document_summary(
|
||||
document_string,
|
||||
long_context_llm,
|
||||
document_metadata,
|
||||
)
|
||||
# Store summary in metadata
|
||||
document_metadata["summary"] = new_summary
|
||||
|
||||
# Add URL and connector_id to metadata
|
||||
document_metadata["url"] = (
|
||||
f"obsidian://{vault_name}/{relative_path}"
|
||||
)
|
||||
document_metadata["connector_id"] = connector_id
|
||||
|
||||
existing_document.content = document_string
|
||||
existing_document.content_hash = content_hash
|
||||
existing_document.document_metadata = document_metadata
|
||||
existing_document.updated_at = get_current_timestamp()
|
||||
|
||||
# Update embedding
|
||||
embedding = config.embedding_model_instance.embed(document_string)
|
||||
existing_document.embedding = embedding
|
||||
|
||||
# Update chunks - delete old and create new
|
||||
existing_document.chunks.clear()
|
||||
new_chunks = await create_document_chunks(document_string)
|
||||
existing_document.chunks = new_chunks
|
||||
|
||||
indexed_count += 1
|
||||
|
||||
else:
|
||||
# Create new document
|
||||
logger.info(f"Indexing new note: {title}")
|
||||
|
||||
# Generate summary
|
||||
summary_content = ""
|
||||
if long_context_llm:
|
||||
summary_content, _ = await generate_document_summary(
|
||||
document_string,
|
||||
long_context_llm,
|
||||
document_metadata,
|
||||
)
|
||||
|
||||
# Generate embedding
|
||||
embedding = config.embedding_model_instance.embed(document_string)
|
||||
|
||||
# Add URL and summary to metadata
|
||||
document_metadata["url"] = (
|
||||
f"obsidian://{vault_name}/{relative_path}"
|
||||
)
|
||||
document_metadata["summary"] = summary_content
|
||||
document_metadata["connector_id"] = connector_id
|
||||
|
||||
# Create chunks
|
||||
chunks = await create_document_chunks(document_string)
|
||||
|
||||
# Create document
|
||||
new_document = Document(
|
||||
search_space_id=search_space_id,
|
||||
title=title,
|
||||
document_type=DocumentType.OBSIDIAN_CONNECTOR,
|
||||
content=document_string,
|
||||
content_hash=content_hash,
|
||||
unique_identifier_hash=unique_identifier_hash,
|
||||
document_metadata=document_metadata,
|
||||
embedding=embedding,
|
||||
chunks=chunks,
|
||||
updated_at=get_current_timestamp(),
|
||||
)
|
||||
|
||||
session.add(new_document)
|
||||
|
||||
indexed_count += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
f"Error processing file {file_info.get('path', 'unknown')}: {e}"
|
||||
)
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
# Update connector's last indexed timestamp
|
||||
await update_connector_last_indexed(session, connector, update_last_indexed)
|
||||
|
||||
# Commit all changes
|
||||
await session.commit()
|
||||
|
||||
await task_logger.log_task_success(
|
||||
log_entry,
|
||||
f"Successfully indexed {indexed_count} Obsidian notes (skipped {skipped_count})",
|
||||
{
|
||||
"indexed_count": indexed_count,
|
||||
"skipped_count": skipped_count,
|
||||
"total_files": len(files),
|
||||
},
|
||||
)
|
||||
|
||||
return indexed_count, None
|
||||
|
||||
except SQLAlchemyError as e:
|
||||
logger.exception(f"Database error during Obsidian indexing: {e}")
|
||||
await session.rollback()
|
||||
await task_logger.log_task_failure(
|
||||
log_entry,
|
||||
f"Database error during Obsidian indexing: {e}",
|
||||
"Database error",
|
||||
{"error_type": "SQLAlchemyError"},
|
||||
)
|
||||
return 0, f"Database error: {e}"
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Error during Obsidian indexing: {e}")
|
||||
await task_logger.log_task_failure(
|
||||
log_entry,
|
||||
f"Error during Obsidian indexing: {e}",
|
||||
"Unexpected error",
|
||||
{"error_type": type(e).__name__},
|
||||
)
|
||||
return 0, str(e)
|
||||
|
|
@ -46,12 +46,9 @@ dependencies = [
|
|||
"boto3>=1.35.0",
|
||||
"langchain-community>=0.3.31",
|
||||
"langchain-unstructured>=1.0.0",
|
||||
"langchain>=1.2.0",
|
||||
"litellm>=1.80.10",
|
||||
"langchain-litellm>=0.3.5",
|
||||
"langgraph>=1.0.5",
|
||||
"fake-useragent>=2.2.0",
|
||||
"deepagents>=0.3.0",
|
||||
"trafilatura>=2.0.0",
|
||||
"fastapi-users[oauth,sqlalchemy]>=15.0.3",
|
||||
"chonkie[all]>=1.5.0",
|
||||
|
|
@ -62,6 +59,9 @@ dependencies = [
|
|||
"sse-starlette>=3.1.1,<3.1.2",
|
||||
"gitingest>=0.3.1",
|
||||
"composio>=0.10.9",
|
||||
"deepagents>=0.3.8",
|
||||
"langchain>=1.2.6",
|
||||
"langgraph>=1.0.5",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
|
|
|
|||
58
surfsense_backend/uv.lock
generated
58
surfsense_backend/uv.lock
generated
|
|
@ -195,7 +195,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "anthropic"
|
||||
version = "0.75.0"
|
||||
version = "0.76.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anyio" },
|
||||
|
|
@ -207,9 +207,9 @@ dependencies = [
|
|||
{ name = "sniffio" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/04/1f/08e95f4b7e2d35205ae5dcbb4ae97e7d477fc521c275c02609e2931ece2d/anthropic-0.75.0.tar.gz", hash = "sha256:e8607422f4ab616db2ea5baacc215dd5f028da99ce2f022e33c7c535b29f3dfb", size = 439565 }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6e/be/d11abafaa15d6304826438170f7574d750218f49a106c54424a40cef4494/anthropic-0.76.0.tar.gz", hash = "sha256:e0cae6a368986d5cf6df743dfbb1b9519e6a9eee9c6c942ad8121c0b34416ffe", size = 495483 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/60/1c/1cd02b7ae64302a6e06724bf80a96401d5313708651d277b1458504a1730/anthropic-0.75.0-py3-none-any.whl", hash = "sha256:ea8317271b6c15d80225a9f3c670152746e88805a7a61e14d4a374577164965b", size = 388164 },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/70/7b0fd9c1a738f59d3babe2b4212031c34ab7d0fda4ffef15b58a55c5bcea/anthropic-0.76.0-py3-none-any.whl", hash = "sha256:81efa3113901192af2f0fe977d3ec73fdadb1e691586306c4256cd6d5ccc331c", size = 390309 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1231,17 +1231,18 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "deepagents"
|
||||
version = "0.3.0"
|
||||
version = "0.3.8"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "langchain" },
|
||||
{ name = "langchain-anthropic" },
|
||||
{ name = "langchain-core" },
|
||||
{ name = "langchain-google-genai" },
|
||||
{ name = "wcmatch" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/d3c2840bd0e66b6cd5948aa69625e129328ad261308e18fcb9a9420709da/deepagents-0.3.0.tar.gz", hash = "sha256:3dd4d2ed53efb1ef78aeb1020a5696c0ec7e58e627b305a6665d33fe6fbdedff", size = 51387 }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/47/69/d8dd80dd5c0c81393cc32623dd51e642c8607ab798276506b3b3e89b1f20/deepagents-0.3.8.tar.gz", hash = "sha256:4b8252f8deaad449ce39426cc2233a597ee079b9b690647a26d128c16d6c6eb8", size = 73956 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/22/e9/60bab7f37ff38bf982ea578e457ed1878ded613a3425462bcd07b00487e9/deepagents-0.3.0-py3-none-any.whl", hash = "sha256:9e23532d8d535dc2b0b4e0834453a1223a6a8f81b77947c0faf54537d05ce89a", size = 54065 },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/6a/35968909bd3184eafee97326bcfd16c99bf9b0e03aadb3327eaf7229ea11/deepagents-0.3.8-py3-none-any.whl", hash = "sha256:7c76205dc014173d795402045b51505517954c7c4a508175f8e4a529f51928cc", size = 79161 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1992,9 +1993,9 @@ dependencies = [
|
|||
{ name = "starlette" },
|
||||
{ name = "tiktoken" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d6/fe/a915f0c32a3d7920206a677f73c185b3eadf4ec151fb05aedd52e64713f7/gitingest-0.3.1.tar.gz", hash = "sha256:4587cab873d4e08bdb16d612bb153c23e0ce59771a1d57a438239c5e39f05ebf", size = 70681, upload-time = "2025-07-31T13:56:19.845Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d6/fe/a915f0c32a3d7920206a677f73c185b3eadf4ec151fb05aedd52e64713f7/gitingest-0.3.1.tar.gz", hash = "sha256:4587cab873d4e08bdb16d612bb153c23e0ce59771a1d57a438239c5e39f05ebf", size = 70681 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/00/15/f200ab2e73287e67d1dce6fbacf421552ae9fbafdc5f0cc8dd0d2fe4fc47/gitingest-0.3.1-py3-none-any.whl", hash = "sha256:8143a5e6a7140ede9f680e13d3931ac07c82ac9bd8bab9ad1fba017c8c1e8666", size = 68343, upload-time = "2025-07-31T13:56:17.729Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/15/f200ab2e73287e67d1dce6fbacf421552ae9fbafdc5f0cc8dd0d2fe4fc47/gitingest-0.3.1-py3-none-any.whl", hash = "sha256:8143a5e6a7140ede9f680e13d3931ac07c82ac9bd8bab9ad1fba017c8c1e8666", size = 68343 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2916,30 +2917,30 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "langchain"
|
||||
version = "1.2.0"
|
||||
version = "1.2.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "langchain-core" },
|
||||
{ name = "langgraph" },
|
||||
{ name = "pydantic" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b1/12/3a74c22abdfddd877dfc2ee666d516f9132877fcd25eb4dd694835c59c79/langchain-1.2.0.tar.gz", hash = "sha256:a087d1e2b2969819e29a91a6d5f98302aafe31bd49ba377ecee3bf5a5dcfe14a", size = 536126 }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f5/bc/d8f506a525baadee99a65c6cc28c1c35c9eaf1cb2009f048e9861d81a600/langchain-1.2.6.tar.gz", hash = "sha256:7d46cbf719d860a16f6fc182d5d3de17453dda187f3d43e9c40ac352a5094fdd", size = 553127 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/23/00/4e3fa0d90f5a5c376ccb8ca983d0f0f7287783dfac48702e18f01d24673b/langchain-1.2.0-py3-none-any.whl", hash = "sha256:82f0d17aa4fbb11560b30e1e7d4aeb75e3ad71ce09b85c90ab208b181a24ffac", size = 102828 },
|
||||
{ url = "https://files.pythonhosted.org/packages/3f/28/d5dc4cb06ccb29d62a590d446072964766555e85863f5044c6e644c07d0d/langchain-1.2.6-py3-none-any.whl", hash = "sha256:a9a6c39f03c09b6eb0f1b47e267ad2a2fd04e124dfaa9753bd6c11d2fe7d944e", size = 108458 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "langchain-anthropic"
|
||||
version = "1.3.0"
|
||||
version = "1.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anthropic" },
|
||||
{ name = "langchain-core" },
|
||||
{ name = "pydantic" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/de/50/cc3b3e0410d86de457d7a100dde763fc1c33c4ce884e883659aa4cf95538/langchain_anthropic-1.3.0.tar.gz", hash = "sha256:497a937ee0310c588196bff37f39f02d43d87bff3a12d16278bdbc3bd0e9a80b", size = 707207 }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0d/b6/ac5ee84e15bf79844c9c791f99a614c7ec7e1a63c2947e55977be01a81b4/langchain_anthropic-1.3.1.tar.gz", hash = "sha256:4f3d7a4a7729ab1aeaf62d32c87d4d227c1b5421668ca9e3734562b383470b07", size = 708940 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/29/ca/0725bc347a9c226da9d76f85bf7d03115caec7dbc87876af68579c4ab24e/langchain_anthropic-1.3.0-py3-none-any.whl", hash = "sha256:3823560e1df15d6082636baa04f87cb59052ba70aada0eba381c4679b1ce0eba", size = 45724 },
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/4f/7a5b32764addf4b757545b89899b9d76688176f19e4ee89868e3b8bbfd0f/langchain_anthropic-1.3.1-py3-none-any.whl", hash = "sha256:1fc28cf8037c30597ee6172fc2ff9e345efe8149a8c2a39897b1eebba2948322", size = 46328 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2967,7 +2968,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "langchain-core"
|
||||
version = "1.2.1"
|
||||
version = "1.2.7"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "jsonpatch" },
|
||||
|
|
@ -2979,9 +2980,24 @@ dependencies = [
|
|||
{ name = "typing-extensions" },
|
||||
{ name = "uuid-utils" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f5/a0/2177f4ef4dfbea8edeba377b7b4889d177b8356ce186640e4651b240fd4d/langchain_core-1.2.1.tar.gz", hash = "sha256:131e6ad105b47ec2adc4d4d973f569276688f48cd890ba44603d48e76d9993ce", size = 802986 }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a2/0e/664d8d81b3493e09cbab72448d2f9d693d1fa5aa2bcc488602203a9b6da0/langchain_core-1.2.7.tar.gz", hash = "sha256:e1460639f96c352b4a41c375f25aeb8d16ffc1769499fb1c20503aad59305ced", size = 837039 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/95/98c47dbb4b6098934ff70e0f52efef3a85505dbcccc9eb63587e21fde4c9/langchain_core-1.2.1-py3-none-any.whl", hash = "sha256:2f63859f85dc3d95f768e35fed605702e3ff5aa3e92c7b253103119613e79768", size = 475972 },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/6f/34a9fba14d191a67f7e2ee3dbce3e9b86d2fa7310e2c7f2c713583481bd2/langchain_core-1.2.7-py3-none-any.whl", hash = "sha256:452f4fef7a3d883357b22600788d37e3d8854ef29da345b7ac7099f33c31828b", size = 490232 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "langchain-google-genai"
|
||||
version = "4.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "filetype" },
|
||||
{ name = "google-genai" },
|
||||
{ name = "langchain-core" },
|
||||
{ name = "pydantic" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d8/0b/eae2305e207574dc633983a8a82a745e0ede1bce1f3a9daff24d2341fadc/langchain_google_genai-4.2.0.tar.gz", hash = "sha256:9a8d9bfc35354983ed29079cefff53c3e7c9c2a44b6ba75cc8f13a0cf8b55c33", size = 277361 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/22/51/39942c0083139652494bb354dddf0ed397703a4882302f7b48aeca531c96/langchain_google_genai-4.2.0-py3-none-any.whl", hash = "sha256:856041aaafceff65a4ef0d5acf5731f2db95229ff041132af011aec51e8279d9", size = 66452 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4516,9 +4532,9 @@ wheels = [
|
|||
name = "pathspec"
|
||||
version = "1.0.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4c/b2/bb8e495d5262bfec41ab5cb18f522f1012933347fb5d9e62452d446baca2/pathspec-1.0.3.tar.gz", hash = "sha256:bac5cf97ae2c2876e2d25ebb15078eb04d76e4b98921ee31c6f85ade8b59444d", size = 130841, upload-time = "2026-01-09T15:46:46.009Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4c/b2/bb8e495d5262bfec41ab5cb18f522f1012933347fb5d9e62452d446baca2/pathspec-1.0.3.tar.gz", hash = "sha256:bac5cf97ae2c2876e2d25ebb15078eb04d76e4b98921ee31c6f85ade8b59444d", size = 130841 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/32/2b/121e912bd60eebd623f873fd090de0e84f322972ab25a7f9044c056804ed/pathspec-1.0.3-py3-none-any.whl", hash = "sha256:e80767021c1cc524aa3fb14bedda9c34406591343cc42797b386ce7b9354fb6c", size = 55021, upload-time = "2026-01-09T15:46:44.652Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/32/2b/121e912bd60eebd623f873fd090de0e84f322972ab25a7f9044c056804ed/pathspec-1.0.3-py3-none-any.whl", hash = "sha256:e80767021c1cc524aa3fb14bedda9c34406591343cc42797b386ce7b9354fb6c", size = 55021 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6606,7 +6622,7 @@ requires-dist = [
|
|||
{ name = "chonkie", extras = ["all"], specifier = ">=1.5.0" },
|
||||
{ name = "composio", specifier = ">=0.10.9" },
|
||||
{ name = "datasets", specifier = ">=2.21.0" },
|
||||
{ name = "deepagents", specifier = ">=0.3.0" },
|
||||
{ name = "deepagents", specifier = ">=0.3.8" },
|
||||
{ name = "discord-py", specifier = ">=2.5.2" },
|
||||
{ name = "docling", specifier = ">=2.15.0" },
|
||||
{ name = "elasticsearch", specifier = ">=9.1.1" },
|
||||
|
|
@ -6622,7 +6638,7 @@ requires-dist = [
|
|||
{ name = "google-api-python-client", specifier = ">=2.156.0" },
|
||||
{ name = "google-auth-oauthlib", specifier = ">=1.2.1" },
|
||||
{ name = "kokoro", specifier = ">=0.9.4" },
|
||||
{ name = "langchain", specifier = ">=1.2.0" },
|
||||
{ name = "langchain", specifier = ">=1.2.6" },
|
||||
{ name = "langchain-community", specifier = ">=0.3.31" },
|
||||
{ name = "langchain-litellm", specifier = ">=0.3.5" },
|
||||
{ name = "langchain-unstructured", specifier = ">=1.0.0" },
|
||||
|
|
|
|||
|
|
@ -39,12 +39,6 @@ export default function DashboardLayout({
|
|||
icon: "SquareLibrary",
|
||||
items: [],
|
||||
},
|
||||
{
|
||||
title: "Logs",
|
||||
url: `/dashboard/${search_space_id}/logs`,
|
||||
icon: "Logs",
|
||||
items: [],
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import {
|
|||
// extractWriteTodosFromContent,
|
||||
hydratePlanStateAtom,
|
||||
} from "@/atoms/chat/plan-state.atom";
|
||||
import { membersAtom } from "@/atoms/members/members-query.atoms";
|
||||
import { currentUserAtom } from "@/atoms/user/user-query.atoms";
|
||||
import { Thread } from "@/components/assistant-ui/thread";
|
||||
import { ChatHeader } from "@/components/new-chat/chat-header";
|
||||
|
|
@ -32,7 +33,9 @@ import { DisplayImageToolUI } from "@/components/tool-ui/display-image";
|
|||
import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast";
|
||||
import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview";
|
||||
import { ScrapeWebpageToolUI } from "@/components/tool-ui/scrape-webpage";
|
||||
import { SaveMemoryToolUI, RecallMemoryToolUI } from "@/components/tool-ui/user-memory";
|
||||
import { RecallMemoryToolUI, SaveMemoryToolUI } from "@/components/tool-ui/user-memory";
|
||||
import { useChatSessionStateSync } from "@/hooks/use-chat-session-state";
|
||||
import { useMessagesElectric } from "@/hooks/use-messages-electric";
|
||||
// import { WriteTodosToolUI } from "@/components/tool-ui/write-todos";
|
||||
import { getBearerToken } from "@/lib/auth-utils";
|
||||
import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter";
|
||||
|
|
@ -258,6 +261,53 @@ export default function NewChatPage() {
|
|||
// Get current user for author info in shared chats
|
||||
const { data: currentUser } = useAtomValue(currentUserAtom);
|
||||
|
||||
// Live collaboration: sync session state and messages via Electric SQL
|
||||
useChatSessionStateSync(threadId);
|
||||
const { data: membersData } = useAtomValue(membersAtom);
|
||||
|
||||
const handleElectricMessagesUpdate = useCallback(
|
||||
(
|
||||
electricMessages: {
|
||||
id: number;
|
||||
thread_id: number;
|
||||
role: string;
|
||||
content: unknown;
|
||||
author_id: string | null;
|
||||
created_at: string;
|
||||
}[]
|
||||
) => {
|
||||
if (isRunning) {
|
||||
return;
|
||||
}
|
||||
|
||||
setMessages((prev) => {
|
||||
if (electricMessages.length < prev.length) {
|
||||
return prev;
|
||||
}
|
||||
|
||||
return electricMessages.map((msg) => {
|
||||
const member = msg.author_id
|
||||
? membersData?.find((m) => m.user_id === msg.author_id)
|
||||
: null;
|
||||
|
||||
return convertToThreadMessage({
|
||||
id: msg.id,
|
||||
thread_id: msg.thread_id,
|
||||
role: msg.role.toLowerCase() as "user" | "assistant" | "system",
|
||||
content: msg.content,
|
||||
author_id: msg.author_id,
|
||||
created_at: msg.created_at,
|
||||
author_display_name: member?.user_display_name ?? null,
|
||||
author_avatar_url: member?.user_avatar_url ?? null,
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
[isRunning, membersData]
|
||||
);
|
||||
|
||||
useMessagesElectric(threadId, handleElectricMessagesUpdate);
|
||||
|
||||
// Create the attachment adapter for file processing
|
||||
const attachmentAdapter = useMemo(() => createAttachmentAdapter(), []);
|
||||
|
||||
|
|
@ -587,8 +637,6 @@ export default function NewChatPage() {
|
|||
content: persistContent,
|
||||
})
|
||||
.then(() => {
|
||||
// For new threads, the backend updates the title from the first user message
|
||||
// Invalidate threads query so sidebar shows the updated title in real-time
|
||||
if (isNewThread) {
|
||||
queryClient.invalidateQueries({ queryKey: ["threads", String(searchSpaceId)] });
|
||||
}
|
||||
|
|
|
|||
15
surfsense_web/atoms/chat/chat-session-state.atom.ts
Normal file
15
surfsense_web/atoms/chat/chat-session-state.atom.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
"use client";
|
||||
|
||||
import { atom } from "jotai";
|
||||
|
||||
export interface ChatSessionStateData {
|
||||
threadId: number;
|
||||
isAiResponding: boolean;
|
||||
respondingToUserId: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Global chat session state atom.
|
||||
* Updated by useChatSessionStateSync hook, read anywhere.
|
||||
*/
|
||||
export const chatSessionStateAtom = atom<ChatSessionStateData | null>(null);
|
||||
|
|
@ -9,7 +9,7 @@ export const membersAtom = atomWithQuery((get) => {
|
|||
return {
|
||||
queryKey: cacheKeys.members.all(searchSpaceId?.toString() ?? ""),
|
||||
enabled: !!searchSpaceId,
|
||||
staleTime: 5 * 60 * 1000, // 5 minutes
|
||||
staleTime: 3 * 1000, // 3 seconds - short staleness for live collaboration
|
||||
queryFn: async () => {
|
||||
if (!searchSpaceId) {
|
||||
return [];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
"use client";
|
||||
|
||||
import { Loader2 } from "lucide-react";
|
||||
import type { FC } from "react";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
interface ChatSessionStatusProps {
|
||||
isAiResponding: boolean;
|
||||
respondingToUserId: string | null;
|
||||
currentUserId: string | null;
|
||||
members: Array<{
|
||||
user_id: string;
|
||||
user_display_name?: string | null;
|
||||
user_email?: string | null;
|
||||
}>;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
export const ChatSessionStatus: FC<ChatSessionStatusProps> = ({
|
||||
isAiResponding,
|
||||
respondingToUserId,
|
||||
currentUserId,
|
||||
members,
|
||||
className,
|
||||
}) => {
|
||||
if (!isAiResponding || !respondingToUserId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (respondingToUserId === currentUserId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const respondingUser = members.find((m) => m.user_id === respondingToUserId);
|
||||
const displayName =
|
||||
respondingUser?.user_display_name || respondingUser?.user_email || "another user";
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"flex items-center gap-2 px-3 py-2 text-sm text-muted-foreground bg-muted/50 rounded-lg",
|
||||
"animate-in fade-in slide-in-from-bottom-2 duration-300 ease-out",
|
||||
className
|
||||
)}
|
||||
>
|
||||
<Loader2 className="size-3.5 animate-spin" />
|
||||
<span>Currently responding to {displayName}</span>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,464 @@
|
|||
"use client";
|
||||
|
||||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import { FolderOpen, Info } from "lucide-react";
|
||||
import type { FC } from "react";
|
||||
import { useRef, useState } from "react";
|
||||
import { useForm } from "react-hook-form";
|
||||
import * as z from "zod";
|
||||
import {
|
||||
Accordion,
|
||||
AccordionContent,
|
||||
AccordionItem,
|
||||
AccordionTrigger,
|
||||
} from "@/components/ui/accordion";
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
||||
import {
|
||||
Form,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
} from "@/components/ui/form";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { EnumConnectorName } from "@/contracts/enums/connector";
|
||||
import { getConnectorBenefits } from "../connector-benefits";
|
||||
import type { ConnectFormProps } from "../index";
|
||||
|
||||
const obsidianConnectorFormSchema = z.object({
|
||||
name: z.string().min(3, {
|
||||
message: "Connector name must be at least 3 characters.",
|
||||
}),
|
||||
vault_path: z.string().min(1, {
|
||||
message: "Vault path is required.",
|
||||
}),
|
||||
vault_name: z.string().min(1, {
|
||||
message: "Vault name is required.",
|
||||
}),
|
||||
exclude_folders: z.string().optional(),
|
||||
include_attachments: z.boolean(),
|
||||
});
|
||||
|
||||
type ObsidianConnectorFormValues = z.infer<typeof obsidianConnectorFormSchema>;
|
||||
|
||||
export const ObsidianConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting }) => {
|
||||
const isSubmittingRef = useRef(false);
|
||||
const [periodicEnabled, setPeriodicEnabled] = useState(true);
|
||||
const [frequencyMinutes, setFrequencyMinutes] = useState("60");
|
||||
const form = useForm<ObsidianConnectorFormValues>({
|
||||
resolver: zodResolver(obsidianConnectorFormSchema),
|
||||
defaultValues: {
|
||||
name: "Obsidian Vault",
|
||||
vault_path: "",
|
||||
vault_name: "",
|
||||
exclude_folders: ".obsidian,.trash",
|
||||
include_attachments: false,
|
||||
},
|
||||
});
|
||||
|
||||
const handleSubmit = async (values: ObsidianConnectorFormValues) => {
|
||||
// Prevent multiple submissions
|
||||
if (isSubmittingRef.current || isSubmitting) {
|
||||
return;
|
||||
}
|
||||
|
||||
isSubmittingRef.current = true;
|
||||
try {
|
||||
// Parse exclude_folders into an array
|
||||
const excludeFolders = values.exclude_folders
|
||||
? values.exclude_folders
|
||||
.split(",")
|
||||
.map((f) => f.trim())
|
||||
.filter(Boolean)
|
||||
: [".obsidian", ".trash"];
|
||||
|
||||
await onSubmit({
|
||||
name: values.name,
|
||||
connector_type: EnumConnectorName.OBSIDIAN_CONNECTOR,
|
||||
config: {
|
||||
vault_path: values.vault_path,
|
||||
vault_name: values.vault_name,
|
||||
exclude_folders: excludeFolders,
|
||||
include_attachments: values.include_attachments,
|
||||
},
|
||||
is_indexable: true,
|
||||
is_active: true,
|
||||
last_indexed_at: null,
|
||||
periodic_indexing_enabled: periodicEnabled,
|
||||
indexing_frequency_minutes: periodicEnabled ? Number.parseInt(frequencyMinutes, 10) : null,
|
||||
next_scheduled_at: null,
|
||||
periodicEnabled,
|
||||
frequencyMinutes,
|
||||
});
|
||||
} finally {
|
||||
isSubmittingRef.current = false;
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-6 pb-6">
|
||||
<Alert className="bg-purple-500/10 dark:bg-purple-500/10 border-purple-500/30 p-2 sm:p-3 flex items-center [&>svg]:relative [&>svg]:left-0 [&>svg]:top-0 [&>svg+div]:translate-y-0">
|
||||
<FolderOpen className="h-3 w-3 sm:h-4 sm:w-4 shrink-0 ml-1 text-purple-500" />
|
||||
<div className="-ml-1">
|
||||
<AlertTitle className="text-xs sm:text-sm">Self-Hosted Only</AlertTitle>
|
||||
<AlertDescription className="text-[10px] sm:text-xs pl-0!">
|
||||
This connector requires direct file system access and only works with self-hosted
|
||||
SurfSense installations.
|
||||
</AlertDescription>
|
||||
</div>
|
||||
</Alert>
|
||||
|
||||
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-3 sm:space-y-4">
|
||||
<Form {...form}>
|
||||
<form
|
||||
id="obsidian-connect-form"
|
||||
onSubmit={form.handleSubmit(handleSubmit)}
|
||||
className="space-y-4 sm:space-y-6"
|
||||
>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="name"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="text-xs sm:text-sm">Connector Name</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
placeholder="My Obsidian Vault"
|
||||
className="h-8 sm:h-10 px-2 sm:px-3 text-xs sm:text-sm border-slate-400/20 focus-visible:border-slate-400/40"
|
||||
disabled={isSubmitting}
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription className="text-[10px] sm:text-xs">
|
||||
A friendly name to identify this connector.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="vault_path"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="text-xs sm:text-sm">Vault Path</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
placeholder="/path/to/your/obsidian/vault"
|
||||
className="h-8 sm:h-10 px-2 sm:px-3 text-xs sm:text-sm border-slate-400/20 focus-visible:border-slate-400/40 font-mono"
|
||||
disabled={isSubmitting}
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription className="text-[10px] sm:text-xs">
|
||||
The absolute path to your Obsidian vault on the server. This must be accessible
|
||||
from the SurfSense backend.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="vault_name"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="text-xs sm:text-sm">Vault Name</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
placeholder="My Knowledge Base"
|
||||
className="h-8 sm:h-10 px-2 sm:px-3 text-xs sm:text-sm border-slate-400/20 focus-visible:border-slate-400/40"
|
||||
disabled={isSubmitting}
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription className="text-[10px] sm:text-xs">
|
||||
A display name for your vault. This will be used in search results.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="exclude_folders"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="text-xs sm:text-sm">Exclude Folders</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
placeholder=".obsidian,.trash,templates"
|
||||
className="h-8 sm:h-10 px-2 sm:px-3 text-xs sm:text-sm border-slate-400/20 focus-visible:border-slate-400/40 font-mono"
|
||||
disabled={isSubmitting}
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription className="text-[10px] sm:text-xs">
|
||||
Comma-separated list of folder names to exclude from indexing.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="include_attachments"
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex flex-row items-center justify-between rounded-lg border border-slate-400/20 p-3">
|
||||
<div className="space-y-0.5">
|
||||
<FormLabel className="text-xs sm:text-sm">Include Attachments</FormLabel>
|
||||
<FormDescription className="text-[10px] sm:text-xs">
|
||||
Index attachment folders and embedded files (images, PDFs, etc.)
|
||||
</FormDescription>
|
||||
</div>
|
||||
<FormControl>
|
||||
<Switch
|
||||
checked={field.value}
|
||||
onCheckedChange={field.onChange}
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
</FormControl>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
{/* Indexing Configuration */}
|
||||
<div className="space-y-4 pt-4 border-t border-slate-400/20">
|
||||
<h3 className="text-sm sm:text-base font-medium">Indexing Configuration</h3>
|
||||
|
||||
{/* Periodic Sync Config */}
|
||||
<div className="rounded-xl bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="space-y-1">
|
||||
<h3 className="font-medium text-sm sm:text-base">Enable Periodic Sync</h3>
|
||||
<p className="text-xs sm:text-sm text-muted-foreground">
|
||||
Automatically re-index at regular intervals
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
checked={periodicEnabled}
|
||||
onCheckedChange={setPeriodicEnabled}
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{periodicEnabled && (
|
||||
<div className="mt-4 pt-4 border-t border-slate-400/20 space-y-3">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="frequency" className="text-xs sm:text-sm">
|
||||
Sync Frequency
|
||||
</Label>
|
||||
<Select
|
||||
value={frequencyMinutes}
|
||||
onValueChange={setFrequencyMinutes}
|
||||
disabled={isSubmitting}
|
||||
>
|
||||
<SelectTrigger
|
||||
id="frequency"
|
||||
className="w-full bg-slate-400/5 dark:bg-slate-400/5 border-slate-400/20 text-xs sm:text-sm"
|
||||
>
|
||||
<SelectValue placeholder="Select frequency" />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="z-100">
|
||||
<SelectItem value="5" className="text-xs sm:text-sm">
|
||||
Every 5 minutes
|
||||
</SelectItem>
|
||||
<SelectItem value="15" className="text-xs sm:text-sm">
|
||||
Every 15 minutes
|
||||
</SelectItem>
|
||||
<SelectItem value="60" className="text-xs sm:text-sm">
|
||||
Every hour
|
||||
</SelectItem>
|
||||
<SelectItem value="360" className="text-xs sm:text-sm">
|
||||
Every 6 hours
|
||||
</SelectItem>
|
||||
<SelectItem value="720" className="text-xs sm:text-sm">
|
||||
Every 12 hours
|
||||
</SelectItem>
|
||||
<SelectItem value="1440" className="text-xs sm:text-sm">
|
||||
Daily
|
||||
</SelectItem>
|
||||
<SelectItem value="10080" className="text-xs sm:text-sm">
|
||||
Weekly
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</Form>
|
||||
</div>
|
||||
|
||||
{/* What you get section */}
|
||||
{getConnectorBenefits(EnumConnectorName.OBSIDIAN_CONNECTOR) && (
|
||||
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 px-3 sm:px-6 py-4 space-y-2">
|
||||
<h4 className="text-xs sm:text-sm font-medium">
|
||||
What you get with Obsidian integration:
|
||||
</h4>
|
||||
<ul className="list-disc pl-5 text-[10px] sm:text-xs text-muted-foreground space-y-1">
|
||||
{getConnectorBenefits(EnumConnectorName.OBSIDIAN_CONNECTOR)?.map((benefit) => (
|
||||
<li key={benefit}>{benefit}</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Documentation Section */}
|
||||
<Accordion
|
||||
type="single"
|
||||
collapsible
|
||||
className="w-full border border-border rounded-xl bg-slate-400/5 dark:bg-white/5"
|
||||
>
|
||||
<AccordionItem value="documentation" className="border-0">
|
||||
<AccordionTrigger className="text-sm sm:text-base font-medium px-3 sm:px-6 no-underline hover:no-underline">
|
||||
Documentation
|
||||
</AccordionTrigger>
|
||||
<AccordionContent className="px-3 sm:px-6 pb-3 sm:pb-6 space-y-6">
|
||||
<div>
|
||||
<h3 className="text-sm sm:text-base font-semibold mb-2">How it works</h3>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground">
|
||||
The Obsidian connector scans your local Obsidian vault directory and indexes all
|
||||
Markdown files. It preserves your note structure and extracts metadata from YAML
|
||||
frontmatter.
|
||||
</p>
|
||||
<ul className="mt-2 list-disc pl-5 text-[10px] sm:text-xs text-muted-foreground space-y-1">
|
||||
<li>
|
||||
The connector parses frontmatter metadata (title, tags, aliases, dates, etc.)
|
||||
</li>
|
||||
<li>Wiki-style links ([[note]]) are extracted and preserved</li>
|
||||
<li>Inline tags (#tag) are recognized and indexed</li>
|
||||
<li>Content is chunked intelligently for optimal search results</li>
|
||||
<li>
|
||||
Subsequent indexing runs use content hashing to skip unchanged files for faster
|
||||
sync
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h3 className="text-sm sm:text-base font-semibold mb-2">Setup</h3>
|
||||
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 mb-4">
|
||||
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
|
||||
<AlertTitle className="text-[10px] sm:text-xs">
|
||||
File System Access Required
|
||||
</AlertTitle>
|
||||
<AlertDescription className="text-[9px] sm:text-[10px]">
|
||||
The SurfSense backend must have read access to your Obsidian vault directory.
|
||||
For Docker deployments, mount your vault as a volume.
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
|
||||
<div className="space-y-4 sm:space-y-6">
|
||||
<div>
|
||||
<h4 className="text-[10px] sm:text-xs font-medium mb-2">
|
||||
Step 1: Locate your vault
|
||||
</h4>
|
||||
<ol className="list-decimal pl-5 space-y-2 text-[10px] sm:text-xs text-muted-foreground">
|
||||
<li>
|
||||
<strong>macOS/Linux:</strong> Right-click any note in Obsidian → "Reveal in
|
||||
Finder" to see the vault folder
|
||||
</li>
|
||||
<li>
|
||||
<strong>Windows:</strong> Right-click any note → "Show in system explorer"
|
||||
</li>
|
||||
<li>
|
||||
<strong>Or:</strong> Click the vault switcher (bottom-left icon) → "Open
|
||||
folder" next to your vault name
|
||||
</li>
|
||||
</ol>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h4 className="text-[10px] sm:text-xs font-medium mb-2">
|
||||
Step 2: Enter the path
|
||||
</h4>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground mb-2">
|
||||
<strong>Running locally (no Docker):</strong> Use the direct path to your
|
||||
vault:
|
||||
</p>
|
||||
<pre className="bg-slate-800 text-slate-200 p-2 rounded text-[9px] sm:text-[10px] overflow-x-auto mb-2">
|
||||
{`/Users/yourname/Documents/MyObsidianVault`}
|
||||
</pre>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground mb-2">
|
||||
<strong>Running in Docker:</strong> Mount your vault as a volume in
|
||||
docker-compose.yml:
|
||||
</p>
|
||||
<pre className="bg-slate-800 text-slate-200 p-2 rounded text-[9px] sm:text-[10px] overflow-x-auto">
|
||||
{`volumes:
|
||||
- /path/to/your/vault:/app/obsidian_vaults/my-vault:ro`}
|
||||
</pre>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground mt-2">
|
||||
Then use <code>/app/obsidian_vaults/my-vault</code> as your vault path.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h4 className="text-[10px] sm:text-xs font-medium mb-2">
|
||||
Step 3: Configure exclusions
|
||||
</h4>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground">
|
||||
Common folders to exclude:
|
||||
</p>
|
||||
<ul className="list-disc pl-5 mt-1 space-y-1 text-[10px] sm:text-xs text-muted-foreground">
|
||||
<li>
|
||||
<code>.obsidian</code> - Obsidian config (always recommended)
|
||||
</li>
|
||||
<li>
|
||||
<code>.trash</code> - Obsidian's trash folder
|
||||
</li>
|
||||
<li>
|
||||
<code>templates</code> - If you have a templates folder
|
||||
</li>
|
||||
<li>
|
||||
<code>daily-notes</code> - If you want to exclude daily notes
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h3 className="text-sm sm:text-base font-semibold mb-2">What Gets Indexed</h3>
|
||||
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20">
|
||||
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
|
||||
<AlertTitle className="text-[10px] sm:text-xs">Indexed Content</AlertTitle>
|
||||
<AlertDescription className="text-[9px] sm:text-[10px]">
|
||||
<p className="mb-2">The Obsidian connector indexes:</p>
|
||||
<ul className="list-disc pl-5 space-y-1">
|
||||
<li>All Markdown files (.md) in your vault</li>
|
||||
<li>YAML frontmatter metadata (title, tags, aliases, dates)</li>
|
||||
<li>Wiki-style links between notes</li>
|
||||
<li>Inline tags throughout your notes</li>
|
||||
<li>Full note content with proper chunking</li>
|
||||
</ul>
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
</div>
|
||||
</div>
|
||||
</AccordionContent>
|
||||
</AccordionItem>
|
||||
</Accordion>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
@ -108,6 +108,14 @@ export function getConnectorBenefits(connectorType: string): string[] | null {
|
|||
"Real-time updates via webhook integration",
|
||||
"No manual indexing required - meetings are added automatically",
|
||||
],
|
||||
OBSIDIAN_CONNECTOR: [
|
||||
"Search through all your Obsidian notes and knowledge base",
|
||||
"Access note content with YAML frontmatter metadata preserved",
|
||||
"Wiki-style links ([[note]]) and #tags are indexed",
|
||||
"Connect your personal knowledge base directly to your search space",
|
||||
"Incremental sync - only changed files are re-indexed",
|
||||
"Full support for your vault's folder structure",
|
||||
],
|
||||
};
|
||||
|
||||
return benefits[connectorType] || null;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { GithubConnectForm } from "./components/github-connect-form";
|
|||
import { LinkupApiConnectForm } from "./components/linkup-api-connect-form";
|
||||
import { LumaConnectForm } from "./components/luma-connect-form";
|
||||
import { MCPConnectForm } from "./components/mcp-connect-form";
|
||||
import { ObsidianConnectForm } from "./components/obsidian-connect-form";
|
||||
import { SearxngConnectForm } from "./components/searxng-connect-form";
|
||||
import { TavilyApiConnectForm } from "./components/tavily-api-connect-form";
|
||||
|
||||
|
|
@ -58,6 +59,8 @@ export function getConnectFormComponent(connectorType: string): ConnectFormCompo
|
|||
return CirclebackConnectForm;
|
||||
case "MCP_CONNECTOR":
|
||||
return MCPConnectForm;
|
||||
case "OBSIDIAN_CONNECTOR":
|
||||
return ObsidianConnectForm;
|
||||
// Add other connector types here as needed
|
||||
default:
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,191 @@
|
|||
"use client";
|
||||
|
||||
import { FolderOpen } from "lucide-react";
|
||||
import type { FC } from "react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import type { ConnectorConfigProps } from "../index";
|
||||
|
||||
export interface ObsidianConfigProps extends ConnectorConfigProps {
|
||||
onNameChange?: (name: string) => void;
|
||||
}
|
||||
|
||||
export const ObsidianConfig: FC<ObsidianConfigProps> = ({
|
||||
connector,
|
||||
onConfigChange,
|
||||
onNameChange,
|
||||
}) => {
|
||||
const [vaultPath, setVaultPath] = useState<string>(
|
||||
(connector.config?.vault_path as string) || ""
|
||||
);
|
||||
const [vaultName, setVaultName] = useState<string>(
|
||||
(connector.config?.vault_name as string) || ""
|
||||
);
|
||||
const [excludeFolders, setExcludeFolders] = useState<string>(() => {
|
||||
const folders = connector.config?.exclude_folders;
|
||||
if (Array.isArray(folders)) {
|
||||
return folders.join(", ");
|
||||
}
|
||||
return (folders as string) || ".obsidian, .trash";
|
||||
});
|
||||
const [includeAttachments, setIncludeAttachments] = useState<boolean>(
|
||||
(connector.config?.include_attachments as boolean) || false
|
||||
);
|
||||
const [name, setName] = useState<string>(connector.name || "");
|
||||
|
||||
// Update values when connector changes
|
||||
useEffect(() => {
|
||||
const path = (connector.config?.vault_path as string) || "";
|
||||
const vName = (connector.config?.vault_name as string) || "";
|
||||
const folders = connector.config?.exclude_folders;
|
||||
const attachments = (connector.config?.include_attachments as boolean) || false;
|
||||
|
||||
setVaultPath(path);
|
||||
setVaultName(vName);
|
||||
setIncludeAttachments(attachments);
|
||||
setName(connector.name || "");
|
||||
|
||||
if (Array.isArray(folders)) {
|
||||
setExcludeFolders(folders.join(", "));
|
||||
} else if (typeof folders === "string") {
|
||||
setExcludeFolders(folders);
|
||||
}
|
||||
}, [connector.config, connector.name]);
|
||||
|
||||
const handleVaultPathChange = (value: string) => {
|
||||
setVaultPath(value);
|
||||
if (onConfigChange) {
|
||||
onConfigChange({
|
||||
...connector.config,
|
||||
vault_path: value,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleVaultNameChange = (value: string) => {
|
||||
setVaultName(value);
|
||||
if (onConfigChange) {
|
||||
onConfigChange({
|
||||
...connector.config,
|
||||
vault_name: value,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleExcludeFoldersChange = (value: string) => {
|
||||
setExcludeFolders(value);
|
||||
const foldersArray = value
|
||||
.split(",")
|
||||
.map((f) => f.trim())
|
||||
.filter(Boolean);
|
||||
if (onConfigChange) {
|
||||
onConfigChange({
|
||||
...connector.config,
|
||||
exclude_folders: foldersArray,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleIncludeAttachmentsChange = (value: boolean) => {
|
||||
setIncludeAttachments(value);
|
||||
if (onConfigChange) {
|
||||
onConfigChange({
|
||||
...connector.config,
|
||||
include_attachments: value,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleNameChange = (value: string) => {
|
||||
setName(value);
|
||||
if (onNameChange) {
|
||||
onNameChange(value);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* Connector Name */}
|
||||
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-3 sm:space-y-4">
|
||||
<div className="space-y-2">
|
||||
<Label className="text-xs sm:text-sm">Connector Name</Label>
|
||||
<Input
|
||||
value={name}
|
||||
onChange={(e) => handleNameChange(e.target.value)}
|
||||
placeholder="My Obsidian Vault"
|
||||
className="border-slate-400/20 focus-visible:border-slate-400/40"
|
||||
/>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground">
|
||||
A friendly name to identify this connector.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Configuration */}
|
||||
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6 space-y-3 sm:space-y-4">
|
||||
<div className="space-y-1 sm:space-y-2">
|
||||
<h3 className="font-medium text-sm sm:text-base flex items-center gap-2">
|
||||
<FolderOpen className="h-4 w-4 text-purple-500" />
|
||||
Vault Configuration
|
||||
</h3>
|
||||
</div>
|
||||
|
||||
<div className="space-y-4">
|
||||
<div className="space-y-2">
|
||||
<Label className="text-xs sm:text-sm">Vault Path</Label>
|
||||
<Input
|
||||
value={vaultPath}
|
||||
onChange={(e) => handleVaultPathChange(e.target.value)}
|
||||
placeholder="/path/to/your/obsidian/vault"
|
||||
className="border-slate-400/20 focus-visible:border-slate-400/40 font-mono"
|
||||
/>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground">
|
||||
The absolute path to your Obsidian vault on the server.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label className="text-xs sm:text-sm">Vault Name</Label>
|
||||
<Input
|
||||
value={vaultName}
|
||||
onChange={(e) => handleVaultNameChange(e.target.value)}
|
||||
placeholder="My Knowledge Base"
|
||||
className="border-slate-400/20 focus-visible:border-slate-400/40"
|
||||
/>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground">
|
||||
A display name for your vault in search results.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label className="text-xs sm:text-sm">Exclude Folders</Label>
|
||||
<Input
|
||||
value={excludeFolders}
|
||||
onChange={(e) => handleExcludeFoldersChange(e.target.value)}
|
||||
placeholder=".obsidian, .trash, templates"
|
||||
className="border-slate-400/20 focus-visible:border-slate-400/40 font-mono"
|
||||
/>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground">
|
||||
Comma-separated list of folder names to exclude from indexing.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between rounded-lg border border-slate-400/20 p-3">
|
||||
<div className="space-y-0.5">
|
||||
<Label className="text-xs sm:text-sm">Include Attachments</Label>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground">
|
||||
Index attachment folders and embedded files
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
checked={includeAttachments}
|
||||
onCheckedChange={handleIncludeAttachmentsChange}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
@ -5,8 +5,8 @@ import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
|||
import { BaiduSearchApiConfig } from "./components/baidu-search-api-config";
|
||||
import { BookStackConfig } from "./components/bookstack-config";
|
||||
import { CirclebackConfig } from "./components/circleback-config";
|
||||
import { ComposioConfig } from "./components/composio-config";
|
||||
import { ClickUpConfig } from "./components/clickup-config";
|
||||
import { ComposioConfig } from "./components/composio-config";
|
||||
import { ConfluenceConfig } from "./components/confluence-config";
|
||||
import { DiscordConfig } from "./components/discord-config";
|
||||
import { ElasticsearchConfig } from "./components/elasticsearch-config";
|
||||
|
|
@ -16,6 +16,7 @@ import { JiraConfig } from "./components/jira-config";
|
|||
import { LinkupApiConfig } from "./components/linkup-api-config";
|
||||
import { LumaConfig } from "./components/luma-config";
|
||||
import { MCPConfig } from "./components/mcp-config";
|
||||
import { ObsidianConfig } from "./components/obsidian-config";
|
||||
import { SearxngConfig } from "./components/searxng-config";
|
||||
import { SlackConfig } from "./components/slack-config";
|
||||
import { TavilyApiConfig } from "./components/tavily-api-config";
|
||||
|
|
@ -74,6 +75,8 @@ export function getConnectorConfigComponent(
|
|||
return CirclebackConfig;
|
||||
case "MCP_CONNECTOR":
|
||||
return MCPConfig;
|
||||
case "OBSIDIAN_CONNECTOR":
|
||||
return ObsidianConfig;
|
||||
case "COMPOSIO_GOOGLE_DRIVE_CONNECTOR":
|
||||
case "COMPOSIO_GMAIL_CONNECTOR":
|
||||
case "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR":
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ export const ConnectorConnectView: FC<ConnectorConnectViewProps> = ({
|
|||
LUMA_CONNECTOR: "luma-connect-form",
|
||||
CIRCLEBACK_CONNECTOR: "circleback-connect-form",
|
||||
MCP_CONNECTOR: "mcp-connect-form",
|
||||
OBSIDIAN_CONNECTOR: "obsidian-connect-form",
|
||||
};
|
||||
const formId = formIdMap[connectorType];
|
||||
if (formId) {
|
||||
|
|
@ -141,12 +142,10 @@ export const ConnectorConnectView: FC<ConnectorConnectViewProps> = ({
|
|||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
Connecting
|
||||
</>
|
||||
) : connectorType === "MCP_CONNECTOR" ? (
|
||||
"Connect"
|
||||
) : (
|
||||
<>
|
||||
{connectorType === "MCP_CONNECTOR"
|
||||
? "Connect"
|
||||
: `Connect ${getConnectorTypeDisplay(connectorType)}`}
|
||||
</>
|
||||
`Connect ${getConnectorTypeDisplay(connectorType)}`
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -166,6 +166,13 @@ export const OTHER_CONNECTORS = [
|
|||
description: "Connect to MCP servers for AI tools",
|
||||
connectorType: EnumConnectorName.MCP_CONNECTOR,
|
||||
},
|
||||
{
|
||||
id: "obsidian-connector",
|
||||
title: "Obsidian",
|
||||
description: "Index your Obsidian vault (self-hosted only)",
|
||||
connectorType: EnumConnectorName.OBSIDIAN_CONNECTOR,
|
||||
selfHostedOnly: true,
|
||||
},
|
||||
] as const;
|
||||
|
||||
// Composio Connectors - Individual entries for each supported toolkit
|
||||
|
|
|
|||
|
|
@ -3,12 +3,13 @@
|
|||
import type { FC } from "react";
|
||||
import { EnumConnectorName } from "@/contracts/enums/connector";
|
||||
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||
import { isSelfHosted } from "@/lib/env-config";
|
||||
import { ConnectorCard } from "../components/connector-card";
|
||||
import {
|
||||
COMPOSIO_CONNECTORS,
|
||||
CRAWLERS,
|
||||
OAUTH_CONNECTORS,
|
||||
OTHER_CONNECTORS,
|
||||
COMPOSIO_CONNECTORS,
|
||||
} from "../constants/connector-constants";
|
||||
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
|
||||
|
||||
|
|
@ -57,23 +58,31 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
|||
onManage,
|
||||
onViewAccountsList,
|
||||
}) => {
|
||||
// Filter connectors based on search
|
||||
// Check if self-hosted mode (for showing self-hosted only connectors)
|
||||
const selfHosted = isSelfHosted();
|
||||
|
||||
// Filter connectors based on search and deployment mode
|
||||
const filteredOAuth = OAUTH_CONNECTORS.filter(
|
||||
(c) =>
|
||||
c.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
|
||||
c.description.toLowerCase().includes(searchQuery.toLowerCase())
|
||||
// Filter by search query
|
||||
(c.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
|
||||
c.description.toLowerCase().includes(searchQuery.toLowerCase())) &&
|
||||
// Filter self-hosted only connectors in cloud mode
|
||||
(!("selfHostedOnly" in c) || !c.selfHostedOnly || selfHosted)
|
||||
);
|
||||
|
||||
const filteredCrawlers = CRAWLERS.filter(
|
||||
(c) =>
|
||||
c.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
|
||||
c.description.toLowerCase().includes(searchQuery.toLowerCase())
|
||||
(c.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
|
||||
c.description.toLowerCase().includes(searchQuery.toLowerCase())) &&
|
||||
(!("selfHostedOnly" in c) || !c.selfHostedOnly || selfHosted)
|
||||
);
|
||||
|
||||
const filteredOther = OTHER_CONNECTORS.filter(
|
||||
(c) =>
|
||||
c.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
|
||||
c.description.toLowerCase().includes(searchQuery.toLowerCase())
|
||||
(c.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
|
||||
c.description.toLowerCase().includes(searchQuery.toLowerCase())) &&
|
||||
(!("selfHostedOnly" in c) || !c.selfHostedOnly || selfHosted)
|
||||
);
|
||||
|
||||
// Filter Composio connectors
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ export const CONNECTOR_TO_DOCUMENT_TYPE: Record<string, string> = {
|
|||
ELASTICSEARCH_CONNECTOR: "ELASTICSEARCH_CONNECTOR",
|
||||
BOOKSTACK_CONNECTOR: "BOOKSTACK_CONNECTOR",
|
||||
CIRCLEBACK_CONNECTOR: "CIRCLEBACK",
|
||||
OBSIDIAN_CONNECTOR: "OBSIDIAN_CONNECTOR",
|
||||
|
||||
// Special mappings (connector type differs from document type)
|
||||
GOOGLE_DRIVE_CONNECTOR: "GOOGLE_DRIVE_FILE",
|
||||
|
|
|
|||
|
|
@ -26,11 +26,13 @@ import {
|
|||
import { useParams } from "next/navigation";
|
||||
import { type FC, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
import { chatSessionStateAtom } from "@/atoms/chat/chat-session-state.atom";
|
||||
import { showCommentsGutterAtom } from "@/atoms/chat/current-thread.atom";
|
||||
import {
|
||||
mentionedDocumentIdsAtom,
|
||||
mentionedDocumentsAtom,
|
||||
} from "@/atoms/chat/mentioned-documents.atom";
|
||||
import { membersAtom } from "@/atoms/members/members-query.atoms";
|
||||
import {
|
||||
globalNewLLMConfigsAtom,
|
||||
llmPreferencesAtom,
|
||||
|
|
@ -39,6 +41,7 @@ import {
|
|||
import { currentUserAtom } from "@/atoms/user/user-query.atoms";
|
||||
import { AssistantMessage } from "@/components/assistant-ui/assistant-message";
|
||||
import { ComposerAddAttachment, ComposerAttachments } from "@/components/assistant-ui/attachment";
|
||||
import { ChatSessionStatus } from "@/components/assistant-ui/chat-session-status";
|
||||
import { ConnectorIndicator } from "@/components/assistant-ui/connector-popup";
|
||||
import {
|
||||
InlineMentionEditor,
|
||||
|
|
@ -59,6 +62,7 @@ import {
|
|||
import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import type { Document } from "@/contracts/types/document.types";
|
||||
import { useCommentsElectric } from "@/hooks/use-comments-electric";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
interface ThreadProps {
|
||||
|
|
@ -86,6 +90,7 @@ const ThreadContent: FC<{ header?: React.ReactNode }> = ({ header }) => {
|
|||
>
|
||||
<ThreadPrimitive.Viewport
|
||||
turnAnchor="top"
|
||||
autoScroll
|
||||
className={cn(
|
||||
"aui-thread-viewport relative flex flex-1 min-h-0 flex-col overflow-y-auto px-4 pt-4 transition-[padding] duration-300 ease-out",
|
||||
showGutter && "lg:pr-30"
|
||||
|
|
@ -215,7 +220,7 @@ const Composer: FC = () => {
|
|||
const editorRef = useRef<InlineMentionEditorRef>(null);
|
||||
const editorContainerRef = useRef<HTMLDivElement>(null);
|
||||
const documentPickerRef = useRef<DocumentMentionPickerRef>(null);
|
||||
const { search_space_id } = useParams();
|
||||
const { search_space_id, chat_id } = useParams();
|
||||
const setMentionedDocumentIds = useSetAtom(mentionedDocumentIdsAtom);
|
||||
const composerRuntime = useComposerRuntime();
|
||||
const hasAutoFocusedRef = useRef(false);
|
||||
|
|
@ -223,6 +228,23 @@ const Composer: FC = () => {
|
|||
const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty);
|
||||
const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning);
|
||||
|
||||
// Live collaboration state
|
||||
const { data: currentUser } = useAtomValue(currentUserAtom);
|
||||
const { data: members } = useAtomValue(membersAtom);
|
||||
const threadId = useMemo(() => {
|
||||
if (Array.isArray(chat_id) && chat_id.length > 0) {
|
||||
return Number.parseInt(chat_id[0], 10) || null;
|
||||
}
|
||||
return typeof chat_id === "string" ? Number.parseInt(chat_id, 10) || null : null;
|
||||
}, [chat_id]);
|
||||
const sessionState = useAtomValue(chatSessionStateAtom);
|
||||
const isAiResponding = sessionState?.isAiResponding ?? false;
|
||||
const respondingToUserId = sessionState?.respondingToUserId ?? null;
|
||||
const isBlockedByOtherUser = isAiResponding && respondingToUserId !== currentUser?.id;
|
||||
|
||||
// Sync comments for the entire thread via Electric SQL (one subscription per thread)
|
||||
useCommentsElectric(threadId);
|
||||
|
||||
// Auto-focus editor on new chat page after mount
|
||||
useEffect(() => {
|
||||
if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) {
|
||||
|
|
@ -298,9 +320,9 @@ const Composer: FC = () => {
|
|||
[showDocumentPopover]
|
||||
);
|
||||
|
||||
// Submit message (blocked during streaming or when document picker is open)
|
||||
// Submit message (blocked during streaming, document picker open, or AI responding to another user)
|
||||
const handleSubmit = useCallback(() => {
|
||||
if (isThreadRunning) {
|
||||
if (isThreadRunning || isBlockedByOtherUser) {
|
||||
return;
|
||||
}
|
||||
if (!showDocumentPopover) {
|
||||
|
|
@ -315,6 +337,7 @@ const Composer: FC = () => {
|
|||
}, [
|
||||
showDocumentPopover,
|
||||
isThreadRunning,
|
||||
isBlockedByOtherUser,
|
||||
composerRuntime,
|
||||
setMentionedDocuments,
|
||||
setMentionedDocumentIds,
|
||||
|
|
@ -374,7 +397,13 @@ const Composer: FC = () => {
|
|||
);
|
||||
|
||||
return (
|
||||
<ComposerPrimitive.Root className="aui-composer-root relative flex w-full flex-col">
|
||||
<ComposerPrimitive.Root className="aui-composer-root relative flex w-full flex-col gap-2">
|
||||
<ChatSessionStatus
|
||||
isAiResponding={isAiResponding}
|
||||
respondingToUserId={respondingToUserId}
|
||||
currentUserId={currentUser?.id ?? null}
|
||||
members={members ?? []}
|
||||
/>
|
||||
<ComposerPrimitive.AttachmentDropzone className="aui-composer-attachment-dropzone flex w-full flex-col rounded-2xl border-input bg-muted px-1 pt-2 outline-none transition-shadow data-[dragging=true]:border-ring data-[dragging=true]:border-dashed data-[dragging=true]:bg-accent/50">
|
||||
<ComposerAttachments />
|
||||
{/* Inline editor with @mention support */}
|
||||
|
|
@ -417,13 +446,17 @@ const Composer: FC = () => {
|
|||
/>,
|
||||
document.body
|
||||
)}
|
||||
<ComposerAction />
|
||||
<ComposerAction isBlockedByOtherUser={isBlockedByOtherUser} />
|
||||
</ComposerPrimitive.AttachmentDropzone>
|
||||
</ComposerPrimitive.Root>
|
||||
);
|
||||
};
|
||||
|
||||
const ComposerAction: FC = () => {
|
||||
interface ComposerActionProps {
|
||||
isBlockedByOtherUser?: boolean;
|
||||
}
|
||||
|
||||
const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false }) => {
|
||||
// Check if any attachments are still being processed (running AND progress < 100)
|
||||
// When progress is 100, processing is done but waiting for send()
|
||||
const hasProcessingAttachments = useAssistantState(({ composer }) =>
|
||||
|
|
@ -458,7 +491,8 @@ const ComposerAction: FC = () => {
|
|||
return userConfigs?.some((c) => c.id === agentLlmId) ?? false;
|
||||
}, [preferences, globalConfigs, userConfigs]);
|
||||
|
||||
const isSendDisabled = hasProcessingAttachments || isComposerEmpty || !hasModelConfigured;
|
||||
const isSendDisabled =
|
||||
hasProcessingAttachments || isComposerEmpty || !hasModelConfigured || isBlockedByOtherUser;
|
||||
|
||||
return (
|
||||
<div className="aui-composer-action-wrapper relative mx-2 mb-2 flex items-center justify-between">
|
||||
|
|
@ -487,13 +521,15 @@ const ComposerAction: FC = () => {
|
|||
<ComposerPrimitive.Send asChild disabled={isSendDisabled}>
|
||||
<TooltipIconButton
|
||||
tooltip={
|
||||
!hasModelConfigured
|
||||
? "Please select a model from the header to start chatting"
|
||||
: hasProcessingAttachments
|
||||
? "Wait for attachments to process"
|
||||
: isComposerEmpty
|
||||
? "Enter a message to send"
|
||||
: "Send message"
|
||||
isBlockedByOtherUser
|
||||
? "Wait for AI to finish responding"
|
||||
: !hasModelConfigured
|
||||
? "Please select a model from the header to start chatting"
|
||||
: hasProcessingAttachments
|
||||
? "Wait for attachments to process"
|
||||
: isComposerEmpty
|
||||
? "Enter a message to send"
|
||||
: "Send message"
|
||||
}
|
||||
side="bottom"
|
||||
type="submit"
|
||||
|
|
|
|||
|
|
@ -179,7 +179,7 @@ export function DashboardBreadcrumb() {
|
|||
|
||||
const breadcrumbs = generateBreadcrumbs(pathname);
|
||||
|
||||
if (breadcrumbs.length <= 1) {
|
||||
if (breadcrumbs.length === 0) {
|
||||
return null; // Don't show breadcrumbs for root dashboard
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -167,12 +167,6 @@ export function LayoutDataProvider({
|
|||
icon: SquareLibrary,
|
||||
isActive: pathname?.includes("/documents"),
|
||||
},
|
||||
// {
|
||||
// title: "Logs",
|
||||
// url: `/dashboard/${searchSpaceId}/logs`,
|
||||
// icon: Logs,
|
||||
// isActive: pathname?.includes("/logs"),
|
||||
// },
|
||||
{
|
||||
title: "Inbox",
|
||||
url: "#inbox", // Special URL to indicate this is handled differently
|
||||
|
|
|
|||
|
|
@ -41,14 +41,14 @@ import { Spinner } from "@/components/ui/spinner";
|
|||
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||
import type { InboxItem } from "@/hooks/use-inbox";
|
||||
import { useMediaQuery } from "@/hooks/use-media-query";
|
||||
import {
|
||||
type ConnectorIndexingMetadata,
|
||||
type NewMentionMetadata,
|
||||
isConnectorIndexingMetadata,
|
||||
isNewMentionMetadata,
|
||||
type NewMentionMetadata,
|
||||
} from "@/contracts/types/inbox.types";
|
||||
import type { InboxItem } from "@/hooks/use-inbox";
|
||||
import { useMediaQuery } from "@/hooks/use-media-query";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -119,11 +119,6 @@ export function Sidebar({
|
|||
)}
|
||||
</div>
|
||||
|
||||
{/* Platform navigation */}
|
||||
{navItems.length > 0 && (
|
||||
<NavSection items={navItems} onItemClick={onNavItemClick} isCollapsed={isCollapsed} />
|
||||
)}
|
||||
|
||||
{/* Scrollable content */}
|
||||
<ScrollArea className="flex-1">
|
||||
{isCollapsed ? (
|
||||
|
|
@ -235,7 +230,12 @@ export function Sidebar({
|
|||
</ScrollArea>
|
||||
|
||||
{/* Footer */}
|
||||
<div className="mt-auto">
|
||||
<div className="mt-auto border-t">
|
||||
{/* Platform navigation */}
|
||||
{navItems.length > 0 && (
|
||||
<NavSection items={navItems} onItemClick={onNavItemClick} isCollapsed={isCollapsed} />
|
||||
)}
|
||||
|
||||
{pageUsage && !isCollapsed && (
|
||||
<PageUsageDisplay pagesUsed={pageUsage.pagesUsed} pagesLimit={pageUsage.pagesLimit} />
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"use client";
|
||||
|
||||
import { ChevronsUpDown, Settings, Users } from "lucide-react";
|
||||
import { ChevronsUpDown, ScrollText, Settings, Users } from "lucide-react";
|
||||
import { useParams, useRouter } from "next/navigation";
|
||||
import { useTranslations } from "next-intl";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
|
|
@ -29,6 +30,9 @@ export function SidebarHeader({
|
|||
className,
|
||||
}: SidebarHeaderProps) {
|
||||
const t = useTranslations("sidebar");
|
||||
const router = useRouter();
|
||||
const params = useParams();
|
||||
const searchSpaceId = params.search_space_id as string;
|
||||
|
||||
return (
|
||||
<div className={cn("flex shrink-0 items-center", className)}>
|
||||
|
|
@ -52,6 +56,10 @@ export function SidebarHeader({
|
|||
<Users className="mr-2 h-4 w-4" />
|
||||
{t("manage_members")}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onClick={() => router.push(`/dashboard/${searchSpaceId}/logs`)}>
|
||||
<ScrollText className="mr-2 h-4 w-4" />
|
||||
{t("logs")}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuSeparator />
|
||||
<DropdownMenuItem onClick={onSettings}>
|
||||
<Settings className="mr-2 h-4 w-4" />
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import { ChevronUp, Laptop, Languages, LogOut, Moon, Settings, Sun } from "lucide-react";
|
||||
import { ChevronUp, Languages, Laptop, LogOut, Moon, Settings, Sun } from "lucide-react";
|
||||
import { useTranslations } from "next-intl";
|
||||
import {
|
||||
DropdownMenu,
|
||||
|
|
|
|||
|
|
@ -123,6 +123,11 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
|
|||
);
|
||||
}, [userConfigs, searchQuery]);
|
||||
|
||||
// Total model count for conditional search display
|
||||
const totalModels = useMemo(() => {
|
||||
return (globalConfigs?.length ?? 0) + (userConfigs?.length ?? 0);
|
||||
}, [globalConfigs, userConfigs]);
|
||||
|
||||
const handleSelectConfig = useCallback(
|
||||
async (config: NewLLMConfigPublic | GlobalNewLLMConfig) => {
|
||||
// If already selected, just close
|
||||
|
|
@ -212,14 +217,16 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
|
|||
shouldFilter={false}
|
||||
className="rounded-lg relative [&_[data-slot=command-input-wrapper]]:border-0 [&_[data-slot=command-input-wrapper]]:px-0 [&_[data-slot=command-input-wrapper]]:gap-2"
|
||||
>
|
||||
<div className="flex items-center gap-1 md:gap-2 px-2 md:px-3 py-1.5 md:py-2">
|
||||
<CommandInput
|
||||
placeholder="Search models"
|
||||
value={searchQuery}
|
||||
onValueChange={setSearchQuery}
|
||||
className="h-7 md:h-8 text-xs md:text-sm border-0 bg-transparent focus:ring-0 placeholder:text-muted-foreground/60"
|
||||
/>
|
||||
</div>
|
||||
{totalModels > 3 && (
|
||||
<div className="flex items-center gap-1 md:gap-2 px-2 md:px-3 py-1.5 md:py-2">
|
||||
<CommandInput
|
||||
placeholder="Search models"
|
||||
value={searchQuery}
|
||||
onValueChange={setSearchQuery}
|
||||
className="h-7 md:h-8 text-xs md:text-sm border-0 bg-transparent focus:ring-0 placeholder:text-muted-foreground/60"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<CommandList className="max-h-[300px] md:max-h-[400px] overflow-y-auto">
|
||||
<CommandEmpty className="py-8 text-center">
|
||||
|
|
@ -245,7 +252,7 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
|
|||
value={`global-${config.id}`}
|
||||
onSelect={() => handleSelectConfig(config)}
|
||||
className={cn(
|
||||
"mx-2 rounded-lg mb-1 cursor-pointer transition-all",
|
||||
"mx-2 rounded-lg mb-1 cursor-pointer group transition-all",
|
||||
"hover:bg-accent/50",
|
||||
isSelected && "bg-accent/80"
|
||||
)}
|
||||
|
|
@ -276,7 +283,7 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
|
|||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="size-7 shrink-0 rounded-md hover:bg-muted"
|
||||
className="size-7 shrink-0 rounded-md hover:bg-muted opacity-0 group-hover:opacity-100 transition-opacity"
|
||||
onClick={(e) => handleEditConfig(e, config, true)}
|
||||
>
|
||||
<Edit3 className="size-3.5 text-muted-foreground" />
|
||||
|
|
@ -307,7 +314,7 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
|
|||
value={`user-${config.id}`}
|
||||
onSelect={() => handleSelectConfig(config)}
|
||||
className={cn(
|
||||
"mx-2 rounded-lg mb-1 cursor-pointer transition-all",
|
||||
"mx-2 rounded-lg mb-1 cursor-pointer group transition-all",
|
||||
"hover:bg-accent/50",
|
||||
isSelected && "bg-accent/80"
|
||||
)}
|
||||
|
|
@ -338,7 +345,7 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
|
|||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="size-7 shrink-0 rounded-md hover:bg-muted"
|
||||
className="size-7 shrink-0 rounded-md hover:bg-muted opacity-0 group-hover:opacity-100 transition-opacity"
|
||||
onClick={(e) => handleEditConfig(e, config, false)}
|
||||
>
|
||||
<Edit3 className="size-3.5 text-muted-foreground" />
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import * as React from "react";
|
||||
import type * as React from "react";
|
||||
import { Drawer as DrawerPrimitive } from "vaul";
|
||||
|
||||
import { cn } from "@/lib/utils";
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ export enum EnumConnectorName {
|
|||
WEBCRAWLER_CONNECTOR = "WEBCRAWLER_CONNECTOR",
|
||||
YOUTUBE_CONNECTOR = "YOUTUBE_CONNECTOR",
|
||||
CIRCLEBACK_CONNECTOR = "CIRCLEBACK_CONNECTOR",
|
||||
OBSIDIAN_CONNECTOR = "OBSIDIAN_CONNECTOR",
|
||||
MCP_CONNECTOR = "MCP_CONNECTOR",
|
||||
COMPOSIO_GOOGLE_DRIVE_CONNECTOR = "COMPOSIO_GOOGLE_DRIVE_CONNECTOR",
|
||||
COMPOSIO_GMAIL_CONNECTOR = "COMPOSIO_GMAIL_CONNECTOR",
|
||||
|
|
|
|||
|
|
@ -66,6 +66,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas
|
|||
return <IconUsersGroup {...iconProps} />;
|
||||
case EnumConnectorName.MCP_CONNECTOR:
|
||||
return <Image src="/connectors/modelcontextprotocol.svg" alt="MCP" {...imgProps} />;
|
||||
case EnumConnectorName.OBSIDIAN_CONNECTOR:
|
||||
return <Image src="/connectors/obsidian.svg" alt="Obsidian" {...imgProps} />;
|
||||
case EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR:
|
||||
return <Image src="/connectors/google-drive.svg" alt="Google Drive" {...imgProps} />;
|
||||
case EnumConnectorName.COMPOSIO_GMAIL_CONNECTOR:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,19 @@
|
|||
import { z } from "zod";
|
||||
|
||||
/**
|
||||
* Raw comment
|
||||
*/
|
||||
export const rawComment = z.object({
|
||||
id: z.number(),
|
||||
message_id: z.number(),
|
||||
thread_id: z.number(), // Denormalized for efficient Electric subscriptions
|
||||
parent_id: z.number().nullable(),
|
||||
author_id: z.string().nullable(),
|
||||
content: z.string(),
|
||||
created_at: z.string(),
|
||||
updated_at: z.string(),
|
||||
});
|
||||
|
||||
export const author = z.object({
|
||||
id: z.string().uuid(),
|
||||
display_name: z.string().nullable(),
|
||||
|
|
@ -122,6 +136,7 @@ export const getMentionsResponse = z.object({
|
|||
total_count: z.number(),
|
||||
});
|
||||
|
||||
export type RawComment = z.infer<typeof rawComment>;
|
||||
export type Author = z.infer<typeof author>;
|
||||
export type CommentReply = z.infer<typeof commentReply>;
|
||||
export type Comment = z.infer<typeof comment>;
|
||||
|
|
|
|||
15
surfsense_web/contracts/types/chat-messages.types.ts
Normal file
15
surfsense_web/contracts/types/chat-messages.types.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
import { z } from "zod";
|
||||
|
||||
/**
|
||||
* Raw message from database (Electric SQL sync)
|
||||
*/
|
||||
export const rawMessage = z.object({
|
||||
id: z.number(),
|
||||
thread_id: z.number(),
|
||||
role: z.string(),
|
||||
content: z.unknown(),
|
||||
author_id: z.string().nullable(),
|
||||
created_at: z.string(),
|
||||
});
|
||||
|
||||
export type RawMessage = z.infer<typeof rawMessage>;
|
||||
24
surfsense_web/contracts/types/chat-session-state.types.ts
Normal file
24
surfsense_web/contracts/types/chat-session-state.types.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { z } from "zod";
|
||||
|
||||
/**
|
||||
* Chat session state for live collaboration.
|
||||
* Tracks which user the AI is currently responding to.
|
||||
*/
|
||||
export const chatSessionState = z.object({
|
||||
id: z.number(),
|
||||
thread_id: z.number(),
|
||||
ai_responding_to_user_id: z.string().uuid().nullable(),
|
||||
updated_at: z.string(),
|
||||
});
|
||||
|
||||
/**
|
||||
* User currently being responded to by the AI.
|
||||
*/
|
||||
export const respondingUser = z.object({
|
||||
id: z.string().uuid(),
|
||||
display_name: z.string().nullable(),
|
||||
email: z.string(),
|
||||
});
|
||||
|
||||
export type ChatSessionState = z.infer<typeof chatSessionState>;
|
||||
export type RespondingUser = z.infer<typeof respondingUser>;
|
||||
|
|
@ -27,6 +27,7 @@ export const searchSourceConnectorTypeEnum = z.enum([
|
|||
"BOOKSTACK_CONNECTOR",
|
||||
"CIRCLEBACK_CONNECTOR",
|
||||
"MCP_CONNECTOR",
|
||||
"OBSIDIAN_CONNECTOR",
|
||||
"COMPOSIO_GOOGLE_DRIVE_CONNECTOR",
|
||||
"COMPOSIO_GMAIL_CONNECTOR",
|
||||
"COMPOSIO_GOOGLE_CALENDAR_CONNECTOR",
|
||||
|
|
|
|||
39
surfsense_web/hooks/use-chat-session-state.ts
Normal file
39
surfsense_web/hooks/use-chat-session-state.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
"use client";
|
||||
|
||||
import { useShape } from "@electric-sql/react";
|
||||
import { useSetAtom } from "jotai";
|
||||
import { useEffect } from "react";
|
||||
import { chatSessionStateAtom } from "@/atoms/chat/chat-session-state.atom";
|
||||
import type { ChatSessionState } from "@/contracts/types/chat-session-state.types";
|
||||
|
||||
const ELECTRIC_URL = process.env.NEXT_PUBLIC_ELECTRIC_URL || "http://localhost:5133";
|
||||
|
||||
/**
|
||||
* Syncs chat session state for a thread via Electric SQL.
|
||||
* Call once per thread (in page.tsx). Updates global atom.
|
||||
*/
|
||||
export function useChatSessionStateSync(threadId: number | null) {
|
||||
const setSessionState = useSetAtom(chatSessionStateAtom);
|
||||
|
||||
const { data } = useShape<ChatSessionState>({
|
||||
url: `${ELECTRIC_URL}/v1/shape`,
|
||||
params: {
|
||||
table: "chat_session_state",
|
||||
where: `thread_id = ${threadId ?? -1}`,
|
||||
},
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (!threadId) {
|
||||
setSessionState(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const row = data?.[0];
|
||||
setSessionState({
|
||||
threadId,
|
||||
isAiResponding: !!row?.ai_responding_to_user_id,
|
||||
respondingToUserId: row?.ai_responding_to_user_id ?? null,
|
||||
});
|
||||
}, [threadId, data, setSessionState]);
|
||||
}
|
||||
405
surfsense_web/hooks/use-comments-electric.ts
Normal file
405
surfsense_web/hooks/use-comments-electric.ts
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
"use client";
|
||||
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import { useAtomValue } from "jotai";
|
||||
import { useCallback, useEffect, useMemo, useRef } from "react";
|
||||
import { membersAtom, myAccessAtom } from "@/atoms/members/members-query.atoms";
|
||||
import { currentUserAtom } from "@/atoms/user/user-query.atoms";
|
||||
import type { Author, Comment, CommentReply } from "@/contracts/types/chat-comments.types";
|
||||
import type { Membership } from "@/contracts/types/members.types";
|
||||
import type { SyncHandle } from "@/lib/electric/client";
|
||||
import { useElectricClient } from "@/lib/electric/context";
|
||||
import { cacheKeys } from "@/lib/query-client/cache-keys";
|
||||
|
||||
// Debounce delay for stream updates (ms)
|
||||
const STREAM_UPDATE_DEBOUNCE_MS = 100;
|
||||
|
||||
// Raw comment from PGlite local database
|
||||
interface RawCommentRow {
|
||||
id: number;
|
||||
message_id: number;
|
||||
thread_id: number;
|
||||
parent_id: number | null;
|
||||
author_id: string | null;
|
||||
content: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
// Regex pattern to match @[uuid] mentions (matches backend MENTION_PATTERN)
|
||||
const MENTION_PATTERN = /@\[([0-9a-fA-F-]{36})\]/g;
|
||||
|
||||
type MemberInfo = Pick<Membership, "user_display_name" | "user_avatar_url" | "user_email">;
|
||||
|
||||
/**
|
||||
* Render mentions in content by replacing @[uuid] with @{DisplayName}
|
||||
*/
|
||||
function renderMentions(content: string, memberMap: Map<string, MemberInfo>): string {
|
||||
return content.replace(MENTION_PATTERN, (match, uuid) => {
|
||||
const member = memberMap.get(uuid);
|
||||
if (member?.user_display_name) {
|
||||
return `@{${member.user_display_name}}`;
|
||||
}
|
||||
return match;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Build member lookup map from membersData
|
||||
*/
|
||||
function buildMemberMap(membersData: Membership[] | undefined): Map<string, MemberInfo> {
|
||||
const map = new Map<string, MemberInfo>();
|
||||
if (membersData) {
|
||||
for (const m of membersData) {
|
||||
map.set(m.user_id, {
|
||||
user_display_name: m.user_display_name,
|
||||
user_avatar_url: m.user_avatar_url,
|
||||
user_email: m.user_email,
|
||||
});
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build author object from member data
|
||||
*/
|
||||
function buildAuthor(authorId: string | null, memberMap: Map<string, MemberInfo>): Author | null {
|
||||
if (!authorId) return null;
|
||||
const m = memberMap.get(authorId);
|
||||
if (!m) return null;
|
||||
return {
|
||||
id: authorId,
|
||||
display_name: m.user_display_name ?? null,
|
||||
avatar_url: m.user_avatar_url ?? null,
|
||||
email: m.user_email ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a comment has been edited by comparing timestamps.
|
||||
* Uses a small threshold to handle precision differences.
|
||||
*/
|
||||
function isEdited(createdAt: string, updatedAt: string): boolean {
|
||||
const created = new Date(createdAt).getTime();
|
||||
const updated = new Date(updatedAt).getTime();
|
||||
// Consider edited if updated_at is more than 1 second after created_at
|
||||
return updated - created > 1000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform raw comment to CommentReply
|
||||
*/
|
||||
function transformReply(
|
||||
raw: RawCommentRow,
|
||||
memberMap: Map<string, MemberInfo>,
|
||||
currentUserId: string | undefined,
|
||||
isOwner: boolean
|
||||
): CommentReply {
|
||||
return {
|
||||
id: raw.id,
|
||||
content: raw.content,
|
||||
content_rendered: renderMentions(raw.content, memberMap),
|
||||
author: buildAuthor(raw.author_id, memberMap),
|
||||
created_at: raw.created_at,
|
||||
updated_at: raw.updated_at,
|
||||
is_edited: isEdited(raw.created_at, raw.updated_at),
|
||||
can_edit: currentUserId === raw.author_id,
|
||||
can_delete: currentUserId === raw.author_id || isOwner,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform raw comments to Comment with replies
|
||||
*/
|
||||
function transformComments(
|
||||
rawComments: RawCommentRow[],
|
||||
memberMap: Map<string, MemberInfo>,
|
||||
currentUserId: string | undefined,
|
||||
isOwner: boolean
|
||||
): Map<number, Comment[]> {
|
||||
// Group comments by message_id
|
||||
const byMessage = new Map<
|
||||
number,
|
||||
{ topLevel: RawCommentRow[]; replies: Map<number, RawCommentRow[]> }
|
||||
>();
|
||||
|
||||
for (const raw of rawComments) {
|
||||
if (!byMessage.has(raw.message_id)) {
|
||||
byMessage.set(raw.message_id, { topLevel: [], replies: new Map() });
|
||||
}
|
||||
const group = byMessage.get(raw.message_id)!;
|
||||
|
||||
if (raw.parent_id === null) {
|
||||
group.topLevel.push(raw);
|
||||
} else {
|
||||
if (!group.replies.has(raw.parent_id)) {
|
||||
group.replies.set(raw.parent_id, []);
|
||||
}
|
||||
group.replies.get(raw.parent_id)!.push(raw);
|
||||
}
|
||||
}
|
||||
|
||||
// Transform to Comment objects grouped by message_id
|
||||
const result = new Map<number, Comment[]>();
|
||||
|
||||
for (const [messageId, group] of byMessage) {
|
||||
const comments: Comment[] = group.topLevel.map((raw) => {
|
||||
const replies = (group.replies.get(raw.id) || [])
|
||||
.sort((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime())
|
||||
.map((r) => transformReply(r, memberMap, currentUserId, isOwner));
|
||||
|
||||
return {
|
||||
id: raw.id,
|
||||
message_id: raw.message_id,
|
||||
content: raw.content,
|
||||
content_rendered: renderMentions(raw.content, memberMap),
|
||||
author: buildAuthor(raw.author_id, memberMap),
|
||||
created_at: raw.created_at,
|
||||
updated_at: raw.updated_at,
|
||||
is_edited: isEdited(raw.created_at, raw.updated_at),
|
||||
can_edit: currentUserId === raw.author_id,
|
||||
can_delete: currentUserId === raw.author_id || isOwner,
|
||||
reply_count: replies.length,
|
||||
replies,
|
||||
};
|
||||
});
|
||||
|
||||
// Sort by created_at
|
||||
comments.sort((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime());
|
||||
result.set(messageId, comments);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for syncing comments with Electric SQL real-time sync.
|
||||
*
|
||||
* Syncs ALL comments for a thread in ONE subscription, then updates
|
||||
* React Query cache for each message. This avoids N subscriptions for N messages.
|
||||
*
|
||||
* @param threadId - The thread ID to sync comments for
|
||||
*/
|
||||
export function useCommentsElectric(threadId: number | null) {
|
||||
const electricClient = useElectricClient();
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const { data: membersData } = useAtomValue(membersAtom);
|
||||
const { data: currentUser } = useAtomValue(currentUserAtom);
|
||||
const { data: myAccess } = useAtomValue(myAccessAtom);
|
||||
|
||||
const memberMap = useMemo(() => buildMemberMap(membersData), [membersData]);
|
||||
const currentUserId = currentUser?.id;
|
||||
const isOwner = myAccess?.is_owner ?? false;
|
||||
|
||||
// Use refs for values needed in live query callback to avoid stale closures
|
||||
const memberMapRef = useRef(memberMap);
|
||||
const currentUserIdRef = useRef(currentUserId);
|
||||
const isOwnerRef = useRef(isOwner);
|
||||
const queryClientRef = useRef(queryClient);
|
||||
|
||||
// Keep refs updated
|
||||
useEffect(() => {
|
||||
memberMapRef.current = memberMap;
|
||||
currentUserIdRef.current = currentUserId;
|
||||
isOwnerRef.current = isOwner;
|
||||
queryClientRef.current = queryClient;
|
||||
}, [memberMap, currentUserId, isOwner, queryClient]);
|
||||
|
||||
const syncHandleRef = useRef<SyncHandle | null>(null);
|
||||
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null);
|
||||
const syncKeyRef = useRef<string | null>(null);
|
||||
const streamUpdateDebounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
// Stable callback that uses refs for fresh values
|
||||
const updateReactQueryCache = useCallback((rows: RawCommentRow[]) => {
|
||||
const commentsByMessage = transformComments(
|
||||
rows,
|
||||
memberMapRef.current,
|
||||
currentUserIdRef.current,
|
||||
isOwnerRef.current
|
||||
);
|
||||
|
||||
for (const [messageId, comments] of commentsByMessage) {
|
||||
const cacheKey = cacheKeys.comments.byMessage(messageId);
|
||||
queryClientRef.current.setQueryData(cacheKey, {
|
||||
comments,
|
||||
total_count: comments.length,
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!threadId || !electricClient) {
|
||||
return;
|
||||
}
|
||||
|
||||
const syncKey = `comments_${threadId}`;
|
||||
if (syncKeyRef.current === syncKey) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Capture in local variable for use in async functions
|
||||
const client = electricClient;
|
||||
|
||||
let mounted = true;
|
||||
syncKeyRef.current = syncKey;
|
||||
|
||||
async function startSync() {
|
||||
try {
|
||||
const handle = await client.syncShape({
|
||||
table: "chat_comments",
|
||||
where: `thread_id = ${threadId}`,
|
||||
columns: [
|
||||
"id",
|
||||
"message_id",
|
||||
"thread_id",
|
||||
"parent_id",
|
||||
"author_id",
|
||||
"content",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
],
|
||||
primaryKey: ["id"],
|
||||
});
|
||||
|
||||
if (!handle.isUpToDate && handle.initialSyncPromise) {
|
||||
try {
|
||||
await Promise.race([
|
||||
handle.initialSyncPromise,
|
||||
new Promise((resolve) => setTimeout(resolve, 3000)),
|
||||
]);
|
||||
} catch {
|
||||
// Initial sync timeout - continue anyway
|
||||
}
|
||||
}
|
||||
|
||||
if (!mounted) {
|
||||
handle.unsubscribe();
|
||||
return;
|
||||
}
|
||||
|
||||
syncHandleRef.current = handle;
|
||||
|
||||
// Fetch initial comments and update cache
|
||||
await fetchAndUpdateCache();
|
||||
|
||||
// Set up live query for real-time updates
|
||||
await setupLiveQuery();
|
||||
|
||||
// Subscribe to the sync stream for real-time updates from Electric SQL
|
||||
// This ensures we catch updates even if PGlite live query misses them
|
||||
if (handle.stream) {
|
||||
const stream = handle.stream as {
|
||||
subscribe?: (callback: (messages: unknown[]) => void) => void;
|
||||
};
|
||||
if (typeof stream.subscribe === "function") {
|
||||
stream.subscribe((messages: unknown[]) => {
|
||||
if (!mounted) return;
|
||||
// When Electric sync receives new data, refresh from PGlite
|
||||
// This handles cases where live query might miss the update
|
||||
if (messages && messages.length > 0) {
|
||||
// Debounce the refresh to avoid excessive queries
|
||||
if (streamUpdateDebounceRef.current) {
|
||||
clearTimeout(streamUpdateDebounceRef.current);
|
||||
}
|
||||
streamUpdateDebounceRef.current = setTimeout(() => {
|
||||
if (mounted) {
|
||||
fetchAndUpdateCache();
|
||||
}
|
||||
}, STREAM_UPDATE_DEBOUNCE_MS);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Sync failed - will retry on next mount
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchAndUpdateCache() {
|
||||
try {
|
||||
const result = await client.db.query<RawCommentRow>(
|
||||
`SELECT id, message_id, thread_id, parent_id, author_id, content, created_at, updated_at
|
||||
FROM chat_comments
|
||||
WHERE thread_id = $1
|
||||
ORDER BY created_at ASC`,
|
||||
[threadId]
|
||||
);
|
||||
|
||||
if (mounted && result.rows) {
|
||||
updateReactQueryCache(result.rows);
|
||||
}
|
||||
} catch {
|
||||
// Query failed - data will be fetched from API
|
||||
}
|
||||
}
|
||||
|
||||
async function setupLiveQuery() {
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const db = client.db as any;
|
||||
|
||||
if (db.live?.query && typeof db.live.query === "function") {
|
||||
const liveQuery = await db.live.query(
|
||||
`SELECT id, message_id, thread_id, parent_id, author_id, content, created_at, updated_at
|
||||
FROM chat_comments
|
||||
WHERE thread_id = $1
|
||||
ORDER BY created_at ASC`,
|
||||
[threadId]
|
||||
);
|
||||
|
||||
if (!mounted) {
|
||||
liveQuery.unsubscribe?.();
|
||||
return;
|
||||
}
|
||||
|
||||
// Set initial results
|
||||
if (liveQuery.initialResults?.rows) {
|
||||
updateReactQueryCache(liveQuery.initialResults.rows);
|
||||
} else if (liveQuery.rows) {
|
||||
updateReactQueryCache(liveQuery.rows);
|
||||
}
|
||||
|
||||
// Subscribe to changes
|
||||
if (typeof liveQuery.subscribe === "function") {
|
||||
liveQuery.subscribe((result: { rows: RawCommentRow[] }) => {
|
||||
if (mounted && result.rows) {
|
||||
updateReactQueryCache(result.rows);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (typeof liveQuery.unsubscribe === "function") {
|
||||
liveQueryRef.current = liveQuery;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Live query setup failed - will use initial fetch only
|
||||
}
|
||||
}
|
||||
|
||||
startSync();
|
||||
|
||||
return () => {
|
||||
mounted = false;
|
||||
syncKeyRef.current = null;
|
||||
|
||||
// Clear debounce timeout
|
||||
if (streamUpdateDebounceRef.current) {
|
||||
clearTimeout(streamUpdateDebounceRef.current);
|
||||
streamUpdateDebounceRef.current = null;
|
||||
}
|
||||
|
||||
if (syncHandleRef.current) {
|
||||
syncHandleRef.current.unsubscribe();
|
||||
syncHandleRef.current = null;
|
||||
}
|
||||
if (liveQueryRef.current) {
|
||||
liveQueryRef.current.unsubscribe();
|
||||
liveQueryRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [threadId, electricClient, updateReactQueryCache]);
|
||||
}
|
||||
154
surfsense_web/hooks/use-messages-electric.ts
Normal file
154
surfsense_web/hooks/use-messages-electric.ts
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useRef } from "react";
|
||||
import type { RawMessage } from "@/contracts/types/chat-messages.types";
|
||||
import type { SyncHandle } from "@/lib/electric/client";
|
||||
import { useElectricClient } from "@/lib/electric/context";
|
||||
|
||||
/**
|
||||
* Syncs chat messages for a thread via Electric SQL.
|
||||
* Calls onMessagesUpdate when messages change.
|
||||
*/
|
||||
export function useMessagesElectric(
|
||||
threadId: number | null,
|
||||
onMessagesUpdate: (messages: RawMessage[]) => void
|
||||
) {
|
||||
const electricClient = useElectricClient();
|
||||
|
||||
const syncHandleRef = useRef<SyncHandle | null>(null);
|
||||
const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null);
|
||||
const syncKeyRef = useRef<string | null>(null);
|
||||
const onMessagesUpdateRef = useRef(onMessagesUpdate);
|
||||
|
||||
useEffect(() => {
|
||||
onMessagesUpdateRef.current = onMessagesUpdate;
|
||||
}, [onMessagesUpdate]);
|
||||
|
||||
const handleMessagesUpdate = useCallback((rows: RawMessage[]) => {
|
||||
onMessagesUpdateRef.current(rows);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!threadId || !electricClient) {
|
||||
return;
|
||||
}
|
||||
|
||||
const syncKey = `messages_${threadId}`;
|
||||
if (syncKeyRef.current === syncKey) {
|
||||
return;
|
||||
}
|
||||
|
||||
const client = electricClient;
|
||||
let mounted = true;
|
||||
syncKeyRef.current = syncKey;
|
||||
|
||||
async function startSync() {
|
||||
try {
|
||||
const handle = await client.syncShape({
|
||||
table: "new_chat_messages",
|
||||
where: `thread_id = ${threadId}`,
|
||||
columns: ["id", "thread_id", "role", "content", "author_id", "created_at"],
|
||||
primaryKey: ["id"],
|
||||
});
|
||||
|
||||
if (!handle.isUpToDate && handle.initialSyncPromise) {
|
||||
try {
|
||||
await Promise.race([
|
||||
handle.initialSyncPromise,
|
||||
new Promise((resolve) => setTimeout(resolve, 3000)),
|
||||
]);
|
||||
} catch {
|
||||
// Timeout
|
||||
}
|
||||
}
|
||||
|
||||
if (!mounted) {
|
||||
handle.unsubscribe();
|
||||
return;
|
||||
}
|
||||
|
||||
syncHandleRef.current = handle;
|
||||
await fetchMessages();
|
||||
await setupLiveQuery();
|
||||
} catch {
|
||||
// Sync failed
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchMessages() {
|
||||
try {
|
||||
const result = await client.db.query<RawMessage>(
|
||||
`SELECT id, thread_id, role, content, author_id, created_at
|
||||
FROM new_chat_messages
|
||||
WHERE thread_id = $1
|
||||
ORDER BY created_at ASC`,
|
||||
[threadId]
|
||||
);
|
||||
|
||||
if (mounted && result.rows) {
|
||||
handleMessagesUpdate(result.rows);
|
||||
}
|
||||
} catch {
|
||||
// Query failed
|
||||
}
|
||||
}
|
||||
|
||||
async function setupLiveQuery() {
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const db = client.db as any;
|
||||
|
||||
if (db.live?.query && typeof db.live.query === "function") {
|
||||
const liveQuery = await db.live.query(
|
||||
`SELECT id, thread_id, role, content, author_id, created_at
|
||||
FROM new_chat_messages
|
||||
WHERE thread_id = $1
|
||||
ORDER BY created_at ASC`,
|
||||
[threadId]
|
||||
);
|
||||
|
||||
if (!mounted) {
|
||||
liveQuery.unsubscribe?.();
|
||||
return;
|
||||
}
|
||||
|
||||
if (liveQuery.initialResults?.rows) {
|
||||
handleMessagesUpdate(liveQuery.initialResults.rows);
|
||||
} else if (liveQuery.rows) {
|
||||
handleMessagesUpdate(liveQuery.rows);
|
||||
}
|
||||
|
||||
if (typeof liveQuery.subscribe === "function") {
|
||||
liveQuery.subscribe((result: { rows: RawMessage[] }) => {
|
||||
if (mounted && result.rows) {
|
||||
handleMessagesUpdate(result.rows);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (typeof liveQuery.unsubscribe === "function") {
|
||||
liveQueryRef.current = liveQuery;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Live query failed
|
||||
}
|
||||
}
|
||||
|
||||
startSync();
|
||||
|
||||
return () => {
|
||||
mounted = false;
|
||||
syncKeyRef.current = null;
|
||||
|
||||
if (syncHandleRef.current) {
|
||||
syncHandleRef.current.unsubscribe();
|
||||
syncHandleRef.current = null;
|
||||
}
|
||||
if (liveQueryRef.current) {
|
||||
liveQueryRef.current.unsubscribe();
|
||||
liveQueryRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [threadId, electricClient, handleMessagesUpdate]);
|
||||
}
|
||||
|
|
@ -2,12 +2,12 @@ import {
|
|||
type GetNotificationsRequest,
|
||||
type GetNotificationsResponse,
|
||||
type GetUnreadCountResponse,
|
||||
type MarkAllNotificationsReadResponse,
|
||||
type MarkNotificationReadRequest,
|
||||
type MarkNotificationReadResponse,
|
||||
getNotificationsRequest,
|
||||
getNotificationsResponse,
|
||||
getUnreadCountResponse,
|
||||
type MarkAllNotificationsReadResponse,
|
||||
type MarkNotificationReadRequest,
|
||||
type MarkNotificationReadResponse,
|
||||
markAllNotificationsReadResponse,
|
||||
markNotificationReadRequest,
|
||||
markNotificationReadResponse,
|
||||
|
|
|
|||
|
|
@ -229,7 +229,6 @@ export async function initElectric(userId: string): Promise<ElectricClient> {
|
|||
CREATE INDEX IF NOT EXISTS idx_documents_search_space_type ON documents(search_space_id, document_type);
|
||||
`);
|
||||
|
||||
// Create the chat_comment_mentions table schema in PGlite
|
||||
await db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS chat_comment_mentions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
|
|
@ -242,6 +241,39 @@ export async function initElectric(userId: string): Promise<ElectricClient> {
|
|||
CREATE INDEX IF NOT EXISTS idx_chat_comment_mentions_comment_id ON chat_comment_mentions(comment_id);
|
||||
`);
|
||||
|
||||
// Create chat_comments table for live comment sync
|
||||
await db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS chat_comments (
|
||||
id INTEGER PRIMARY KEY,
|
||||
message_id INTEGER NOT NULL,
|
||||
thread_id INTEGER NOT NULL,
|
||||
parent_id INTEGER,
|
||||
author_id TEXT,
|
||||
content TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_comments_thread_id ON chat_comments(thread_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_comments_message_id ON chat_comments(message_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_comments_parent_id ON chat_comments(parent_id);
|
||||
`);
|
||||
|
||||
// Create new_chat_messages table for live message sync
|
||||
await db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS new_chat_messages (
|
||||
id INTEGER PRIMARY KEY,
|
||||
thread_id INTEGER NOT NULL,
|
||||
role TEXT NOT NULL,
|
||||
content JSONB NOT NULL,
|
||||
author_id TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_new_chat_messages_thread_id ON new_chat_messages(thread_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_new_chat_messages_created_at ON new_chat_messages(created_at);
|
||||
`);
|
||||
|
||||
const electricUrl = getElectricUrl();
|
||||
|
||||
// STEP 4: Create the client wrapper
|
||||
|
|
|
|||
|
|
@ -21,8 +21,21 @@ export const BACKEND_URL = process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http:
|
|||
// Placeholder: __NEXT_PUBLIC_ETL_SERVICE__
|
||||
export const ETL_SERVICE = process.env.NEXT_PUBLIC_ETL_SERVICE || "DOCLING";
|
||||
|
||||
// Deployment Mode: "self-hosted" or "cloud"
|
||||
// Matches backend's SURFSENSE_DEPLOYMENT_MODE - defaults to "self-hosted"
|
||||
// self-hosted: Full access to local file system connectors (Obsidian, etc.)
|
||||
// cloud: Only cloud-based connectors available
|
||||
// Placeholder: __NEXT_PUBLIC_DEPLOYMENT_MODE__
|
||||
export const DEPLOYMENT_MODE = process.env.NEXT_PUBLIC_DEPLOYMENT_MODE || "self-hosted";
|
||||
|
||||
// Helper to check if local auth is enabled
|
||||
export const isLocalAuth = () => AUTH_TYPE === "LOCAL";
|
||||
|
||||
// Helper to check if Google auth is enabled
|
||||
export const isGoogleAuth = () => AUTH_TYPE === "GOOGLE";
|
||||
|
||||
// Helper to check if running in self-hosted mode
|
||||
export const isSelfHosted = () => DEPLOYMENT_MODE === "self-hosted";
|
||||
|
||||
// Helper to check if running in cloud mode
|
||||
export const isCloud = () => DEPLOYMENT_MODE === "cloud";
|
||||
|
|
|
|||
|
|
@ -683,6 +683,7 @@
|
|||
"select_search_space": "Select Search Space",
|
||||
"manage_members": "Manage members",
|
||||
"search_space_settings": "Search Space settings",
|
||||
"logs": "Logs",
|
||||
"see_all_search_spaces": "See all search spaces",
|
||||
"expand_sidebar": "Expand sidebar",
|
||||
"collapse_sidebar": "Collapse sidebar",
|
||||
|
|
|
|||
|
|
@ -668,6 +668,7 @@
|
|||
"select_search_space": "选择搜索空间",
|
||||
"manage_members": "管理成员",
|
||||
"search_space_settings": "搜索空间设置",
|
||||
"logs": "日志",
|
||||
"see_all_search_spaces": "查看所有搜索空间",
|
||||
"expand_sidebar": "展开侧边栏",
|
||||
"collapse_sidebar": "收起侧边栏",
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@
|
|||
"@electric-sql/react": "^1.0.26",
|
||||
"@hookform/resolvers": "^5.2.2",
|
||||
"@number-flow/react": "^0.5.10",
|
||||
"@posthog/react": "^1.5.2",
|
||||
"@posthog/react": "^1.7.0",
|
||||
"@radix-ui/react-accordion": "^1.2.11",
|
||||
"@radix-ui/react-alert-dialog": "^1.1.14",
|
||||
"@radix-ui/react-avatar": "^1.1.10",
|
||||
|
|
@ -86,8 +86,8 @@
|
|||
"next-themes": "^0.4.6",
|
||||
"pg": "^8.16.3",
|
||||
"postgres": "^3.4.7",
|
||||
"posthog-js": "^1.310.1",
|
||||
"posthog-node": "^5.18.0",
|
||||
"posthog-js": "^1.334.1",
|
||||
"posthog-node": "^5.24.1",
|
||||
"react": "^19.2.3",
|
||||
"react-day-picker": "^9.8.1",
|
||||
"react-dom": "^19.2.3",
|
||||
|
|
|
|||
289
surfsense_web/pnpm-lock.yaml
generated
289
surfsense_web/pnpm-lock.yaml
generated
|
|
@ -51,8 +51,8 @@ importers:
|
|||
specifier: ^0.5.10
|
||||
version: 0.5.10(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
|
||||
'@posthog/react':
|
||||
specifier: ^1.5.2
|
||||
version: 1.5.2(@types/react@19.2.7)(posthog-js@1.310.1)(react@19.2.3)
|
||||
specifier: ^1.7.0
|
||||
version: 1.7.0(@types/react@19.2.7)(posthog-js@1.334.1)(react@19.2.3)
|
||||
'@radix-ui/react-accordion':
|
||||
specifier: ^1.2.11
|
||||
version: 1.2.12(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
|
||||
|
|
@ -204,11 +204,11 @@ importers:
|
|||
specifier: ^3.4.7
|
||||
version: 3.4.7
|
||||
posthog-js:
|
||||
specifier: ^1.310.1
|
||||
version: 1.310.1
|
||||
specifier: ^1.334.1
|
||||
version: 1.334.1
|
||||
posthog-node:
|
||||
specifier: ^5.18.0
|
||||
version: 5.18.0
|
||||
specifier: ^5.24.1
|
||||
version: 5.24.1
|
||||
react:
|
||||
specifier: ^19.2.3
|
||||
version: 19.2.3
|
||||
|
|
@ -1447,10 +1447,78 @@ packages:
|
|||
react: ^18 || ^19
|
||||
react-dom: ^18 || ^19
|
||||
|
||||
'@opentelemetry/api-logs@0.208.0':
|
||||
resolution: {integrity: sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==}
|
||||
engines: {node: '>=8.0.0'}
|
||||
|
||||
'@opentelemetry/api@1.9.0':
|
||||
resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==}
|
||||
engines: {node: '>=8.0.0'}
|
||||
|
||||
'@opentelemetry/core@2.2.0':
|
||||
resolution: {integrity: sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.0.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/core@2.5.0':
|
||||
resolution: {integrity: sha512-ka4H8OM6+DlUhSAZpONu0cPBtPPTQKxbxVzC4CzVx5+K4JnroJVBtDzLAMx4/3CDTJXRvVFhpFjtl4SaiTNoyQ==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.0.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/exporter-logs-otlp-http@0.208.0':
|
||||
resolution: {integrity: sha512-jOv40Bs9jy9bZVLo/i8FwUiuCvbjWDI+ZW13wimJm4LjnlwJxGgB+N/VWOZUTpM+ah/awXeQqKdNlpLf2EjvYg==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': ^1.3.0
|
||||
|
||||
'@opentelemetry/otlp-exporter-base@0.208.0':
|
||||
resolution: {integrity: sha512-gMd39gIfVb2OgxldxUtOwGJYSH8P1kVFFlJLuut32L6KgUC4gl1dMhn+YC2mGn0bDOiQYSk/uHOdSjuKp58vvA==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': ^1.3.0
|
||||
|
||||
'@opentelemetry/otlp-transformer@0.208.0':
|
||||
resolution: {integrity: sha512-DCFPY8C6lAQHUNkzcNT9R+qYExvsk6C5Bto2pbNxgicpcSWbe2WHShLxkOxIdNcBiYPdVHv/e7vH7K6TI+C+fQ==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': ^1.3.0
|
||||
|
||||
'@opentelemetry/resources@2.2.0':
|
||||
resolution: {integrity: sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.3.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/resources@2.5.0':
|
||||
resolution: {integrity: sha512-F8W52ApePshpoSrfsSk1H2yJn9aKjCrbpQF1M9Qii0GHzbfVeFUB+rc3X4aggyZD8x9Gu3Slua+s6krmq6Dt8g==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.3.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/sdk-logs@0.208.0':
|
||||
resolution: {integrity: sha512-QlAyL1jRpOeaqx7/leG1vJMp84g0xKP6gJmfELBpnI4O/9xPX+Hu5m1POk9Kl+veNkyth5t19hRlN6tNY1sjbA==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.4.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/sdk-metrics@2.2.0':
|
||||
resolution: {integrity: sha512-G5KYP6+VJMZzpGipQw7Giif48h6SGQ2PFKEYCybeXJsOCB4fp8azqMAAzE5lnnHK3ZVwYQrgmFbsUJO/zOnwGw==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.9.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/sdk-trace-base@2.2.0':
|
||||
resolution: {integrity: sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw==}
|
||||
engines: {node: ^18.19.0 || >=20.6.0}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.3.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/semantic-conventions@1.39.0':
|
||||
resolution: {integrity: sha512-R5R9tb2AXs2IRLNKLBJDynhkfmx7mX0vi8NkhZb3gUkPWHn6HXk5J8iQ/dql0U3ApfWym4kXXmBDRGO+oeOfjg==}
|
||||
engines: {node: '>=14'}
|
||||
|
||||
'@orama/orama@3.1.18':
|
||||
resolution: {integrity: sha512-a61ljmRVVyG5MC/698C8/FfFDw5a8LOIvyOLW5fztgUXqUpc1jOfQzOitSCbge657OgXXThmY3Tk8fpiDb4UcA==}
|
||||
engines: {node: '>= 20.0.0'}
|
||||
|
|
@ -1537,11 +1605,11 @@ packages:
|
|||
resolution: {integrity: sha512-dfUnCxiN9H4ap84DvD2ubjw+3vUNpstxa0TneY/Paat8a3R4uQZDLSvWjmznAY/DoahqTHl9V46HF/Zs3F29pg==}
|
||||
engines: {node: '>= 10.0.0'}
|
||||
|
||||
'@posthog/core@1.9.0':
|
||||
resolution: {integrity: sha512-j7KSWxJTUtNyKynLt/p0hfip/3I46dWU2dk+pt7dKRoz2l5CYueHuHK4EO7Wlgno5yo1HO4sc4s30MXMTICHJw==}
|
||||
'@posthog/core@1.13.0':
|
||||
resolution: {integrity: sha512-knjncrk7qRmssFRbGzBl1Tunt21GRpe0Wv+uVelyL0Rh7PdQUsgguulzXFTps8hA6wPwTU4kq85qnbAJ3eH6Wg==}
|
||||
|
||||
'@posthog/react@1.5.2':
|
||||
resolution: {integrity: sha512-KHdXbV1yba7Y2l8BVmwXlySWxqKVLNQ5ZiVvWOf7r3Eo7GIFxCM4CaNK/z83kKWn8KTskmKy7AGF6Hl6INWK3g==}
|
||||
'@posthog/react@1.7.0':
|
||||
resolution: {integrity: sha512-pM7GL7z/rKjiIwosbRiQA3buhLI6vUo+wg+T/ZrVZC7O5bVU07TfgNZTcuOj8E9dx7vDbfNrc1kjDN7PKMM8ug==}
|
||||
peerDependencies:
|
||||
'@types/react': '>=16.8.0'
|
||||
posthog-js: '>=1.257.2'
|
||||
|
|
@ -1550,6 +1618,9 @@ packages:
|
|||
'@types/react':
|
||||
optional: true
|
||||
|
||||
'@posthog/types@1.334.1':
|
||||
resolution: {integrity: sha512-ypFnwTO7qbV7icylLbujbamPdQXbJq0a61GUUBnJAeTbBw/qYPIss5IRYICcbCj0uunQrwD7/CGxVb5TOYKWgA==}
|
||||
|
||||
'@prisma/client@4.8.1':
|
||||
resolution: {integrity: sha512-d4xhZhETmeXK/yZ7K0KcVOzEfI5YKGGEr4F5SBV04/MU4ncN/HcE28sy3e4Yt8UFW0ZuImKFQJE+9rWt9WbGSQ==}
|
||||
engines: {node: '>=14.17'}
|
||||
|
|
@ -1562,6 +1633,36 @@ packages:
|
|||
'@prisma/engines-version@4.8.0-61.d6e67a83f971b175a593ccc12e15c4a757f93ffe':
|
||||
resolution: {integrity: sha512-MHSOSexomRMom8QN4t7bu87wPPD+pa+hW9+71JnVcF3DqyyO/ycCLhRL1we3EojRpZxKvuyGho2REQsMCvxcJw==}
|
||||
|
||||
'@protobufjs/aspromise@1.1.2':
|
||||
resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
|
||||
|
||||
'@protobufjs/base64@1.1.2':
|
||||
resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}
|
||||
|
||||
'@protobufjs/codegen@2.0.4':
|
||||
resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==}
|
||||
|
||||
'@protobufjs/eventemitter@1.1.0':
|
||||
resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==}
|
||||
|
||||
'@protobufjs/fetch@1.1.0':
|
||||
resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==}
|
||||
|
||||
'@protobufjs/float@1.0.2':
|
||||
resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}
|
||||
|
||||
'@protobufjs/inquire@1.1.0':
|
||||
resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==}
|
||||
|
||||
'@protobufjs/path@1.1.2':
|
||||
resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}
|
||||
|
||||
'@protobufjs/pool@1.1.0':
|
||||
resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}
|
||||
|
||||
'@protobufjs/utf8@1.1.0':
|
||||
resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==}
|
||||
|
||||
'@radix-ui/number@1.1.1':
|
||||
resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==}
|
||||
|
||||
|
|
@ -4923,6 +5024,9 @@ packages:
|
|||
lodash.merge@4.6.2:
|
||||
resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==}
|
||||
|
||||
long@5.3.2:
|
||||
resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==}
|
||||
|
||||
longest-streak@3.1.0:
|
||||
resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==}
|
||||
|
||||
|
|
@ -5497,12 +5601,12 @@ packages:
|
|||
resolution: {integrity: sha512-Jtc2612XINuBjIl/QTWsV5UvE8UHuNblcO3vVADSrKsrc6RqGX6lOW1cEo3CM2v0XG4Nat8nI+YM7/f26VxXLw==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
posthog-js@1.310.1:
|
||||
resolution: {integrity: sha512-UkR6zzlWNtqHDXHJl2Yk062DOmZyVKTPL5mX4j4V+u3RiYbMHJe47+PpMMUsvK1R2e1r/m9uSlHaJMJRzyUjGg==}
|
||||
posthog-js@1.334.1:
|
||||
resolution: {integrity: sha512-5cDzLICr2afnwX/cR9fwoLC0vN0Nb5gP5HiCigzHkgHdO+E3WsYefla3EFMQz7U4r01CBPZ+nZ9/srkzeACxtQ==}
|
||||
|
||||
posthog-node@5.18.0:
|
||||
resolution: {integrity: sha512-SLBEs+sCThxzTGSSDEe97nZHuFFYh6DupObR1yQdvQND3CJh0ogZ0Sa1Vb+Tbrnf0cWbfBC9XNkm44yhaWf3aA==}
|
||||
engines: {node: '>=20'}
|
||||
posthog-node@5.24.1:
|
||||
resolution: {integrity: sha512-1+wsosb5fjuor9zpp3h2uq0xKYY7rDz8gpw/10Scz8Ob/uVNrsHSwGy76D9rgt4cfyaEgpJwyYv+hPi2+YjWtw==}
|
||||
engines: {node: ^20.20.0 || >=22.22.0}
|
||||
|
||||
preact@10.28.1:
|
||||
resolution: {integrity: sha512-u1/ixq/lVQI0CakKNvLDEcW5zfCjUQfZdK9qqWuIJtsezuyG6pk9TWj75GMuI/EzRSZB/VAE43sNWWZfiy8psw==}
|
||||
|
|
@ -5626,6 +5730,10 @@ packages:
|
|||
prosemirror-view@1.41.4:
|
||||
resolution: {integrity: sha512-WkKgnyjNncri03Gjaz3IFWvCAE94XoiEgvtr0/r2Xw7R8/IjK3sKLSiDoCHWcsXSAinVaKlGRZDvMCsF1kbzjA==}
|
||||
|
||||
protobufjs@7.5.4:
|
||||
resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==}
|
||||
engines: {node: '>=12.0.0'}
|
||||
|
||||
pump@3.0.3:
|
||||
resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==}
|
||||
|
||||
|
|
@ -5637,6 +5745,9 @@ packages:
|
|||
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
query-selector-shadow-dom@1.0.1:
|
||||
resolution: {integrity: sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==}
|
||||
|
||||
queue-microtask@1.2.3:
|
||||
resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
|
||||
|
||||
|
|
@ -6472,8 +6583,8 @@ packages:
|
|||
web-namespaces@2.0.1:
|
||||
resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==}
|
||||
|
||||
web-vitals@4.2.4:
|
||||
resolution: {integrity: sha512-r4DIlprAGwJ7YM11VZp4R884m0Vmgr6EAKe3P+kO0PPj3Unqyvv59rczf6UiGcb9Z8QxZVcqKNwv/g0WNdWwsw==}
|
||||
web-vitals@5.1.0:
|
||||
resolution: {integrity: sha512-ArI3kx5jI0atlTtmV0fWU3fjpLmq/nD3Zr1iFFlJLaqa5wLBkUSzINwBPySCX/8jRyjlmy1Volw1kz1g9XE4Jg==}
|
||||
|
||||
webidl-conversions@7.0.0:
|
||||
resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==}
|
||||
|
|
@ -7611,8 +7722,82 @@ snapshots:
|
|||
react: 19.2.3
|
||||
react-dom: 19.2.3(react@19.2.3)
|
||||
|
||||
'@opentelemetry/api-logs@0.208.0':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
|
||||
'@opentelemetry/api@1.9.0': {}
|
||||
|
||||
'@opentelemetry/core@2.2.0(@opentelemetry/api@1.9.0)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/semantic-conventions': 1.39.0
|
||||
|
||||
'@opentelemetry/core@2.5.0(@opentelemetry/api@1.9.0)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/semantic-conventions': 1.39.0
|
||||
|
||||
'@opentelemetry/exporter-logs-otlp-http@0.208.0(@opentelemetry/api@1.9.0)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/api-logs': 0.208.0
|
||||
'@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/otlp-exporter-base': 0.208.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/otlp-transformer': 0.208.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/sdk-logs': 0.208.0(@opentelemetry/api@1.9.0)
|
||||
|
||||
'@opentelemetry/otlp-exporter-base@0.208.0(@opentelemetry/api@1.9.0)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/otlp-transformer': 0.208.0(@opentelemetry/api@1.9.0)
|
||||
|
||||
'@opentelemetry/otlp-transformer@0.208.0(@opentelemetry/api@1.9.0)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/api-logs': 0.208.0
|
||||
'@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/sdk-logs': 0.208.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/sdk-metrics': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/sdk-trace-base': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
protobufjs: 7.5.4
|
||||
|
||||
'@opentelemetry/resources@2.2.0(@opentelemetry/api@1.9.0)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/semantic-conventions': 1.39.0
|
||||
|
||||
'@opentelemetry/resources@2.5.0(@opentelemetry/api@1.9.0)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/core': 2.5.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/semantic-conventions': 1.39.0
|
||||
|
||||
'@opentelemetry/sdk-logs@0.208.0(@opentelemetry/api@1.9.0)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/api-logs': 0.208.0
|
||||
'@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
|
||||
'@opentelemetry/sdk-metrics@2.2.0(@opentelemetry/api@1.9.0)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
|
||||
'@opentelemetry/sdk-trace-base@2.2.0(@opentelemetry/api@1.9.0)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/semantic-conventions': 1.39.0
|
||||
|
||||
'@opentelemetry/semantic-conventions@1.39.0': {}
|
||||
|
||||
'@orama/orama@3.1.18': {}
|
||||
|
||||
'@parcel/watcher-android-arm64@2.5.1':
|
||||
|
|
@ -7675,17 +7860,19 @@ snapshots:
|
|||
'@parcel/watcher-win32-ia32': 2.5.1
|
||||
'@parcel/watcher-win32-x64': 2.5.1
|
||||
|
||||
'@posthog/core@1.9.0':
|
||||
'@posthog/core@1.13.0':
|
||||
dependencies:
|
||||
cross-spawn: 7.0.6
|
||||
|
||||
'@posthog/react@1.5.2(@types/react@19.2.7)(posthog-js@1.310.1)(react@19.2.3)':
|
||||
'@posthog/react@1.7.0(@types/react@19.2.7)(posthog-js@1.334.1)(react@19.2.3)':
|
||||
dependencies:
|
||||
posthog-js: 1.310.1
|
||||
posthog-js: 1.334.1
|
||||
react: 19.2.3
|
||||
optionalDependencies:
|
||||
'@types/react': 19.2.7
|
||||
|
||||
'@posthog/types@1.334.1': {}
|
||||
|
||||
'@prisma/client@4.8.1':
|
||||
dependencies:
|
||||
'@prisma/engines-version': 4.8.0-61.d6e67a83f971b175a593ccc12e15c4a757f93ffe
|
||||
|
|
@ -7694,6 +7881,29 @@ snapshots:
|
|||
'@prisma/engines-version@4.8.0-61.d6e67a83f971b175a593ccc12e15c4a757f93ffe':
|
||||
optional: true
|
||||
|
||||
'@protobufjs/aspromise@1.1.2': {}
|
||||
|
||||
'@protobufjs/base64@1.1.2': {}
|
||||
|
||||
'@protobufjs/codegen@2.0.4': {}
|
||||
|
||||
'@protobufjs/eventemitter@1.1.0': {}
|
||||
|
||||
'@protobufjs/fetch@1.1.0':
|
||||
dependencies:
|
||||
'@protobufjs/aspromise': 1.1.2
|
||||
'@protobufjs/inquire': 1.1.0
|
||||
|
||||
'@protobufjs/float@1.0.2': {}
|
||||
|
||||
'@protobufjs/inquire@1.1.0': {}
|
||||
|
||||
'@protobufjs/path@1.1.2': {}
|
||||
|
||||
'@protobufjs/pool@1.1.0': {}
|
||||
|
||||
'@protobufjs/utf8@1.1.0': {}
|
||||
|
||||
'@radix-ui/number@1.1.1': {}
|
||||
|
||||
'@radix-ui/primitive@1.0.0':
|
||||
|
|
@ -11383,6 +11593,8 @@ snapshots:
|
|||
|
||||
lodash.merge@4.6.2: {}
|
||||
|
||||
long@5.3.2: {}
|
||||
|
||||
longest-streak@3.1.0: {}
|
||||
|
||||
loose-envify@1.4.0:
|
||||
|
|
@ -12272,17 +12484,25 @@ snapshots:
|
|||
|
||||
postgres@3.4.7: {}
|
||||
|
||||
posthog-js@1.310.1:
|
||||
posthog-js@1.334.1:
|
||||
dependencies:
|
||||
'@posthog/core': 1.9.0
|
||||
'@opentelemetry/api': 1.9.0
|
||||
'@opentelemetry/api-logs': 0.208.0
|
||||
'@opentelemetry/exporter-logs-otlp-http': 0.208.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/resources': 2.5.0(@opentelemetry/api@1.9.0)
|
||||
'@opentelemetry/sdk-logs': 0.208.0(@opentelemetry/api@1.9.0)
|
||||
'@posthog/core': 1.13.0
|
||||
'@posthog/types': 1.334.1
|
||||
core-js: 3.47.0
|
||||
dompurify: 3.3.1
|
||||
fflate: 0.4.8
|
||||
preact: 10.28.1
|
||||
web-vitals: 4.2.4
|
||||
query-selector-shadow-dom: 1.0.1
|
||||
web-vitals: 5.1.0
|
||||
|
||||
posthog-node@5.18.0:
|
||||
posthog-node@5.24.1:
|
||||
dependencies:
|
||||
'@posthog/core': 1.9.0
|
||||
'@posthog/core': 1.13.0
|
||||
|
||||
preact@10.28.1: {}
|
||||
|
||||
|
|
@ -12433,6 +12653,21 @@ snapshots:
|
|||
prosemirror-state: 1.4.4
|
||||
prosemirror-transform: 1.10.5
|
||||
|
||||
protobufjs@7.5.4:
|
||||
dependencies:
|
||||
'@protobufjs/aspromise': 1.1.2
|
||||
'@protobufjs/base64': 1.1.2
|
||||
'@protobufjs/codegen': 2.0.4
|
||||
'@protobufjs/eventemitter': 1.1.0
|
||||
'@protobufjs/fetch': 1.1.0
|
||||
'@protobufjs/float': 1.0.2
|
||||
'@protobufjs/inquire': 1.1.0
|
||||
'@protobufjs/path': 1.1.2
|
||||
'@protobufjs/pool': 1.1.0
|
||||
'@protobufjs/utf8': 1.1.0
|
||||
'@types/node': 20.19.27
|
||||
long: 5.3.2
|
||||
|
||||
pump@3.0.3:
|
||||
dependencies:
|
||||
end-of-stream: 1.4.5
|
||||
|
|
@ -12443,6 +12678,8 @@ snapshots:
|
|||
|
||||
punycode@2.3.1: {}
|
||||
|
||||
query-selector-shadow-dom@1.0.1: {}
|
||||
|
||||
queue-microtask@1.2.3: {}
|
||||
|
||||
rc@1.2.8:
|
||||
|
|
@ -13524,7 +13761,7 @@ snapshots:
|
|||
|
||||
web-namespaces@2.0.1: {}
|
||||
|
||||
web-vitals@4.2.4: {}
|
||||
web-vitals@5.1.0: {}
|
||||
|
||||
webidl-conversions@7.0.0: {}
|
||||
|
||||
|
|
|
|||
12
surfsense_web/public/connectors/obsidian.svg
Normal file
12
surfsense_web/public/connectors/obsidian.svg
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100" fill="none">
|
||||
<defs>
|
||||
<linearGradient id="obsidian-gradient" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#7C3AED"/>
|
||||
<stop offset="100%" style="stop-color:#4F46E5"/>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
<path d="M50 5 L90 35 L75 95 L25 95 L10 35 Z" fill="url(#obsidian-gradient)" stroke="#6D28D9" stroke-width="2"/>
|
||||
<path d="M50 20 L70 38 L62 75 L38 75 L30 38 Z" fill="#A78BFA" opacity="0.4"/>
|
||||
<path d="M50 5 L50 95" stroke="#8B5CF6" stroke-width="1" opacity="0.5"/>
|
||||
<path d="M10 35 L90 35" stroke="#8B5CF6" stroke-width="1" opacity="0.3"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 657 B |
Loading…
Add table
Add a link
Reference in a new issue