apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-17 18:35:19 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #222 from CREDO23/feature/add-jira-connector

Browse source 
[Feat]: Add  JIRA connector
This commit is contained in:
Rohan Verma 2025-07-25 21:58:49 +05:30 • committed by GitHub
commit 665074e6e9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 7710 additions and 4614 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.pre-commit-config.yaml
View file

@ -67,7 +67,7 @@ repos:
hooks:
- id: mypy
files: ^surfsense_backend/
additional_dependencies: []
additional_dependencies: ['types-requests']
args: [--ignore-missing-imports, --disallow-untyped-defs]
- repo: https://github.com/PyCQA/bandit
@ -75,7 +75,7 @@ repos:
hooks:
- id: bandit
files: ^surfsense_backend/
args: ['-r', '.', '-f', 'json']
args: ['-r', '-f', 'json']
exclude: ^surfsense_backend/(tests/|alembic/)
# Frontend/Extension Hooks (TypeScript/JavaScript)

1
node_modules/.cache/prettier/.prettier-caches/a2ecb2962bf19c1099cfe708e42daa0097f94976.json generated vendored Normal file
View file

@ -0,0 +1 @@
{"2d0ec64d93969318101ee479b664221b32241665":{"files":{"surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx":["EHKKvlOK0vfy0GgHwlG/J2Bx5rw=",true]},"modified":1753426633288}}

184
surfsense_backend/alembic/versions/11_add_llm_config_table_and_relationships.py
View file

@ -6,10 +6,8 @@ Revises: 10
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import UUID, JSON
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "11"
@ -20,67 +18,145 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema - add LiteLLMProvider enum, LLMConfig table and user LLM preferences."""
# Check if enum type exists and create if it doesn't
op.execute("""
# Create enum only if not exists
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'litellmprovider') THEN
CREATE TYPE litellmprovider AS ENUM ('OPENAI', 'ANTHROPIC', 'GROQ', 'COHERE', 'HUGGINGFACE', 'AZURE_OPENAI', 'GOOGLE', 'AWS_BEDROCK', 'OLLAMA', 'MISTRAL', 'TOGETHER_AI', 'REPLICATE', 'PALM', 'VERTEX_AI', 'ANYSCALE', 'PERPLEXITY', 'DEEPINFRA', 'AI21', 'NLPCLOUD', 'ALEPH_ALPHA', 'PETALS', 'CUSTOM');
CREATE TYPE litellmprovider AS ENUM (
'OPENAI', 'ANTHROPIC', 'GROQ', 'COHERE', 'HUGGINGFACE',
'AZURE_OPENAI', 'GOOGLE', 'AWS_BEDROCK', 'OLLAMA', 'MISTRAL',
'TOGETHER_AI', 'REPLICATE', 'PALM', 'VERTEX_AI', 'ANYSCALE',
'PERPLEXITY', 'DEEPINFRA', 'AI21', 'NLPCLOUD', 'ALEPH_ALPHA',
'PETALS', 'CUSTOM'
);
END IF;
END$$;
""")
# Create llm_configs table using raw SQL to avoid enum creation conflicts
op.execute("""
CREATE TABLE llm_configs (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
name VARCHAR(100) NOT NULL,
provider litellmprovider NOT NULL,
custom_provider VARCHAR(100),
model_name VARCHAR(100) NOT NULL,
api_key TEXT NOT NULL,
api_base VARCHAR(500),
litellm_params JSONB,
user_id UUID NOT NULL REFERENCES "user"(id) ON DELETE CASCADE
)
""")
# Create indexes
op.create_index(op.f('ix_llm_configs_id'), 'llm_configs', ['id'], unique=False)
op.create_index(op.f('ix_llm_configs_created_at'), 'llm_configs', ['created_at'], unique=False)
op.create_index(op.f('ix_llm_configs_name'), 'llm_configs', ['name'], unique=False)
# Add LLM preference columns to user table
op.add_column('user', sa.Column('long_context_llm_id', sa.Integer(), nullable=True))
op.add_column('user', sa.Column('fast_llm_id', sa.Integer(), nullable=True))
op.add_column('user', sa.Column('strategic_llm_id', sa.Integer(), nullable=True))
# Create foreign key constraints for LLM preferences
op.create_foreign_key(op.f('fk_user_long_context_llm_id_llm_configs'), 'user', 'llm_configs', ['long_context_llm_id'], ['id'], ondelete='SET NULL')
op.create_foreign_key(op.f('fk_user_fast_llm_id_llm_configs'), 'user', 'llm_configs', ['fast_llm_id'], ['id'], ondelete='SET NULL')
op.create_foreign_key(op.f('fk_user_strategic_llm_id_llm_configs'), 'user', 'llm_configs', ['strategic_llm_id'], ['id'], ondelete='SET NULL')
"""
)
# Create llm_configs table only if it doesn't already exist
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT FROM information_schema.tables
WHERE table_name = 'llm_configs'
) THEN
CREATE TABLE llm_configs (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
name VARCHAR(100) NOT NULL,
provider litellmprovider NOT NULL,
custom_provider VARCHAR(100),
model_name VARCHAR(100) NOT NULL,
api_key TEXT NOT NULL,
api_base VARCHAR(500),
litellm_params JSONB,
user_id UUID NOT NULL REFERENCES "user"(id) ON DELETE CASCADE
);
END IF;
END$$;
"""
)
# Create indexes if they don't exist
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_indexes
WHERE tablename = 'llm_configs' AND indexname = 'ix_llm_configs_id'
) THEN
CREATE INDEX ix_llm_configs_id ON llm_configs(id);
END IF;
IF NOT EXISTS (
SELECT 1 FROM pg_indexes
WHERE tablename = 'llm_configs' AND indexname = 'ix_llm_configs_created_at'
) THEN
CREATE INDEX ix_llm_configs_created_at ON llm_configs(created_at);
END IF;
IF NOT EXISTS (
SELECT 1 FROM pg_indexes
WHERE tablename = 'llm_configs' AND indexname = 'ix_llm_configs_name'
) THEN
CREATE INDEX ix_llm_configs_name ON llm_configs(name);
END IF;
END$$;
"""
)
# Safely add columns to user table
bind = op.get_bind()
inspector = sa.inspect(bind)
existing_columns = [col["name"] for col in inspector.get_columns("user")]
with op.batch_alter_table("user") as batch_op:
if "long_context_llm_id" not in existing_columns:
batch_op.add_column(
sa.Column("long_context_llm_id", sa.Integer(), nullable=True)
)
batch_op.create_foreign_key(
op.f("fk_user_long_context_llm_id_llm_configs"),
"llm_configs",
["long_context_llm_id"],
["id"],
ondelete="SET NULL",
)
if "fast_llm_id" not in existing_columns:
batch_op.add_column(sa.Column("fast_llm_id", sa.Integer(), nullable=True))
batch_op.create_foreign_key(
op.f("fk_user_fast_llm_id_llm_configs"),
"llm_configs",
["fast_llm_id"],
["id"],
ondelete="SET NULL",
)
if "strategic_llm_id" not in existing_columns:
batch_op.add_column(
sa.Column("strategic_llm_id", sa.Integer(), nullable=True)
)
batch_op.create_foreign_key(
op.f("fk_user_strategic_llm_id_llm_configs"),
"llm_configs",
["strategic_llm_id"],
["id"],
ondelete="SET NULL",
)
def downgrade() -> None:
"""Downgrade schema - remove LLMConfig table and user LLM preferences."""
# Drop foreign key constraints
op.drop_constraint(op.f('fk_user_strategic_llm_id_llm_configs'), 'user', type_='foreignkey')
op.drop_constraint(op.f('fk_user_fast_llm_id_llm_configs'), 'user', type_='foreignkey')
op.drop_constraint(op.f('fk_user_long_context_llm_id_llm_configs'), 'user', type_='foreignkey')
op.drop_constraint(
op.f("fk_user_strategic_llm_id_llm_configs"), "user", type_="foreignkey"
)
op.drop_constraint(
op.f("fk_user_fast_llm_id_llm_configs"), "user", type_="foreignkey"
)
op.drop_constraint(
op.f("fk_user_long_context_llm_id_llm_configs"), "user", type_="foreignkey"
)
# Drop LLM preference columns from user table
op.drop_column('user', 'strategic_llm_id')
op.drop_column('user', 'fast_llm_id')
op.drop_column('user', 'long_context_llm_id')
op.drop_column("user", "strategic_llm_id")
op.drop_column("user", "fast_llm_id")
op.drop_column("user", "long_context_llm_id")
# Drop indexes and table
op.drop_index(op.f('ix_llm_configs_name'), table_name='llm_configs')
op.drop_index(op.f('ix_llm_configs_created_at'), table_name='llm_configs')
op.drop_index(op.f('ix_llm_configs_id'), table_name='llm_configs')
op.drop_table('llm_configs')
op.drop_index(op.f("ix_llm_configs_name"), table_name="llm_configs")
op.drop_index(op.f("ix_llm_configs_created_at"), table_name="llm_configs")
op.drop_index(op.f("ix_llm_configs_id"), table_name="llm_configs")
op.drop_table("llm_configs")
# Drop LiteLLMProvider enum
op.execute("DROP TYPE IF EXISTS litellmprovider")
op.execute("DROP TYPE IF EXISTS litellmprovider")

98
surfsense_backend/alembic/versions/12_add_logs_table.py
View file

@ -7,9 +7,7 @@ Revises: 11
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = "12"
@ -20,52 +18,78 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema - add LogLevel and LogStatus enums and logs table."""
# Create LogLevel enum
op.execute("""
CREATE TYPE loglevel AS ENUM ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')
""")
# Create LogStatus enum
op.execute("""
CREATE TYPE logstatus AS ENUM ('IN_PROGRESS', 'SUCCESS', 'FAILED')
""")
# Create logs table
op.execute("""
CREATE TABLE logs (
# Create LogLevel enum if it doesn't exist
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'loglevel') THEN
CREATE TYPE loglevel AS ENUM ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL');
END IF;
END$$;
"""
)
# Create LogStatus enum if it doesn't exist
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'logstatus') THEN
CREATE TYPE logstatus AS ENUM ('IN_PROGRESS', 'SUCCESS', 'FAILED');
END IF;
END$$;
"""
)
# Create logs table if it doesn't exist
op.execute(
"""
CREATE TABLE IF NOT EXISTS logs (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
level loglevel NOT NULL,
status logstatus NOT NULL,
message TEXT NOT NULL,
source VARCHAR(200),
log_metadata JSONB DEFAULT '{}',
search_space_id INTEGER NOT NULL REFERENCES searchspaces(id) ON DELETE CASCADE
)
""")
# Create indexes
op.create_index(op.f('ix_logs_id'), 'logs', ['id'], unique=False)
op.create_index(op.f('ix_logs_created_at'), 'logs', ['created_at'], unique=False)
op.create_index(op.f('ix_logs_level'), 'logs', ['level'], unique=False)
op.create_index(op.f('ix_logs_status'), 'logs', ['status'], unique=False)
op.create_index(op.f('ix_logs_source'), 'logs', ['source'], unique=False)
);
"""
)
# Get existing indexes
conn = op.get_bind()
inspector = inspect(conn)
existing_indexes = [idx["name"] for idx in inspector.get_indexes("logs")]
# Create indexes only if they don't already exist
if "ix_logs_id" not in existing_indexes:
op.create_index("ix_logs_id", "logs", ["id"])
if "ix_logs_created_at" not in existing_indexes:
op.create_index("ix_logs_created_at", "logs", ["created_at"])
if "ix_logs_level" not in existing_indexes:
op.create_index("ix_logs_level", "logs", ["level"])
if "ix_logs_status" not in existing_indexes:
op.create_index("ix_logs_status", "logs", ["status"])
if "ix_logs_source" not in existing_indexes:
op.create_index("ix_logs_source", "logs", ["source"])
def downgrade() -> None:
"""Downgrade schema - remove logs table and enums."""
# Drop indexes
op.drop_index(op.f('ix_logs_source'), table_name='logs')
op.drop_index(op.f('ix_logs_status'), table_name='logs')
op.drop_index(op.f('ix_logs_level'), table_name='logs')
op.drop_index(op.f('ix_logs_created_at'), table_name='logs')
op.drop_index(op.f('ix_logs_id'), table_name='logs')
op.drop_index("ix_logs_source", table_name="logs")
op.drop_index("ix_logs_status", table_name="logs")
op.drop_index("ix_logs_level", table_name="logs")
op.drop_index("ix_logs_created_at", table_name="logs")
op.drop_index("ix_logs_id", table_name="logs")
# Drop logs table
op.drop_table('logs')
op.drop_table("logs")
# Drop enums
op.execute("DROP TYPE IF EXISTS logstatus")
op.execute("DROP TYPE IF EXISTS loglevel")
op.execute("DROP TYPE IF EXISTS loglevel")

61
surfsense_backend/alembic/versions/13_add_jira_connector_enums.py Normal file
View file

@ -0,0 +1,61 @@
"""Add JIRA_CONNECTOR to enums
Revision ID: 13
Revises: 12
"""
from typing import Sequence, Union
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "13"
down_revision: Union[str, None] = "12"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Safely add 'JIRA_CONNECTOR' to enum types if missing."""
# Add to searchsourceconnectortype enum
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_type t
JOIN pg_enum e ON t.oid = e.enumtypid
WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'JIRA_CONNECTOR'
) THEN
ALTER TYPE searchsourceconnectortype ADD VALUE 'JIRA_CONNECTOR';
END IF;
END
$$;
"""
)
# Add to documenttype enum
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_type t
JOIN pg_enum e ON t.oid = e.enumtypid
WHERE t.typname = 'documenttype' AND e.enumlabel = 'JIRA_CONNECTOR'
) THEN
ALTER TYPE documenttype ADD VALUE 'JIRA_CONNECTOR';
END IF;
END
$$;
"""
)
def downgrade() -> None:
"""
Downgrade logic not implemented since PostgreSQL
does not support removing enum values.
"""
pass

54
surfsense_backend/alembic/versions/1_add_github_connector_enum.py
View file

@ -1,19 +1,20 @@
"""Add GITHUB_CONNECTOR to SearchSourceConnectorType enum
Revision ID: 1
Revises:
Revises:
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# Import pgvector if needed for other types, though not for this ENUM change
# import pgvector
# revision identifiers, used by Alembic.
revision: str = '1'
revision: str = "1"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
@ -21,11 +22,27 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# Manually add the command to add the enum value
# Note: It's generally better to let autogenerate handle this, but we're bypassing it
op.execute("ALTER TYPE searchsourceconnectortype ADD VALUE 'GITHUB_CONNECTOR'")
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_enum
WHERE enumlabel = 'GITHUB_CONNECTOR'
AND enumtypid = (
SELECT oid FROM pg_type WHERE typname = 'searchsourceconnectortype'
)
) THEN
ALTER TYPE searchsourceconnectortype ADD VALUE 'GITHUB_CONNECTOR';
END IF;
END$$;
"""
)
# Pass for the rest, as autogenerate didn't run to add other schema details
pass
# ### end Alembic commands ###
@ -33,20 +50,25 @@ def upgrade() -> None:
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# Downgrading removal of an enum value is complex and potentially dangerous
# if the value is in use. Often omitted or requires manual SQL based on context.
# For now, we'll just pass. If you needed to reverse this, you'd likely
# For now, we'll just pass. If you needed to reverse this, you'd likely
# have to manually check if 'GITHUB_CONNECTOR' is used in the table
# and then potentially recreate the type without it.
op.execute("ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old")
op.execute("CREATE TYPE searchsourceconnectortype AS ENUM('SERPER_API', 'TAVILY_API', 'SLACK_CONNECTOR', 'NOTION_CONNECTOR')")
op.execute((
"ALTER TABLE search_source_connectors ALTER COLUMN connector_type TYPE searchsourceconnectortype USING "
"connector_type::text::searchsourceconnectortype"
))
op.execute(
"ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old"
)
op.execute(
"CREATE TYPE searchsourceconnectortype AS ENUM('SERPER_API', 'TAVILY_API', 'SLACK_CONNECTOR', 'NOTION_CONNECTOR')"
)
op.execute(
(
"ALTER TABLE search_source_connectors ALTER COLUMN connector_type TYPE searchsourceconnectortype USING "
"connector_type::text::searchsourceconnectortype"
)
)
op.execute("DROP TYPE searchsourceconnectortype_old")
pass
# ### end Alembic commands ###
# ### end Alembic commands ###

56
surfsense_backend/alembic/versions/2_add_linear_connector_enum.py
View file

@ -4,41 +4,57 @@ Revision ID: 2
Revises: e55302644c51
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '2'
down_revision: Union[str, None] = 'e55302644c51'
revision: str = "2"
down_revision: Union[str, None] = "e55302644c51"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# Manually add the command to add the enum value
op.execute("ALTER TYPE searchsourceconnectortype ADD VALUE 'LINEAR_CONNECTOR'")
# Pass for the rest, as autogenerate didn't run to add other schema details
pass
# ### end Alembic commands ###
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = 'LINEAR_CONNECTOR'
AND enumtypid = (
SELECT oid FROM pg_type WHERE typname = 'searchsourceconnectortype'
)
) THEN
ALTER TYPE searchsourceconnectortype ADD VALUE 'LINEAR_CONNECTOR';
END IF;
END$$;
"""
)
#
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# Downgrading removal of an enum value requires recreating the type
op.execute("ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old")
op.execute("CREATE TYPE searchsourceconnectortype AS ENUM('SERPER_API', 'TAVILY_API', 'SLACK_CONNECTOR', 'NOTION_CONNECTOR', 'GITHUB_CONNECTOR')")
op.execute((
"ALTER TABLE search_source_connectors ALTER COLUMN connector_type TYPE searchsourceconnectortype USING "
"connector_type::text::searchsourceconnectortype"
))
op.execute(
"ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old"
)
op.execute(
"CREATE TYPE searchsourceconnectortype AS ENUM('SERPER_API', 'TAVILY_API', 'SLACK_CONNECTOR', 'NOTION_CONNECTOR', 'GITHUB_CONNECTOR')"
)
op.execute(
(
"ALTER TABLE search_source_connectors ALTER COLUMN connector_type TYPE searchsourceconnectortype USING "
"connector_type::text::searchsourceconnectortype"
)
)
op.execute("DROP TYPE searchsourceconnectortype_old")
pass
# ### end Alembic commands ###
# ### end Alembic commands ###

57
surfsense_backend/alembic/versions/3_add_linear_connector_to_documenttype_.py
View file

@ -4,26 +4,41 @@ Revision ID: 3
Revises: 2
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '3'
down_revision: Union[str, None] = '2'
revision: str = "3"
down_revision: Union[str, None] = "2"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
# Define the ENUM type name and the new value
ENUM_NAME = 'documenttype' # Make sure this matches the name in your DB (usually lowercase class name)
NEW_VALUE = 'LINEAR_CONNECTOR'
ENUM_NAME = "documenttype" # Make sure this matches the name in your DB (usually lowercase class name)
NEW_VALUE = "LINEAR_CONNECTOR"
def upgrade() -> None:
"""Upgrade schema."""
op.execute(f"ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}'")
op.execute(
f"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = '{NEW_VALUE}'
AND enumtypid = (
SELECT oid FROM pg_type WHERE typname = '{ENUM_NAME}'
)
) THEN
ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}';
END IF;
END$$;
"""
)
# Warning: This will delete all rows with the new value
def downgrade() -> None:
@ -34,19 +49,19 @@ def downgrade() -> None:
# Enum values *before* LINEAR_CONNECTOR was added
old_values = (
'EXTENSION',
'CRAWLED_URL',
'FILE',
'SLACK_CONNECTOR',
'NOTION_CONNECTOR',
'YOUTUBE_VIDEO',
'GITHUB_CONNECTOR'
"EXTENSION",
"CRAWLED_URL",
"FILE",
"SLACK_CONNECTOR",
"NOTION_CONNECTOR",
"YOUTUBE_VIDEO",
"GITHUB_CONNECTOR",
)
old_values_sql = ", ".join([f"'{v}'" for v in old_values])
# Table and column names (adjust if different)
table_name = 'documents'
column_name = 'document_type'
table_name = "documents"
column_name = "document_type"
# 1. Rename the current enum type
op.execute(f"ALTER TYPE {ENUM_NAME} RENAME TO {old_enum_name}")
@ -54,10 +69,8 @@ def downgrade() -> None:
# 2. Create the new enum type with the old values
op.execute(f"CREATE TYPE {ENUM_NAME} AS ENUM({old_values_sql})")
# 3. Update the table:
op.execute(
f"DELETE FROM {table_name} WHERE {column_name}::text = '{NEW_VALUE}'"
)
# 3. Update the table:
op.execute(f"DELETE FROM {table_name} WHERE {column_name}::text = '{NEW_VALUE}'")
# 4. Alter the column to use the new enum type (casting old values)
op.execute(
@ -67,4 +80,4 @@ def downgrade() -> None:
# 5. Drop the old enum type
op.execute(f"DROP TYPE {old_enum_name}")
# ### end Alembic commands ###
# ### end Alembic commands ###

56
surfsense_backend/alembic/versions/4_add_linkup_api_enum.py
View file

@ -4,41 +4,57 @@ Revision ID: 4
Revises: 3
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '4'
down_revision: Union[str, None] = '3'
revision: str = "4"
down_revision: Union[str, None] = "3"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# Manually add the command to add the enum value
op.execute("ALTER TYPE searchsourceconnectortype ADD VALUE 'LINKUP_API'")
# Pass for the rest, as autogenerate didn't run to add other schema details
pass
# ### end Alembic commands ###
ENUM_NAME = "searchsourceconnectortype"
NEW_VALUE = "LINKUP_API"
op.execute(
f"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = '{NEW_VALUE}'
AND enumtypid = (
SELECT oid FROM pg_type WHERE typname = '{ENUM_NAME}'
)
) THEN
ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}';
END IF;
END$$;
"""
)
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# Downgrading removal of an enum value requires recreating the type
op.execute("ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old")
op.execute("CREATE TYPE searchsourceconnectortype AS ENUM('SERPER_API', 'TAVILY_API', 'SLACK_CONNECTOR', 'NOTION_CONNECTOR', 'GITHUB_CONNECTOR', 'LINEAR_CONNECTOR')")
op.execute((
"ALTER TABLE search_source_connectors ALTER COLUMN connector_type TYPE searchsourceconnectortype USING "
"connector_type::text::searchsourceconnectortype"
))
op.execute(
"ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old"
)
op.execute(
"CREATE TYPE searchsourceconnectortype AS ENUM('SERPER_API', 'TAVILY_API', 'SLACK_CONNECTOR', 'NOTION_CONNECTOR', 'GITHUB_CONNECTOR', 'LINEAR_CONNECTOR')"
)
op.execute(
(
"ALTER TABLE search_source_connectors ALTER COLUMN connector_type TYPE searchsourceconnectortype USING "
"connector_type::text::searchsourceconnectortype"
)
)
op.execute("DROP TYPE searchsourceconnectortype_old")
pass
# ### end Alembic commands ###
# ### end Alembic commands ###

56
surfsense_backend/alembic/versions/6_change_podcast_content_to_transcript.py
View file

@ -4,40 +4,58 @@ Revision ID: 6
Revises: 5
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from sqlalchemy.dialects.postgresql import JSON
# revision identifiers, used by Alembic.
revision: str = '6'
down_revision: Union[str, None] = '5'
revision: str = "6"
down_revision: Union[str, None] = "5"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Drop the old column and create a new one with the new name and type
# We need to do this because PostgreSQL doesn't support direct column renames with type changes
op.add_column('podcasts', sa.Column('podcast_transcript', JSON, nullable=False, server_default='{}'))
# Copy data from old column to new column
# Convert text to JSON by storing it as a JSON string value
op.execute("UPDATE podcasts SET podcast_transcript = jsonb_build_object('text', podcast_content) WHERE podcast_content != ''")
# Drop the old column
op.drop_column('podcasts', 'podcast_content')
bind = op.get_bind()
inspector = inspect(bind)
columns = [col["name"] for col in inspector.get_columns("podcasts")]
if "podcast_transcript" not in columns:
op.add_column(
"podcasts",
sa.Column("podcast_transcript", JSON, nullable=False, server_default="{}"),
)
# Copy data from old column to new column
op.execute(
"""
UPDATE podcasts
SET podcast_transcript = jsonb_build_object('text', podcast_content)
WHERE podcast_content != ''
"""
)
# Drop the old column only if it exists
if "podcast_content" in columns:
op.drop_column("podcasts", "podcast_content")
def downgrade() -> None:
# Add back the original column
op.add_column('podcasts', sa.Column('podcast_content', sa.Text(), nullable=False, server_default=''))
op.add_column(
"podcasts",
sa.Column("podcast_content", sa.Text(), nullable=False, server_default=""),
)
# Copy data from JSON column back to text column
# Extract the 'text' field if it exists, otherwise use empty string
op.execute("UPDATE podcasts SET podcast_content = COALESCE((podcast_transcript->>'text'), '')")
op.execute(
"UPDATE podcasts SET podcast_content = COALESCE((podcast_transcript->>'text'), '')"
)
# Drop the new column
op.drop_column('podcasts', 'podcast_transcript')
op.drop_column("podcasts", "podcast_transcript")

24
surfsense_backend/alembic/versions/7_remove_is_generated_column.py
View file

@ -4,24 +4,34 @@ Revision ID: 7
Revises: 6
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = '7'
down_revision: Union[str, None] = '6'
revision: str = "7"
down_revision: Union[str, None] = "6"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Drop the is_generated column
op.drop_column('podcasts', 'is_generated')
# Get the current database connection
bind = op.get_bind()
inspector = inspect(bind)
# Check if the column exists before attempting to drop it
columns = [col["name"] for col in inspector.get_columns("podcasts")]
if "is_generated" in columns:
op.drop_column("podcasts", "is_generated")
def downgrade() -> None:
# Add back the is_generated column with its original constraints
op.add_column('podcasts', sa.Column('is_generated', sa.Boolean(), nullable=False, server_default='false'))
op.add_column(
"podcasts",
sa.Column("is_generated", sa.Boolean(), nullable=False, server_default="false"),
)

88
surfsense_backend/alembic/versions/8_add_content_hash_to_documents.py
View file

@ -3,54 +3,68 @@
Revision ID: 8
Revises: 7
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
# revision identifiers, used by Alembic.
revision: str = '8'
down_revision: Union[str, None] = '7'
revision: str = "8"
down_revision: Union[str, None] = "7"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Add content_hash column as nullable first to handle existing data
op.add_column('documents', sa.Column('content_hash', sa.String(), nullable=True))
# Update existing documents to generate content hashes
# Using SHA-256 hash of the content column with proper UTF-8 encoding
op.execute("""
UPDATE documents
SET content_hash = encode(sha256(convert_to(content, 'UTF8')), 'hex')
WHERE content_hash IS NULL
""")
# Handle duplicate content hashes by keeping only the oldest document for each hash
# Delete newer documents with duplicate content hashes
op.execute("""
DELETE FROM documents
WHERE id NOT IN (
SELECT MIN(id)
FROM documents
GROUP BY content_hash
bind = op.get_bind()
inspector = inspect(bind)
columns = [col["name"] for col in inspector.get_columns("documents")]
# Only add the column if it doesn't already exist
if "content_hash" not in columns:
op.add_column(
"documents", sa.Column("content_hash", sa.String(), nullable=True)
)
""")
# Now alter the column to match the model: nullable=False, index=True, unique=True
op.alter_column('documents', 'content_hash',
existing_type=sa.String(),
nullable=False)
op.create_index(op.f('ix_documents_content_hash'), 'documents', ['content_hash'], unique=False)
op.create_unique_constraint(op.f('uq_documents_content_hash'), 'documents', ['content_hash'])
# Populate the content_hash column
op.execute(
"""
UPDATE documents
SET content_hash = encode(sha256(convert_to(content, 'UTF8')), 'hex')
WHERE content_hash IS NULL
"""
)
op.execute(
"""
DELETE FROM documents
WHERE id NOT IN (
SELECT MIN(id)
FROM documents
GROUP BY content_hash
)
"""
)
op.alter_column(
"documents", "content_hash", existing_type=sa.String(), nullable=False
)
op.create_index(
op.f("ix_documents_content_hash"),
"documents",
["content_hash"],
unique=False,
)
op.create_unique_constraint(
op.f("uq_documents_content_hash"), "documents", ["content_hash"]
)
else:
print("Column 'content_hash' already exists. Skipping column creation.")
def downgrade() -> None:
# Remove constraints and index first
op.drop_constraint(op.f('uq_documents_content_hash'), 'documents', type_='unique')
op.drop_index(op.f('ix_documents_content_hash'), table_name='documents')
# Remove content_hash column from documents table
op.drop_column('documents', 'content_hash')
op.drop_constraint(op.f("uq_documents_content_hash"), "documents", type_="unique")
op.drop_index(op.f("ix_documents_content_hash"), table_name="documents")
op.drop_column("documents", "content_hash")

40
surfsense_backend/alembic/versions/9_add_discord_connector_enum_and_documenttype.py
View file

@ -7,8 +7,6 @@ Revises: 8
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "9"
@ -24,11 +22,38 @@ DOCUMENT_NEW_VALUE = "DISCORD_CONNECTOR"
def upgrade() -> None:
"""Upgrade schema - add DISCORD_CONNECTOR to connector and document enum."""
# Add DISCORD_CONNECTOR to searchsourceconnectortype
op.execute(f"ALTER TYPE {CONNECTOR_ENUM} ADD VALUE '{CONNECTOR_NEW_VALUE}'")
# Add DISCORD_CONNECTOR to documenttype
op.execute(f"ALTER TYPE {DOCUMENT_ENUM} ADD VALUE '{DOCUMENT_NEW_VALUE}'")
"""Upgrade schema - add DISCORD_CONNECTOR to connector and document enum safely."""
# Add DISCORD_CONNECTOR to searchsourceconnectortype only if not exists
op.execute(
f"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = '{CONNECTOR_NEW_VALUE}'
AND enumtypid = (SELECT oid FROM pg_type WHERE typname = '{CONNECTOR_ENUM}')
) THEN
ALTER TYPE {CONNECTOR_ENUM} ADD VALUE '{CONNECTOR_NEW_VALUE}';
END IF;
END$$;
"""
)
# Add DISCORD_CONNECTOR to documenttype only if not exists
op.execute(
f"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = '{DOCUMENT_NEW_VALUE}'
AND enumtypid = (SELECT oid FROM pg_type WHERE typname = '{DOCUMENT_ENUM}')
) THEN
ALTER TYPE {DOCUMENT_ENUM} ADD VALUE '{DOCUMENT_NEW_VALUE}';
END IF;
END$$;
"""
)
def downgrade() -> None:
@ -85,7 +110,6 @@ def downgrade() -> None:
# 4. Drop the old connector enum type
op.execute(f"DROP TYPE {old_connector_enum_name}")
# Document Enum Downgrade Steps
# 1. Rename the current document enum type
op.execute(f"ALTER TYPE {DOCUMENT_ENUM} RENAME TO {old_document_enum_name}")

80
surfsense_backend/alembic/versions/e55302644c51_add_github_connector_to_documenttype_.py
View file

@ -1,69 +1,67 @@
"""Add GITHUB_CONNECTOR to DocumentType enum
Revision ID: e55302644c51
Revises: 1
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'e55302644c51'
down_revision: Union[str, None] = '1'
revision: str = "e55302644c51"
down_revision: Union[str, None] = "1"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
# Define the ENUM type name and the new value
ENUM_NAME = 'documenttype' # Make sure this matches the name in your DB (usually lowercase class name)
NEW_VALUE = 'GITHUB_CONNECTOR'
ENUM_NAME = "documenttype"
NEW_VALUE = "GITHUB_CONNECTOR"
def upgrade() -> None:
"""Upgrade schema."""
op.execute(f"ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}'")
op.execute(
f"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = '{NEW_VALUE}'
AND enumtypid = (
SELECT oid FROM pg_type WHERE typname = '{ENUM_NAME}'
)
) THEN
ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}';
END IF;
END$$;
"""
)
# Warning: This will delete all rows with the new value
def downgrade() -> None:
"""Downgrade schema - remove GITHUB_CONNECTOR from enum."""
# The old type name
old_enum_name = f"{ENUM_NAME}_old"
# Enum values *before* GITHUB_CONNECTOR was added
old_values = (
'EXTENSION',
'CRAWLED_URL',
'FILE',
'SLACK_CONNECTOR',
'NOTION_CONNECTOR',
'YOUTUBE_VIDEO'
"EXTENSION",
"CRAWLED_URL",
"FILE",
"SLACK_CONNECTOR",
"NOTION_CONNECTOR",
"YOUTUBE_VIDEO",
)
old_values_sql = ", ".join([f"'{v}'" for v in old_values])
# Table and column names (adjust if different)
table_name = 'documents'
column_name = 'document_type'
table_name = "documents"
column_name = "document_type"
# 1. Rename the current enum type
op.execute(f"ALTER TYPE {ENUM_NAME} RENAME TO {old_enum_name}")
# 1. Create the new enum type with the old values
op.execute(f"CREATE TYPE {old_enum_name} AS ENUM({old_values_sql})")
# 2. Create the new enum type with the old values
op.execute(f"CREATE TYPE {ENUM_NAME} AS ENUM({old_values_sql})")
# 2. Delete rows using the new value
op.execute(f"DELETE FROM {table_name} WHERE {column_name}::text = '{NEW_VALUE}'")
# 3. Update the table:
op.execute(
f"DELETE FROM {table_name} WHERE {column_name}::text = '{NEW_VALUE}'"
)
# 4. Alter the column to use the new enum type (casting old values)
# 3. Alter the column to use the old enum type
op.execute(
f"ALTER TABLE {table_name} ALTER COLUMN {column_name} "
f"TYPE {ENUM_NAME} USING {column_name}::text::{ENUM_NAME}"
f"TYPE {old_enum_name} USING {column_name}::text::{old_enum_name}"
)
# 5. Drop the old enum type
op.execute(f"DROP TYPE {old_enum_name}")
# ### end Alembic commands ###
# 4. Drop the current enum type and rename the old one
op.execute(f"DROP TYPE {ENUM_NAME}")
op.execute(f"ALTER TYPE {old_enum_name} RENAME TO {ENUM_NAME}")

837
surfsense_backend/app/agents/researcher/nodes.py
View file

File diff suppressed because it is too large Load diff

5
surfsense_backend/app/agents/researcher/qna_agent/prompts.py
View file

@ -15,7 +15,8 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel
- YOUTUBE_VIDEO: "YouTube video transcripts and metadata" (personally saved videos)
- GITHUB_CONNECTOR: "GitHub repository content and issues" (personal repositories and interactions)
- LINEAR_CONNECTOR: "Linear project issues and discussions" (personal project management)
- DISCORD_CONNECTOR: "Discord server messages and channels" (personal community interactions)
- JIRA_CONNECTOR: "Jira project issues, tickets, and comments" (personal project tracking)
- DISCORD_CONNECTOR: "Discord server conversations and shared content" (personal community communications)
- TAVILY_API: "Tavily search API results" (personalized search results)
- LINKUP_API: "Linkup search API results" (personalized search results)
</knowledge_sources>
@ -71,7 +72,7 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel
Python's asyncio library provides tools for writing concurrent code using the async/await syntax. It's particularly useful for I/O-bound and high-level structured network code.
</content>
</document>
<document>
<metadata>
<source_id>12</source_id>

4
surfsense_backend/app/agents/researcher/utils.py
View file

@ -33,6 +33,8 @@ def get_connector_emoji(connector_name: str) -> str:
"NOTION_CONNECTOR": "📘",
"GITHUB_CONNECTOR": "🐙",
"LINEAR_CONNECTOR": "📊",
"JIRA_CONNECTOR": "🎫",
"DISCORD_CONNECTOR": "🗨️",
"TAVILY_API": "🔍",
"LINKUP_API": "🔗"
}
@ -50,6 +52,8 @@ def get_connector_friendly_name(connector_name: str) -> str:
"NOTION_CONNECTOR": "Notion",
"GITHUB_CONNECTOR": "GitHub",
"LINEAR_CONNECTOR": "Linear",
"JIRA_CONNECTOR": "Jira",
"DISCORD_CONNECTOR": "Discord",
"TAVILY_API": "Tavily Search",
"LINKUP_API": "Linkup Search"
}

487
surfsense_backend/app/connectors/jira_connector.py Normal file
View file

@ -0,0 +1,487 @@
"""
Jira Connector Module
A module for retrieving data from Jira.
Allows fetching issue lists and their comments, projects and more.
"""
import base64
from datetime import datetime
from typing import Any, Dict, List, Optional
import requests
class JiraConnector:
"""Class for retrieving data from Jira."""
def __init__(
self,
base_url: Optional[str] = None,
email: Optional[str] = None,
api_token: Optional[str] = None,
):
"""
Initialize the JiraConnector class.
Args:
base_url: Jira instance base URL (e.g., 'https://yourcompany.atlassian.net') (optional)
email: Jira account email address (optional)
api_token: Jira API token (optional)
"""
self.base_url = base_url.rstrip("/") if base_url else None
self.email = email
self.api_token = api_token
self.api_version = "3" # Jira Cloud API version
def set_credentials(self, base_url: str, email: str, api_token: str) -> None:
"""
Set the Jira credentials.
Args:
base_url: Jira instance base URL
email: Jira account email address
api_token: Jira API token
"""
self.base_url = base_url.rstrip("/")
self.email = email
self.api_token = api_token
def set_email(self, email: str) -> None:
"""
Set the Jira account email.
Args:
email: Jira account email address
"""
self.email = email
def set_api_token(self, api_token: str) -> None:
"""
Set the Jira API token.
Args:
api_token: Jira API token
"""
self.api_token = api_token
def get_headers(self) -> Dict[str, str]:
"""
Get headers for Jira API requests using Basic Authentication.
Returns:
Dictionary of headers
Raises:
ValueError: If email, api_token, or base_url have not been set
"""
if not all([self.base_url, self.email, self.api_token]):
raise ValueError(
"Jira credentials not initialized. Call set_credentials() first."
)
# Create Basic Auth header using email:api_token
auth_str = f"{self.email}:{self.api_token}"
auth_bytes = auth_str.encode("utf-8")
auth_header = "Basic " + base64.b64encode(auth_bytes).decode("ascii")
return {
"Content-Type": "application/json",
"Authorization": auth_header,
"Accept": "application/json",
}
def make_api_request(
self, endpoint: str, params: Optional[Dict[str, Any]] = None
) -> Dict[str, Any]:
"""
Make a request to the Jira API.
Args:
endpoint: API endpoint (without base URL)
params: Query parameters for the request (optional)
Returns:
Response data from the API
Raises:
ValueError: If email, api_token, or base_url have not been set
Exception: If the API request fails
"""
if not all([self.base_url, self.email, self.api_token]):
raise ValueError(
"Jira credentials not initialized. Call set_credentials() first."
)
url = f"{self.base_url}/rest/api/{self.api_version}/{endpoint}"
headers = self.get_headers()
response = requests.get(url, headers=headers, params=params, timeout=500)
if response.status_code == 200:
return response.json()
else:
raise Exception(
f"API request failed with status code {response.status_code}: {response.text}"
)
def get_all_projects(self) -> dict[str, Any]:
"""
Fetch all projects from Jira.
Returns:
List of project objects
Raises:
ValueError: If credentials have not been set
Exception: If the API request fails
"""
return self.make_api_request("project/search")
def get_all_issues(self, project_key: Optional[str] = None) -> List[Dict[str, Any]]:
"""
Fetch all issues from Jira.
Args:
project_key: Optional project key to filter issues (e.g., 'PROJ')
Returns:
List of issue objects
Raises:
ValueError: If credentials have not been set
Exception: If the API request fails
"""
jql = "ORDER BY created DESC"
if project_key:
jql = f'project = "{project_key}" ' + jql
fields = [
"summary",
"description",
"status",
"assignee",
"reporter",
"created",
"updated",
"priority",
"issuetype",
"project",
]
params = {
"jql": jql,
"fields": ",".join(fields),
"maxResults": 100,
"startAt": 0,
}
all_issues = []
start_at = 0
while True:
params["startAt"] = start_at
result = self.make_api_request("search", params)
if not isinstance(result, dict) or "issues" not in result:
raise Exception("Invalid response from Jira API")
issues = result["issues"]
all_issues.extend(issues)
print(f"Fetched {len(issues)} issues (startAt={start_at})")
total = result.get("total", 0)
if start_at + len(issues) >= total:
break
start_at += len(issues)
return all_issues
def get_issues_by_date_range(
self,
start_date: str,
end_date: str,
include_comments: bool = True,
project_key: Optional[str] = None,
) -> tuple[List[Dict[str, Any]], Optional[str]]:
"""
Fetch issues within a date range.
Args:
start_date: Start date in YYYY-MM-DD format
end_date: End date in YYYY-MM-DD format (inclusive)
include_comments: Whether to include comments in the response
project_key: Optional project key to filter issues
Returns:
Tuple containing (issues list, error message or None)
"""
try:
# Build JQL query for date range
# Query issues that were either created OR updated within the date range
date_filter = (
f"(createdDate >= '{start_date}' AND createdDate <= '{end_date}')"
)
# TODO : This JQL needs some improvement to work as expected
jql = f"{date_filter}"
if project_key:
jql = (
f'project = "{project_key}" AND {date_filter} ORDER BY created DESC'
)
# Define fields to retrieve
fields = [
"summary",
"description",
"status",
"assignee",
"reporter",
"created",
"updated",
"priority",
"issuetype",
"project",
]
if include_comments:
fields.append("comment")
params = {
# "jql": "", TODO : Add a JQL query to filter from a date range
"fields": ",".join(fields),
"maxResults": 100,
"startAt": 0,
}
all_issues = []
start_at = 0
while True:
params["startAt"] = start_at
result = self.make_api_request("search", params)
if not isinstance(result, dict) or "issues" not in result:
return [], "Invalid response from Jira API"
issues = result["issues"]
all_issues.extend(issues)
# Check if there are more issues to fetch
total = result.get("total", 0)
if start_at + len(issues) >= total:
break
start_at += len(issues)
if not all_issues:
return [], "No issues found in the specified date range."
return all_issues, None
except Exception as e:
return [], f"Error fetching issues: {str(e)}"
def format_issue(self, issue: Dict[str, Any]) -> Dict[str, Any]:
"""
Format an issue for easier consumption.
Args:
issue: The issue object from Jira API
Returns:
Formatted issue dictionary
"""
fields = issue.get("fields", {})
# Extract basic issue details
formatted = {
"id": issue.get("id", ""),
"key": issue.get("key", ""),
"title": fields.get("summary", ""),
"description": fields.get("description", ""),
"status": (
fields.get("status", {}).get("name", "Unknown")
if fields.get("status")
else "Unknown"
),
"status_category": (
fields.get("status", {})
.get("statusCategory", {})
.get("name", "Unknown")
if fields.get("status")
else "Unknown"
),
"priority": (
fields.get("priority", {}).get("name", "Unknown")
if fields.get("priority")
else "Unknown"
),
"issue_type": (
fields.get("issuetype", {}).get("name", "Unknown")
if fields.get("issuetype")
else "Unknown"
),
"project": (
fields.get("project", {}).get("key", "Unknown")
if fields.get("project")
else "Unknown"
),
"created_at": fields.get("created", ""),
"updated_at": fields.get("updated", ""),
"reporter": (
{
"account_id": (
fields.get("reporter", {}).get("accountId", "")
if fields.get("reporter")
else ""
),
"display_name": (
fields.get("reporter", {}).get("displayName", "Unknown")
if fields.get("reporter")
else "Unknown"
),
"email": (
fields.get("reporter", {}).get("emailAddress", "")
if fields.get("reporter")
else ""
),
}
if fields.get("reporter")
else {"account_id": "", "display_name": "Unknown", "email": ""}
),
"assignee": (
{
"account_id": fields.get("assignee", {}).get("accountId", ""),
"display_name": fields.get("assignee", {}).get(
"displayName", "Unknown"
),
"email": fields.get("assignee", {}).get("emailAddress", ""),
}
if fields.get("assignee")
else None
),
"comments": [],
}
# Extract comments if available
if "comment" in fields and "comments" in fields["comment"]:
for comment in fields["comment"]["comments"]:
formatted_comment = {
"id": comment.get("id", ""),
"body": comment.get("body", ""),
"created_at": comment.get("created", ""),
"updated_at": comment.get("updated", ""),
"author": (
{
"account_id": (
comment.get("author", {}).get("accountId", "")
if comment.get("author")
else ""
),
"display_name": (
comment.get("author", {}).get("displayName", "Unknown")
if comment.get("author")
else "Unknown"
),
"email": (
comment.get("author", {}).get("emailAddress", "")
if comment.get("author")
else ""
),
}
if comment.get("author")
else {"account_id": "", "display_name": "Unknown", "email": ""}
),
}
formatted["comments"].append(formatted_comment)
return formatted
def format_issue_to_markdown(self, issue: Dict[str, Any]) -> str:
"""
Convert an issue to markdown format.
Args:
issue: The issue object (either raw or formatted)
Returns:
Markdown string representation of the issue
"""
# Format the issue if it's not already formatted
if "key" not in issue:
issue = self.format_issue(issue)
# Build the markdown content
markdown = (
f"# {issue.get('key', 'No Key')}: {issue.get('title', 'No Title')}\n\n"
)
if issue.get("status"):
markdown += f"**Status:** {issue['status']}\n"
if issue.get("priority"):
markdown += f"**Priority:** {issue['priority']}\n"
if issue.get("issue_type"):
markdown += f"**Type:** {issue['issue_type']}\n"
if issue.get("project"):
markdown += f"**Project:** {issue['project']}\n\n"
if issue.get("assignee") and issue["assignee"].get("display_name"):
markdown += f"**Assignee:** {issue['assignee']['display_name']}\n"
if issue.get("reporter") and issue["reporter"].get("display_name"):
markdown += f"**Reporter:** {issue['reporter']['display_name']}\n"
if issue.get("created_at"):
created_date = self.format_date(issue["created_at"])
markdown += f"**Created:** {created_date}\n"
if issue.get("updated_at"):
updated_date = self.format_date(issue["updated_at"])
markdown += f"**Updated:** {updated_date}\n\n"
if issue.get("description"):
markdown += f"## Description\n\n{issue['description']}\n\n"
if issue.get("comments"):
markdown += f"## Comments ({len(issue['comments'])})\n\n"
for comment in issue["comments"]:
author_name = "Unknown"
if comment.get("author") and comment["author"].get("display_name"):
author_name = comment["author"]["display_name"]
comment_date = "Unknown date"
if comment.get("created_at"):
comment_date = self.format_date(comment["created_at"])
markdown += f"### {author_name} ({comment_date})\n\n{comment.get('body', '')}\n\n---\n\n"
return markdown
@staticmethod
def format_date(iso_date: str) -> str:
"""
Format an ISO date string to a more readable format.
Args:
iso_date: ISO format date string
Returns:
Formatted date string
"""
if not iso_date or not isinstance(iso_date, str):
return "Unknown date"
try:
# Jira dates are typically in format: 2023-01-01T12:00:00.000+0000
dt = datetime.fromisoformat(iso_date.replace("Z", "+00:00"))
return dt.strftime("%Y-%m-%d %H:%M:%S")
except ValueError:
return iso_date

283
surfsense_backend/app/db.py
View file

@ -2,30 +2,30 @@ from collections.abc import AsyncGenerator
from datetime import datetime, timezone
from enum import Enum
from app.config import config
from app.retriver.chunks_hybrid_search import ChucksHybridSearchRetriever
from app.retriver.documents_hybrid_search import DocumentHybridSearchRetriever
from fastapi import Depends
from pgvector.sqlalchemy import Vector
from sqlalchemy import (
ARRAY,
JSON,
TIMESTAMP,
Boolean,
Column,
Enum as SQLAlchemyEnum,
)
from sqlalchemy import Enum as SQLAlchemyEnum
from sqlalchemy import (
ForeignKey,
Integer,
JSON,
String,
Text,
text,
TIMESTAMP
)
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase, Mapped, declared_attr, relationship
from app.config import config
from app.retriver.chunks_hybrid_search import ChucksHybridSearchRetriever
from app.retriver.documents_hybrid_search import DocumentHybridSearchRetriever
if config.AUTH_TYPE == "GOOGLE":
from fastapi_users.db import (
SQLAlchemyBaseOAuthAccountTableUUID,
@ -51,9 +51,11 @@ class DocumentType(str, Enum):
GITHUB_CONNECTOR = "GITHUB_CONNECTOR"
LINEAR_CONNECTOR = "LINEAR_CONNECTOR"
DISCORD_CONNECTOR = "DISCORD_CONNECTOR"
JIRA_CONNECTOR = "JIRA_CONNECTOR"
class SearchSourceConnectorType(str, Enum):
SERPER_API = "SERPER_API" # NOT IMPLEMENTED YET : DON'T REMEMBER WHY : MOST PROBABLY BECAUSE WE NEED TO CRAWL THE RESULTS RETURNED BY IT
SERPER_API = "SERPER_API" # NOT IMPLEMENTED YET : DON'T REMEMBER WHY : MOST PROBABLY BECAUSE WE NEED TO CRAWL THE RESULTS RETURNED BY IT
TAVILY_API = "TAVILY_API"
LINKUP_API = "LINKUP_API"
SLACK_CONNECTOR = "SLACK_CONNECTOR"
@ -61,13 +63,16 @@ class SearchSourceConnectorType(str, Enum):
GITHUB_CONNECTOR = "GITHUB_CONNECTOR"
LINEAR_CONNECTOR = "LINEAR_CONNECTOR"
DISCORD_CONNECTOR = "DISCORD_CONNECTOR"
JIRA_CONNECTOR = "JIRA_CONNECTOR"
class ChatType(str, Enum):
QNA = "QNA"
REPORT_GENERAL = "REPORT_GENERAL"
REPORT_DEEP = "REPORT_DEEP"
REPORT_DEEPER = "REPORT_DEEPER"
class LiteLLMProvider(str, Enum):
OPENAI = "OPENAI"
ANTHROPIC = "ANTHROPIC"
@ -92,6 +97,7 @@ class LiteLLMProvider(str, Enum):
PETALS = "PETALS"
CUSTOM = "CUSTOM"
class LogLevel(str, Enum):
DEBUG = "DEBUG"
INFO = "INFO"
@ -99,18 +105,27 @@ class LogLevel(str, Enum):
ERROR = "ERROR"
CRITICAL = "CRITICAL"
class LogStatus(str, Enum):
IN_PROGRESS = "IN_PROGRESS"
SUCCESS = "SUCCESS"
FAILED = "FAILED"
class Base(DeclarativeBase):
pass
class TimestampMixin:
@declared_attr
def created_at(cls):
return Column(TIMESTAMP(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc), index=True)
return Column(
TIMESTAMP(timezone=True),
nullable=False,
default=lambda: datetime.now(timezone.utc),
index=True,
)
class BaseModel(Base):
__abstract__ = True
@ -118,6 +133,7 @@ class BaseModel(Base):
id = Column(Integer, primary_key=True, index=True)
class Chat(BaseModel, TimestampMixin):
__tablename__ = "chats"
@ -125,73 +141,115 @@ class Chat(BaseModel, TimestampMixin):
title = Column(String, nullable=False, index=True)
initial_connectors = Column(ARRAY(String), nullable=True)
messages = Column(JSON, nullable=False)
search_space_id = Column(Integer, ForeignKey('searchspaces.id', ondelete='CASCADE'), nullable=False)
search_space = relationship('SearchSpace', back_populates='chats')
search_space_id = Column(
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False
)
search_space = relationship("SearchSpace", back_populates="chats")
class Document(BaseModel, TimestampMixin):
__tablename__ = "documents"
title = Column(String, nullable=False, index=True)
document_type = Column(SQLAlchemyEnum(DocumentType), nullable=False)
document_metadata = Column(JSON, nullable=True)
content = Column(Text, nullable=False)
content_hash = Column(String, nullable=False, index=True, unique=True)
embedding = Column(Vector(config.embedding_model_instance.dimension))
search_space_id = Column(Integer, ForeignKey("searchspaces.id", ondelete='CASCADE'), nullable=False)
search_space_id = Column(
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False
)
search_space = relationship("SearchSpace", back_populates="documents")
chunks = relationship("Chunk", back_populates="document", cascade="all, delete-orphan")
chunks = relationship(
"Chunk", back_populates="document", cascade="all, delete-orphan"
)
class Chunk(BaseModel, TimestampMixin):
__tablename__ = "chunks"
content = Column(Text, nullable=False)
embedding = Column(Vector(config.embedding_model_instance.dimension))
document_id = Column(Integer, ForeignKey("documents.id", ondelete='CASCADE'), nullable=False)
document_id = Column(
Integer, ForeignKey("documents.id", ondelete="CASCADE"), nullable=False
)
document = relationship("Document", back_populates="chunks")
class Podcast(BaseModel, TimestampMixin):
__tablename__ = "podcasts"
title = Column(String, nullable=False, index=True)
podcast_transcript = Column(JSON, nullable=False, default={})
file_location = Column(String(500), nullable=False, default="")
search_space_id = Column(Integer, ForeignKey("searchspaces.id", ondelete='CASCADE'), nullable=False)
search_space_id = Column(
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False
)
search_space = relationship("SearchSpace", back_populates="podcasts")
class SearchSpace(BaseModel, TimestampMixin):
__tablename__ = "searchspaces"
name = Column(String(100), nullable=False, index=True)
description = Column(String(500), nullable=True)
user_id = Column(UUID(as_uuid=True), ForeignKey("user.id", ondelete='CASCADE'), nullable=False)
user_id = Column(
UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False
)
user = relationship("User", back_populates="search_spaces")
documents = relationship("Document", back_populates="search_space", order_by="Document.id", cascade="all, delete-orphan")
podcasts = relationship("Podcast", back_populates="search_space", order_by="Podcast.id", cascade="all, delete-orphan")
chats = relationship('Chat', back_populates='search_space', order_by='Chat.id', cascade="all, delete-orphan")
logs = relationship("Log", back_populates="search_space", order_by="Log.id", cascade="all, delete-orphan")
documents = relationship(
"Document",
back_populates="search_space",
order_by="Document.id",
cascade="all, delete-orphan",
)
podcasts = relationship(
"Podcast",
back_populates="search_space",
order_by="Podcast.id",
cascade="all, delete-orphan",
)
chats = relationship(
"Chat",
back_populates="search_space",
order_by="Chat.id",
cascade="all, delete-orphan",
)
logs = relationship(
"Log",
back_populates="search_space",
order_by="Log.id",
cascade="all, delete-orphan",
)
class SearchSourceConnector(BaseModel, TimestampMixin):
__tablename__ = "search_source_connectors"
name = Column(String(100), nullable=False, index=True)
connector_type = Column(SQLAlchemyEnum(SearchSourceConnectorType), nullable=False, unique=True)
connector_type = Column(
SQLAlchemyEnum(SearchSourceConnectorType), nullable=False, unique=True
)
is_indexable = Column(Boolean, nullable=False, default=False)
last_indexed_at = Column(TIMESTAMP(timezone=True), nullable=True)
config = Column(JSON, nullable=False)
user_id = Column(UUID(as_uuid=True), ForeignKey("user.id", ondelete='CASCADE'), nullable=False)
user_id = Column(
UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False
)
user = relationship("User", back_populates="search_source_connectors")
class LLMConfig(BaseModel, TimestampMixin):
__tablename__ = "llm_configs"
name = Column(String(100), nullable=False, index=True)
# Provider from the enum
provider = Column(SQLAlchemyEnum(LiteLLMProvider), nullable=False)
@ -202,78 +260,142 @@ class LLMConfig(BaseModel, TimestampMixin):
# API Key should be encrypted before storing
api_key = Column(String, nullable=False)
api_base = Column(String(500), nullable=True)
# For any other parameters that litellm supports
litellm_params = Column(JSON, nullable=True, default={})
user_id = Column(UUID(as_uuid=True), ForeignKey("user.id", ondelete='CASCADE'), nullable=False)
user_id = Column(
UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False
)
user = relationship("User", back_populates="llm_configs", foreign_keys=[user_id])
class Log(BaseModel, TimestampMixin):
__tablename__ = "logs"
level = Column(SQLAlchemyEnum(LogLevel), nullable=False, index=True)
status = Column(SQLAlchemyEnum(LogStatus), nullable=False, index=True)
message = Column(Text, nullable=False)
source = Column(String(200), nullable=True, index=True) # Service/component that generated the log
source = Column(
String(200), nullable=True, index=True
) # Service/component that generated the log
log_metadata = Column(JSON, nullable=True, default={}) # Additional context data
search_space_id = Column(Integer, ForeignKey("searchspaces.id", ondelete='CASCADE'), nullable=False)
search_space_id = Column(
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False
)
search_space = relationship("SearchSpace", back_populates="logs")
if config.AUTH_TYPE == "GOOGLE":
class OAuthAccount(SQLAlchemyBaseOAuthAccountTableUUID, Base):
pass
class User(SQLAlchemyBaseUserTableUUID, Base):
oauth_accounts: Mapped[list[OAuthAccount]] = relationship(
"OAuthAccount", lazy="joined"
)
search_spaces = relationship("SearchSpace", back_populates="user")
search_source_connectors = relationship("SearchSourceConnector", back_populates="user")
llm_configs = relationship("LLMConfig", back_populates="user", foreign_keys="LLMConfig.user_id", cascade="all, delete-orphan")
search_source_connectors = relationship(
"SearchSourceConnector", back_populates="user"
)
llm_configs = relationship(
"LLMConfig",
back_populates="user",
foreign_keys="LLMConfig.user_id",
cascade="all, delete-orphan",
)
long_context_llm_id = Column(Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True)
fast_llm_id = Column(Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True)
strategic_llm_id = Column(Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True)
long_context_llm_id = Column(
Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True
)
fast_llm_id = Column(
Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True
)
strategic_llm_id = Column(
Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True
)
long_context_llm = relationship(
"LLMConfig", foreign_keys=[long_context_llm_id], post_update=True
)
fast_llm = relationship(
"LLMConfig", foreign_keys=[fast_llm_id], post_update=True
)
strategic_llm = relationship(
"LLMConfig", foreign_keys=[strategic_llm_id], post_update=True
)
long_context_llm = relationship("LLMConfig", foreign_keys=[long_context_llm_id], post_update=True)
fast_llm = relationship("LLMConfig", foreign_keys=[fast_llm_id], post_update=True)
strategic_llm = relationship("LLMConfig", foreign_keys=[strategic_llm_id], post_update=True)
else:
class User(SQLAlchemyBaseUserTableUUID, Base):
search_spaces = relationship("SearchSpace", back_populates="user")
search_source_connectors = relationship("SearchSourceConnector", back_populates="user")
llm_configs = relationship("LLMConfig", back_populates="user", foreign_keys="LLMConfig.user_id", cascade="all, delete-orphan")
search_source_connectors = relationship(
"SearchSourceConnector", back_populates="user"
)
llm_configs = relationship(
"LLMConfig",
back_populates="user",
foreign_keys="LLMConfig.user_id",
cascade="all, delete-orphan",
)
long_context_llm_id = Column(Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True)
fast_llm_id = Column(Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True)
strategic_llm_id = Column(Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True)
long_context_llm_id = Column(
Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True
)
fast_llm_id = Column(
Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True
)
strategic_llm_id = Column(
Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True
)
long_context_llm = relationship("LLMConfig", foreign_keys=[long_context_llm_id], post_update=True)
fast_llm = relationship("LLMConfig", foreign_keys=[fast_llm_id], post_update=True)
strategic_llm = relationship("LLMConfig", foreign_keys=[strategic_llm_id], post_update=True)
long_context_llm = relationship(
"LLMConfig", foreign_keys=[long_context_llm_id], post_update=True
)
fast_llm = relationship(
"LLMConfig", foreign_keys=[fast_llm_id], post_update=True
)
strategic_llm = relationship(
"LLMConfig", foreign_keys=[strategic_llm_id], post_update=True
)
engine = create_async_engine(DATABASE_URL)
async_session_maker = async_sessionmaker(engine, expire_on_commit=False)
async def setup_indexes():
async with engine.begin() as conn:
# Create indexes
# Create indexes
# Document Summary Indexes
await conn.execute(text('CREATE INDEX IF NOT EXISTS document_vector_index ON documents USING hnsw (embedding public.vector_cosine_ops)'))
await conn.execute(text('CREATE INDEX IF NOT EXISTS document_search_index ON documents USING gin (to_tsvector(\'english\', content))'))
await conn.execute(
text(
"CREATE INDEX IF NOT EXISTS document_vector_index ON documents USING hnsw (embedding public.vector_cosine_ops)"
)
)
await conn.execute(
text(
"CREATE INDEX IF NOT EXISTS document_search_index ON documents USING gin (to_tsvector('english', content))"
)
)
# Document Chuck Indexes
await conn.execute(text('CREATE INDEX IF NOT EXISTS chucks_vector_index ON chunks USING hnsw (embedding public.vector_cosine_ops)'))
await conn.execute(text('CREATE INDEX IF NOT EXISTS chucks_search_index ON chunks USING gin (to_tsvector(\'english\', content))'))
await conn.execute(
text(
"CREATE INDEX IF NOT EXISTS chucks_vector_index ON chunks USING hnsw (embedding public.vector_cosine_ops)"
)
)
await conn.execute(
text(
"CREATE INDEX IF NOT EXISTS chucks_search_index ON chunks USING gin (to_tsvector('english', content))"
)
)
async def create_db_and_tables():
async with engine.begin() as conn:
await conn.execute(text('CREATE EXTENSION IF NOT EXISTS vector'))
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
await conn.run_sync(Base.metadata.create_all)
await setup_indexes()
@ -284,14 +406,23 @@ async def get_async_session() -> AsyncGenerator[AsyncSession, None]:
if config.AUTH_TYPE == "GOOGLE":
async def get_user_db(session: AsyncSession = Depends(get_async_session)):
yield SQLAlchemyUserDatabase(session, User, OAuthAccount)
else:
async def get_user_db(session: AsyncSession = Depends(get_async_session)):
yield SQLAlchemyUserDatabase(session, User)
async def get_chucks_hybrid_search_retriever(session: AsyncSession = Depends(get_async_session)):
async def get_chucks_hybrid_search_retriever(
session: AsyncSession = Depends(get_async_session),
):
return ChucksHybridSearchRetriever(session)
async def get_documents_hybrid_search_retriever(session: AsyncSession = Depends(get_async_session)):
async def get_documents_hybrid_search_retriever(
session: AsyncSession = Depends(get_async_session),
):
return DocumentHybridSearchRetriever(session)

544
surfsense_backend/app/routes/search_source_connectors_routes.py
View file

@ -9,35 +9,58 @@ POST /search-source-connectors/{connector_id}/index - Index content from a conne
Note: Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, NOTION_CONNECTOR, GITHUB_CONNECTOR, LINEAR_CONNECTOR, DISCORD_CONNECTOR).
"""
from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks, Body
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from sqlalchemy.exc import IntegrityError
from typing import List, Dict, Any
from app.db import get_async_session, User, SearchSourceConnector, SearchSourceConnectorType, SearchSpace, async_session_maker
from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead, SearchSourceConnectorBase
import logging
from datetime import datetime, timedelta
from typing import Any, Dict, List
from app.connectors.github_connector import GitHubConnector
from app.db import (
SearchSourceConnector,
SearchSourceConnectorType,
SearchSpace,
User,
async_session_maker,
get_async_session,
)
from app.schemas import (
SearchSourceConnectorBase,
SearchSourceConnectorCreate,
SearchSourceConnectorRead,
SearchSourceConnectorUpdate,
)
from app.tasks.connectors_indexing_tasks import (
index_discord_messages,
index_github_repos,
index_jira_issues,
index_linear_issues,
index_notion_pages,
index_slack_messages,
)
from app.users import current_active_user
from app.utils.check_ownership import check_ownership
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query
from pydantic import BaseModel, Field, ValidationError
from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos, index_linear_issues, index_discord_messages
from app.connectors.github_connector import GitHubConnector
from datetime import datetime, timedelta
import logging
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
# Set up logging
logger = logging.getLogger(__name__)
router = APIRouter()
# Use Pydantic's BaseModel here
class GitHubPATRequest(BaseModel):
github_pat: str = Field(..., description="GitHub Personal Access Token")
# --- New Endpoint to list GitHub Repositories ---
@router.post("/github/repositories/", response_model=List[Dict[str, Any]])
async def list_github_repositories(
pat_request: GitHubPATRequest,
user: User = Depends(current_active_user) # Ensure the user is logged in
user: User = Depends(current_active_user), # Ensure the user is logged in
):
"""
Fetches a list of repositories accessible by the provided GitHub PAT.
@ -54,35 +77,39 @@ async def list_github_repositories(
logger.error(f"GitHub PAT validation failed for user {user.id}: {str(e)}")
raise HTTPException(status_code=400, detail=f"Invalid GitHub PAT: {str(e)}")
except Exception as e:
logger.error(f"Failed to fetch GitHub repositories for user {user.id}: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to fetch GitHub repositories.")
logger.error(
f"Failed to fetch GitHub repositories for user {user.id}: {str(e)}"
)
raise HTTPException(
status_code=500, detail="Failed to fetch GitHub repositories."
)
@router.post("/search-source-connectors/", response_model=SearchSourceConnectorRead)
async def create_search_source_connector(
connector: SearchSourceConnectorCreate,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user)
user: User = Depends(current_active_user),
):
"""
Create a new search source connector.
Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, etc.).
The config must contain the appropriate keys for the connector type.
"""
try:
# Check if a connector with the same type already exists for this user
result = await session.execute(
select(SearchSourceConnector)
.filter(
select(SearchSourceConnector).filter(
SearchSourceConnector.user_id == user.id,
SearchSourceConnector.connector_type == connector.connector_type
SearchSourceConnector.connector_type == connector.connector_type,
)
)
existing_connector = result.scalars().first()
if existing_connector:
raise HTTPException(
status_code=409,
detail=f"A connector with type {connector.connector_type} already exists. Each user can have only one connector of each type."
detail=f"A connector with type {connector.connector_type} already exists. Each user can have only one connector of each type.",
)
db_connector = SearchSourceConnector(**connector.model_dump(), user_id=user.id)
session.add(db_connector)
@ -91,15 +118,12 @@ async def create_search_source_connector(
return db_connector
except ValidationError as e:
await session.rollback()
raise HTTPException(
status_code=422,
detail=f"Validation error: {str(e)}"
)
raise HTTPException(status_code=422, detail=f"Validation error: {str(e)}")
except IntegrityError as e:
await session.rollback()
raise HTTPException(
status_code=409,
detail=f"Integrity error: A connector with this type already exists. {str(e)}"
detail=f"Integrity error: A connector with this type already exists. {str(e)}",
)
except HTTPException:
await session.rollback()
@ -109,38 +133,44 @@ async def create_search_source_connector(
await session.rollback()
raise HTTPException(
status_code=500,
detail=f"Failed to create search source connector: {str(e)}"
detail=f"Failed to create search source connector: {str(e)}",
)
@router.get("/search-source-connectors/", response_model=List[SearchSourceConnectorRead])
@router.get(
"/search-source-connectors/", response_model=List[SearchSourceConnectorRead]
)
async def read_search_source_connectors(
skip: int = 0,
limit: int = 100,
search_space_id: int = None,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user)
user: User = Depends(current_active_user),
):
"""List all search source connectors for the current user."""
try:
query = select(SearchSourceConnector).filter(SearchSourceConnector.user_id == user.id)
# No need to filter by search_space_id as connectors are user-owned, not search space specific
result = await session.execute(
query.offset(skip).limit(limit)
query = select(SearchSourceConnector).filter(
SearchSourceConnector.user_id == user.id
)
# No need to filter by search_space_id as connectors are user-owned, not search space specific
result = await session.execute(query.offset(skip).limit(limit))
return result.scalars().all()
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to fetch search source connectors: {str(e)}"
detail=f"Failed to fetch search source connectors: {str(e)}",
)
@router.get("/search-source-connectors/{connector_id}", response_model=SearchSourceConnectorRead)
@router.get(
"/search-source-connectors/{connector_id}", response_model=SearchSourceConnectorRead
)
async def read_search_source_connector(
connector_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user)
user: User = Depends(current_active_user),
):
"""Get a specific search source connector by ID."""
try:
@ -149,31 +179,37 @@ async def read_search_source_connector(
raise
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to fetch search source connector: {str(e)}"
status_code=500, detail=f"Failed to fetch search source connector: {str(e)}"
)
@router.put("/search-source-connectors/{connector_id}", response_model=SearchSourceConnectorRead)
@router.put(
"/search-source-connectors/{connector_id}", response_model=SearchSourceConnectorRead
)
async def update_search_source_connector(
connector_id: int,
connector_update: SearchSourceConnectorUpdate,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user)
user: User = Depends(current_active_user),
):
"""
Update a search source connector.
Handles partial updates, including merging changes into the 'config' field.
"""
db_connector = await check_ownership(session, SearchSourceConnector, connector_id, user)
db_connector = await check_ownership(
session, SearchSourceConnector, connector_id, user
)
# Convert the sparse update data (only fields present in request) to a dict
update_data = connector_update.model_dump(exclude_unset=True)
# Special handling for 'config' field
if "config" in update_data:
incoming_config = update_data["config"] # Config data from the request
existing_config = db_connector.config if db_connector.config else {} # Current config from DB
incoming_config = update_data["config"] # Config data from the request
existing_config = (
db_connector.config if db_connector.config else {}
) # Current config from DB
# Merge incoming config into existing config
# This preserves existing keys (like GITHUB_PAT) if they are not in the incoming data
merged_config = existing_config.copy()
@ -182,26 +218,29 @@ async def update_search_source_connector(
# -- Validation after merging --
# Validate the *merged* config based on the connector type
# We need the connector type - use the one from the update if provided, else the existing one
current_connector_type = connector_update.connector_type if connector_update.connector_type is not None else db_connector.connector_type
current_connector_type = (
connector_update.connector_type
if connector_update.connector_type is not None
else db_connector.connector_type
)
try:
# We can reuse the base validator by creating a temporary base model instance
# Note: This assumes 'name' and 'is_indexable' are not crucial for config validation itself
temp_data_for_validation = {
"name": db_connector.name, # Use existing name
"name": db_connector.name, # Use existing name
"connector_type": current_connector_type,
"is_indexable": db_connector.is_indexable, # Use existing value
"last_indexed_at": db_connector.last_indexed_at, # Not used by validator
"config": merged_config
"is_indexable": db_connector.is_indexable, # Use existing value
"last_indexed_at": db_connector.last_indexed_at, # Not used by validator
"config": merged_config,
}
SearchSourceConnectorBase.model_validate(temp_data_for_validation)
except ValidationError as e:
# Raise specific validation error for the merged config
raise HTTPException(
status_code=422,
detail=f"Validation error for merged config: {str(e)}"
status_code=422, detail=f"Validation error for merged config: {str(e)}"
)
# If validation passes, update the main update_data dict with the merged config
update_data["config"] = merged_config
@ -210,20 +249,19 @@ async def update_search_source_connector(
# Prevent changing connector_type if it causes a duplicate (check moved here)
if key == "connector_type" and value != db_connector.connector_type:
result = await session.execute(
select(SearchSourceConnector)
.filter(
select(SearchSourceConnector).filter(
SearchSourceConnector.user_id == user.id,
SearchSourceConnector.connector_type == value,
SearchSourceConnector.id != connector_id
SearchSourceConnector.id != connector_id,
)
)
existing_connector = result.scalars().first()
if existing_connector:
raise HTTPException(
status_code=409,
detail=f"A connector with type {value} already exists. Each user can have only one connector of each type."
detail=f"A connector with type {value} already exists. Each user can have only one connector of each type.",
)
setattr(db_connector, key, value)
try:
@ -234,26 +272,31 @@ async def update_search_source_connector(
await session.rollback()
# This might occur if connector_type constraint is violated somehow after the check
raise HTTPException(
status_code=409,
detail=f"Database integrity error during update: {str(e)}"
status_code=409, detail=f"Database integrity error during update: {str(e)}"
)
except Exception as e:
await session.rollback()
logger.error(f"Failed to update search source connector {connector_id}: {e}", exc_info=True)
logger.error(
f"Failed to update search source connector {connector_id}: {e}",
exc_info=True,
)
raise HTTPException(
status_code=500,
detail=f"Failed to update search source connector: {str(e)}"
detail=f"Failed to update search source connector: {str(e)}",
)
@router.delete("/search-source-connectors/{connector_id}", response_model=dict)
async def delete_search_source_connector(
connector_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user)
user: User = Depends(current_active_user),
):
"""Delete a search source connector."""
try:
db_connector = await check_ownership(session, SearchSourceConnector, connector_id, user)
db_connector = await check_ownership(
session, SearchSourceConnector, connector_id, user
)
await session.delete(db_connector)
await session.commit()
return {"message": "Search source connector deleted successfully"}
@ -263,48 +306,64 @@ async def delete_search_source_connector(
await session.rollback()
raise HTTPException(
status_code=500,
detail=f"Failed to delete search source connector: {str(e)}"
detail=f"Failed to delete search source connector: {str(e)}",
)
@router.post("/search-source-connectors/{connector_id}/index", response_model=Dict[str, Any])
@router.post(
"/search-source-connectors/{connector_id}/index", response_model=Dict[str, Any]
)
async def index_connector_content(
connector_id: int,
search_space_id: int = Query(..., description="ID of the search space to store indexed content"),
start_date: str = Query(None, description="Start date for indexing (YYYY-MM-DD format). If not provided, uses last_indexed_at or defaults to 365 days ago"),
end_date: str = Query(None, description="End date for indexing (YYYY-MM-DD format). If not provided, uses today's date"),
search_space_id: int = Query(
..., description="ID of the search space to store indexed content"
),
start_date: str = Query(
None,
description="Start date for indexing (YYYY-MM-DD format). If not provided, uses last_indexed_at or defaults to 365 days ago",
),
end_date: str = Query(
None,
description="End date for indexing (YYYY-MM-DD format). If not provided, uses today's date",
),
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
background_tasks: BackgroundTasks = None
background_tasks: BackgroundTasks = None,
):
"""
Index content from a connector to a search space.
Currently supports:
- SLACK_CONNECTOR: Indexes messages from all accessible Slack channels
- NOTION_CONNECTOR: Indexes pages from all accessible Notion pages
- GITHUB_CONNECTOR: Indexes code and documentation from GitHub repositories
- LINEAR_CONNECTOR: Indexes issues and comments from Linear
- JIRA_CONNECTOR: Indexes issues and comments from Jira
- DISCORD_CONNECTOR: Indexes messages from all accessible Discord channels
Args:
connector_id: ID of the connector to use
search_space_id: ID of the search space to store indexed content
background_tasks: FastAPI background tasks
Returns:
Dictionary with indexing status
"""
try:
# Check if the connector belongs to the user
connector = await check_ownership(session, SearchSourceConnector, connector_id, user)
connector = await check_ownership(
session, SearchSourceConnector, connector_id, user
)
# Check if the search space belongs to the user
search_space = await check_ownership(session, SearchSpace, search_space_id, user)
search_space = await check_ownership(
session, SearchSpace, search_space_id, user
)
# Handle different connector types
response_message = ""
today_str = datetime.now().strftime("%Y-%m-%d")
# Determine the actual date range to use
if start_date is None:
# Use last_indexed_at or default to 365 days ago
@ -316,10 +375,12 @@ async def index_connector_content(
else:
indexing_from = connector.last_indexed_at.strftime("%Y-%m-%d")
else:
indexing_from = (datetime.now() - timedelta(days=365)).strftime("%Y-%m-%d")
indexing_from = (datetime.now() - timedelta(days=365)).strftime(
"%Y-%m-%d"
)
else:
indexing_from = start_date
if end_date is None:
indexing_to = today_str
else:
@ -327,99 +388,162 @@ async def index_connector_content(
if connector.connector_type == SearchSourceConnectorType.SLACK_CONNECTOR:
# Run indexing in background
logger.info(f"Triggering Slack indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}")
background_tasks.add_task(run_slack_indexing_with_new_session, connector_id, search_space_id, str(user.id), indexing_from, indexing_to)
logger.info(
f"Triggering Slack indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}"
)
background_tasks.add_task(
run_slack_indexing_with_new_session,
connector_id,
search_space_id,
str(user.id),
indexing_from,
indexing_to,
)
response_message = "Slack indexing started in the background."
elif connector.connector_type == SearchSourceConnectorType.NOTION_CONNECTOR:
# Run indexing in background
logger.info(f"Triggering Notion indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}")
background_tasks.add_task(run_notion_indexing_with_new_session, connector_id, search_space_id, str(user.id), indexing_from, indexing_to)
logger.info(
f"Triggering Notion indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}"
)
background_tasks.add_task(
run_notion_indexing_with_new_session,
connector_id,
search_space_id,
str(user.id),
indexing_from,
indexing_to,
)
response_message = "Notion indexing started in the background."
elif connector.connector_type == SearchSourceConnectorType.GITHUB_CONNECTOR:
# Run indexing in background
logger.info(f"Triggering GitHub indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}")
background_tasks.add_task(run_github_indexing_with_new_session, connector_id, search_space_id, str(user.id), indexing_from, indexing_to)
logger.info(
f"Triggering GitHub indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}"
)
background_tasks.add_task(
run_github_indexing_with_new_session,
connector_id,
search_space_id,
str(user.id),
indexing_from,
indexing_to,
)
response_message = "GitHub indexing started in the background."
elif connector.connector_type == SearchSourceConnectorType.LINEAR_CONNECTOR:
# Run indexing in background
logger.info(f"Triggering Linear indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}")
background_tasks.add_task(run_linear_indexing_with_new_session, connector_id, search_space_id, str(user.id), indexing_from, indexing_to)
logger.info(
f"Triggering Linear indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}"
)
background_tasks.add_task(
run_linear_indexing_with_new_session,
connector_id,
search_space_id,
str(user.id),
indexing_from,
indexing_to,
)
response_message = "Linear indexing started in the background."
elif connector.connector_type == SearchSourceConnectorType.JIRA_CONNECTOR:
# Run indexing in background
logger.info(
f"Triggering Jira indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}"
)
background_tasks.add_task(
run_jira_indexing_with_new_session,
connector_id,
search_space_id,
str(user.id),
indexing_from,
indexing_to,
)
response_message = "Jira indexing started in the background."
elif connector.connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR:
# Run indexing in background
logger.info(
f"Triggering Discord indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}"
)
background_tasks.add_task(
run_discord_indexing_with_new_session, connector_id, search_space_id, str(user.id), indexing_from, indexing_to
run_discord_indexing_with_new_session,
connector_id,
search_space_id,
str(user.id),
indexing_from,
indexing_to,
)
response_message = "Discord indexing started in the background."
else:
raise HTTPException(
status_code=400,
detail=f"Indexing not supported for connector type: {connector.connector_type}"
detail=f"Indexing not supported for connector type: {connector.connector_type}",
)
return {
"message": response_message,
"connector_id": connector_id,
"message": response_message,
"connector_id": connector_id,
"search_space_id": search_space_id,
"indexing_from": indexing_from,
"indexing_to": indexing_to
"indexing_to": indexing_to,
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to initiate indexing for connector {connector_id}: {e}", exc_info=True)
raise HTTPException(
status_code=500,
detail=f"Failed to initiate indexing: {str(e)}"
logger.error(
f"Failed to initiate indexing for connector {connector_id}: {e}",
exc_info=True,
)
async def update_connector_last_indexed(
session: AsyncSession,
connector_id: int
):
raise HTTPException(
status_code=500, detail=f"Failed to initiate indexing: {str(e)}"
)
async def update_connector_last_indexed(session: AsyncSession, connector_id: int):
"""
Update the last_indexed_at timestamp for a connector.
Args:
session: Database session
connector_id: ID of the connector to update
"""
try:
result = await session.execute(
select(SearchSourceConnector)
.filter(SearchSourceConnector.id == connector_id)
select(SearchSourceConnector).filter(
SearchSourceConnector.id == connector_id
)
)
connector = result.scalars().first()
if connector:
connector.last_indexed_at = datetime.now()
await session.commit()
logger.info(f"Updated last_indexed_at for connector {connector_id}")
except Exception as e:
logger.error(f"Failed to update last_indexed_at for connector {connector_id}: {str(e)}")
logger.error(
f"Failed to update last_indexed_at for connector {connector_id}: {str(e)}"
)
await session.rollback()
async def run_slack_indexing_with_new_session(
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str
end_date: str,
):
"""
Create a new session and run the Slack indexing task.
This prevents session leaks by creating a dedicated session for the background task.
"""
async with async_session_maker() as session:
await run_slack_indexing(session, connector_id, search_space_id, user_id, start_date, end_date)
await run_slack_indexing(
session, connector_id, search_space_id, user_id, start_date, end_date
)
async def run_slack_indexing(
session: AsyncSession,
@ -427,11 +551,11 @@ async def run_slack_indexing(
search_space_id: int,
user_id: str,
start_date: str,
end_date: str
end_date: str,
):
"""
Background task to run Slack indexing.
Args:
session: Database session
connector_id: ID of the Slack connector
@ -449,31 +573,39 @@ async def run_slack_indexing(
user_id=user_id,
start_date=start_date,
end_date=end_date,
update_last_indexed=False # Don't update timestamp in the indexing function
update_last_indexed=False, # Don't update timestamp in the indexing function
)
# Only update last_indexed_at if indexing was successful (either new docs or updated docs)
if documents_processed > 0:
await update_connector_last_indexed(session, connector_id)
logger.info(f"Slack indexing completed successfully: {documents_processed} documents processed")
logger.info(
f"Slack indexing completed successfully: {documents_processed} documents processed"
)
else:
logger.error(f"Slack indexing failed or no documents processed: {error_or_warning}")
logger.error(
f"Slack indexing failed or no documents processed: {error_or_warning}"
)
except Exception as e:
logger.error(f"Error in background Slack indexing task: {str(e)}")
async def run_notion_indexing_with_new_session(
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str
end_date: str,
):
"""
Create a new session and run the Notion indexing task.
This prevents session leaks by creating a dedicated session for the background task.
"""
async with async_session_maker() as session:
await run_notion_indexing(session, connector_id, search_space_id, user_id, start_date, end_date)
await run_notion_indexing(
session, connector_id, search_space_id, user_id, start_date, end_date
)
async def run_notion_indexing(
session: AsyncSession,
@ -481,11 +613,11 @@ async def run_notion_indexing(
search_space_id: int,
user_id: str,
start_date: str,
end_date: str
end_date: str,
):
"""
Background task to run Notion indexing.
Args:
session: Database session
connector_id: ID of the Notion connector
@ -503,112 +635,158 @@ async def run_notion_indexing(
user_id=user_id,
start_date=start_date,
end_date=end_date,
update_last_indexed=False # Don't update timestamp in the indexing function
update_last_indexed=False, # Don't update timestamp in the indexing function
)
# Only update last_indexed_at if indexing was successful (either new docs or updated docs)
if documents_processed > 0:
await update_connector_last_indexed(session, connector_id)
logger.info(f"Notion indexing completed successfully: {documents_processed} documents processed")
logger.info(
f"Notion indexing completed successfully: {documents_processed} documents processed"
)
else:
logger.error(f"Notion indexing failed or no documents processed: {error_or_warning}")
logger.error(
f"Notion indexing failed or no documents processed: {error_or_warning}"
)
except Exception as e:
logger.error(f"Error in background Notion indexing task: {str(e)}")
# Add new helper functions for GitHub indexing
async def run_github_indexing_with_new_session(
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str
end_date: str,
):
"""Wrapper to run GitHub indexing with its own database session."""
logger.info(f"Background task started: Indexing GitHub connector {connector_id} into space {search_space_id} from {start_date} to {end_date}")
logger.info(
f"Background task started: Indexing GitHub connector {connector_id} into space {search_space_id} from {start_date} to {end_date}"
)
async with async_session_maker() as session:
await run_github_indexing(session, connector_id, search_space_id, user_id, start_date, end_date)
await run_github_indexing(
session, connector_id, search_space_id, user_id, start_date, end_date
)
logger.info(f"Background task finished: Indexing GitHub connector {connector_id}")
async def run_github_indexing(
session: AsyncSession,
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str
end_date: str,
):
"""Runs the GitHub indexing task and updates the timestamp."""
try:
indexed_count, error_message = await index_github_repos(
session, connector_id, search_space_id, user_id, start_date, end_date, update_last_indexed=False
session,
connector_id,
search_space_id,
user_id,
start_date,
end_date,
update_last_indexed=False,
)
if error_message:
logger.error(f"GitHub indexing failed for connector {connector_id}: {error_message}")
logger.error(
f"GitHub indexing failed for connector {connector_id}: {error_message}"
)
# Optionally update status in DB to indicate failure
else:
logger.info(f"GitHub indexing successful for connector {connector_id}. Indexed {indexed_count} documents.")
logger.info(
f"GitHub indexing successful for connector {connector_id}. Indexed {indexed_count} documents."
)
# Update the last indexed timestamp only on success
await update_connector_last_indexed(session, connector_id)
await session.commit() # Commit timestamp update
await session.commit() # Commit timestamp update
except Exception as e:
await session.rollback()
logger.error(f"Critical error in run_github_indexing for connector {connector_id}: {e}", exc_info=True)
logger.error(
f"Critical error in run_github_indexing for connector {connector_id}: {e}",
exc_info=True,
)
# Optionally update status in DB to indicate failure
# Add new helper functions for Linear indexing
async def run_linear_indexing_with_new_session(
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str
end_date: str,
):
"""Wrapper to run Linear indexing with its own database session."""
logger.info(f"Background task started: Indexing Linear connector {connector_id} into space {search_space_id} from {start_date} to {end_date}")
logger.info(
f"Background task started: Indexing Linear connector {connector_id} into space {search_space_id} from {start_date} to {end_date}"
)
async with async_session_maker() as session:
await run_linear_indexing(session, connector_id, search_space_id, user_id, start_date, end_date)
await run_linear_indexing(
session, connector_id, search_space_id, user_id, start_date, end_date
)
logger.info(f"Background task finished: Indexing Linear connector {connector_id}")
async def run_linear_indexing(
session: AsyncSession,
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str
end_date: str,
):
"""Runs the Linear indexing task and updates the timestamp."""
try:
indexed_count, error_message = await index_linear_issues(
session, connector_id, search_space_id, user_id, start_date, end_date, update_last_indexed=False
session,
connector_id,
search_space_id,
user_id,
start_date,
end_date,
update_last_indexed=False,
)
if error_message:
logger.error(f"Linear indexing failed for connector {connector_id}: {error_message}")
logger.error(
f"Linear indexing failed for connector {connector_id}: {error_message}"
)
# Optionally update status in DB to indicate failure
else:
logger.info(f"Linear indexing successful for connector {connector_id}. Indexed {indexed_count} documents.")
logger.info(
f"Linear indexing successful for connector {connector_id}. Indexed {indexed_count} documents."
)
# Update the last indexed timestamp only on success
await update_connector_last_indexed(session, connector_id)
await session.commit() # Commit timestamp update
await session.commit() # Commit timestamp update
except Exception as e:
await session.rollback()
logger.error(f"Critical error in run_linear_indexing for connector {connector_id}: {e}", exc_info=True)
logger.error(
f"Critical error in run_linear_indexing for connector {connector_id}: {e}",
exc_info=True,
)
# Optionally update status in DB to indicate failure
# Add new helper functions for discord indexing
async def run_discord_indexing_with_new_session(
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str
end_date: str,
):
"""
Create a new session and run the Discord indexing task.
This prevents session leaks by creating a dedicated session for the background task.
"""
async with async_session_maker() as session:
await run_discord_indexing(session, connector_id, search_space_id, user_id, start_date, end_date)
await run_discord_indexing(
session, connector_id, search_space_id, user_id, start_date, end_date
)
async def run_discord_indexing(
session: AsyncSession,
@ -616,7 +794,7 @@ async def run_discord_indexing(
search_space_id: int,
user_id: str,
start_date: str,
end_date: str
end_date: str,
):
"""
Background task to run Discord indexing.
@ -637,14 +815,76 @@ async def run_discord_indexing(
user_id=user_id,
start_date=start_date,
end_date=end_date,
update_last_indexed=False # Don't update timestamp in the indexing function
update_last_indexed=False, # Don't update timestamp in the indexing function
)
# Only update last_indexed_at if indexing was successful (either new docs or updated docs)
if documents_processed > 0:
await update_connector_last_indexed(session, connector_id)
logger.info(f"Discord indexing completed successfully: {documents_processed} documents processed")
logger.info(
f"Discord indexing completed successfully: {documents_processed} documents processed"
)
else:
logger.error(f"Discord indexing failed or no documents processed: {error_or_warning}")
logger.error(
f"Discord indexing failed or no documents processed: {error_or_warning}"
)
except Exception as e:
logger.error(f"Error in background Discord indexing task: {str(e)}")
logger.error(f"Error in background Discord indexing task: {str(e)}")
# Add new helper functions for Jira indexing
async def run_jira_indexing_with_new_session(
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str,
):
"""Wrapper to run Jira indexing with its own database session."""
logger.info(
f"Background task started: Indexing Jira connector {connector_id} into space {search_space_id} from {start_date} to {end_date}"
)
async with async_session_maker() as session:
await run_jira_indexing(
session, connector_id, search_space_id, user_id, start_date, end_date
)
logger.info(f"Background task finished: Indexing Jira connector {connector_id}")
async def run_jira_indexing(
session: AsyncSession,
connector_id: int,
search_space_id: int,
user_id: str,
start_date: str,
end_date: str,
):
"""Runs the Jira indexing task and updates the timestamp."""
try:
indexed_count, error_message = await index_jira_issues(
session,
connector_id,
search_space_id,
user_id,
start_date,
end_date,
update_last_indexed=False,
)
if error_message:
logger.error(
f"Jira indexing failed for connector {connector_id}: {error_message}"
)
# Optionally update status in DB to indicate failure
else:
logger.info(
f"Jira indexing successful for connector {connector_id}. Indexed {indexed_count} documents."
)
# Update the last indexed timestamp only on success
await update_connector_last_indexed(session, connector_id)
await session.commit() # Commit timestamp update
except Exception as e:
logger.error(
f"Critical error in run_jira_indexing for connector {connector_id}: {e}",
exc_info=True,
)
# Optionally update status in DB to indicate failure

107
surfsense_backend/app/schemas/search_source_connector.py
View file

@ -1,9 +1,12 @@
from datetime import datetime
import uuid
from typing import Dict, Any, Optional
from pydantic import BaseModel, field_validator, ConfigDict
from .base import IDModel, TimestampModel
from datetime import datetime
from typing import Any, Dict, Optional
from app.db import SearchSourceConnectorType
from pydantic import BaseModel, ConfigDict, field_validator
from .base import IDModel, TimestampModel
class SearchSourceConnectorBase(BaseModel):
name: str
@ -11,102 +14,141 @@ class SearchSourceConnectorBase(BaseModel):
is_indexable: bool
last_indexed_at: Optional[datetime] = None
config: Dict[str, Any]
@field_validator('config')
@field_validator("config")
@classmethod
def validate_config_for_connector_type(cls, config: Dict[str, Any], values: Dict[str, Any]) -> Dict[str, Any]:
connector_type = values.data.get('connector_type')
def validate_config_for_connector_type(
cls, config: Dict[str, Any], values: Dict[str, Any]
) -> Dict[str, Any]:
connector_type = values.data.get("connector_type")
if connector_type == SearchSourceConnectorType.SERPER_API:
# For SERPER_API, only allow SERPER_API_KEY
allowed_keys = ["SERPER_API_KEY"]
if set(config.keys()) != set(allowed_keys):
raise ValueError(f"For SERPER_API connector type, config must only contain these keys: {allowed_keys}")
raise ValueError(
f"For SERPER_API connector type, config must only contain these keys: {allowed_keys}"
)
# Ensure the API key is not empty
if not config.get("SERPER_API_KEY"):
raise ValueError("SERPER_API_KEY cannot be empty")
elif connector_type == SearchSourceConnectorType.TAVILY_API:
# For TAVILY_API, only allow TAVILY_API_KEY
allowed_keys = ["TAVILY_API_KEY"]
if set(config.keys()) != set(allowed_keys):
raise ValueError(f"For TAVILY_API connector type, config must only contain these keys: {allowed_keys}")
raise ValueError(
f"For TAVILY_API connector type, config must only contain these keys: {allowed_keys}"
)
# Ensure the API key is not empty
if not config.get("TAVILY_API_KEY"):
raise ValueError("TAVILY_API_KEY cannot be empty")
elif connector_type == SearchSourceConnectorType.LINKUP_API:
# For LINKUP_API, only allow LINKUP_API_KEY
allowed_keys = ["LINKUP_API_KEY"]
if set(config.keys()) != set(allowed_keys):
raise ValueError(f"For LINKUP_API connector type, config must only contain these keys: {allowed_keys}")
raise ValueError(
f"For LINKUP_API connector type, config must only contain these keys: {allowed_keys}"
)
# Ensure the API key is not empty
if not config.get("LINKUP_API_KEY"):
raise ValueError("LINKUP_API_KEY cannot be empty")
elif connector_type == SearchSourceConnectorType.SLACK_CONNECTOR:
# For SLACK_CONNECTOR, only allow SLACK_BOT_TOKEN
allowed_keys = ["SLACK_BOT_TOKEN"]
if set(config.keys()) != set(allowed_keys):
raise ValueError(f"For SLACK_CONNECTOR connector type, config must only contain these keys: {allowed_keys}")
raise ValueError(
f"For SLACK_CONNECTOR connector type, config must only contain these keys: {allowed_keys}"
)
# Ensure the bot token is not empty
if not config.get("SLACK_BOT_TOKEN"):
raise ValueError("SLACK_BOT_TOKEN cannot be empty")
elif connector_type == SearchSourceConnectorType.NOTION_CONNECTOR:
# For NOTION_CONNECTOR, only allow NOTION_INTEGRATION_TOKEN
allowed_keys = ["NOTION_INTEGRATION_TOKEN"]
if set(config.keys()) != set(allowed_keys):
raise ValueError(f"For NOTION_CONNECTOR connector type, config must only contain these keys: {allowed_keys}")
raise ValueError(
f"For NOTION_CONNECTOR connector type, config must only contain these keys: {allowed_keys}"
)
# Ensure the integration token is not empty
if not config.get("NOTION_INTEGRATION_TOKEN"):
raise ValueError("NOTION_INTEGRATION_TOKEN cannot be empty")
elif connector_type == SearchSourceConnectorType.GITHUB_CONNECTOR:
# For GITHUB_CONNECTOR, only allow GITHUB_PAT and repo_full_names
allowed_keys = ["GITHUB_PAT", "repo_full_names"]
if set(config.keys()) != set(allowed_keys):
raise ValueError(f"For GITHUB_CONNECTOR connector type, config must only contain these keys: {allowed_keys}")
raise ValueError(
f"For GITHUB_CONNECTOR connector type, config must only contain these keys: {allowed_keys}"
)
# Ensure the token is not empty
if not config.get("GITHUB_PAT"):
raise ValueError("GITHUB_PAT cannot be empty")
# Ensure the repo_full_names is present and is a non-empty list
repo_full_names = config.get("repo_full_names")
if not isinstance(repo_full_names, list) or not repo_full_names:
raise ValueError("repo_full_names must be a non-empty list of strings")
elif connector_type == SearchSourceConnectorType.LINEAR_CONNECTOR:
# For LINEAR_CONNECTOR, only allow LINEAR_API_KEY
allowed_keys = ["LINEAR_API_KEY"]
if set(config.keys()) != set(allowed_keys):
raise ValueError(f"For LINEAR_CONNECTOR connector type, config must only contain these keys: {allowed_keys}")
raise ValueError(
f"For LINEAR_CONNECTOR connector type, config must only contain these keys: {allowed_keys}"
)
# Ensure the token is not empty
if not config.get("LINEAR_API_KEY"):
raise ValueError("LINEAR_API_KEY cannot be empty")
elif connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR:
# For DISCORD_CONNECTOR, only allow DISCORD_BOT_TOKEN
allowed_keys = ["DISCORD_BOT_TOKEN"]
if set(config.keys()) != set(allowed_keys):
raise ValueError(f"For DISCORD_CONNECTOR connector type, config must only contain these keys: {allowed_keys}")
raise ValueError(
f"For DISCORD_CONNECTOR connector type, config must only contain these keys: {allowed_keys}"
)
# Ensure the bot token is not empty
if not config.get("DISCORD_BOT_TOKEN"):
raise ValueError("DISCORD_BOT_TOKEN cannot be empty")
elif connector_type == SearchSourceConnectorType.JIRA_CONNECTOR:
# For JIRA_CONNECTOR, require JIRA_EMAIL, JIRA_API_TOKEN and JIRA_BASE_URL
allowed_keys = ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"]
if set(config.keys()) != set(allowed_keys):
raise ValueError(
f"For JIRA_CONNECTOR connector type, config must only contain these keys: {allowed_keys}"
)
# Ensure the email is not empty
if not config.get("JIRA_EMAIL"):
raise ValueError("JIRA_EMAIL cannot be empty")
# Ensure the API token is not empty
if not config.get("JIRA_API_TOKEN"):
raise ValueError("JIRA_API_TOKEN cannot be empty")
# Ensure the base URL is not empty
if not config.get("JIRA_BASE_URL"):
raise ValueError("JIRA_BASE_URL cannot be empty")
return config
class SearchSourceConnectorCreate(SearchSourceConnectorBase):
pass
class SearchSourceConnectorUpdate(BaseModel):
name: Optional[str] = None
connector_type: Optional[SearchSourceConnectorType] = None
@ -114,7 +156,8 @@ class SearchSourceConnectorUpdate(BaseModel):
last_indexed_at: Optional[datetime] = None
config: Optional[Dict[str, Any]] = None
class SearchSourceConnectorRead(SearchSourceConnectorBase, IDModel, TimestampModel):
user_id: uuid.UUID
model_config = ConfigDict(from_attributes=True)
model_config = ConfigDict(from_attributes=True)

718
surfsense_backend/app/services/connector_service.py
View file

File diff suppressed because it is too large Load diff

1552
surfsense_backend/app/tasks/connectors_indexing_tasks.py
View file

File diff suppressed because it is too large Load diff

404
surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx
View file

@ -9,12 +9,12 @@ import { ArrowLeft, Check, Loader2, Github } from "lucide-react";
import { Form } from "@/components/ui/form";
import { Button } from "@/components/ui/button";
import {
Card,
CardContent,
CardDescription,
CardFooter,
CardHeader,
CardTitle,
Card,
CardContent,
CardDescription,
CardFooter,
CardHeader,
CardTitle,
} from "@/components/ui/card";
// Import Utils, Types, Hook, and Components
@ -27,201 +27,227 @@ import { EditSimpleTokenForm } from "@/components/editConnector/EditSimpleTokenF
import { getConnectorIcon } from "@/components/chat";
export default function EditConnectorPage() {
const router = useRouter();
const params = useParams();
const searchSpaceId = params.search_space_id as string;
// Ensure connectorId is parsed safely
const connectorIdParam = params.connector_id as string;
const connectorId = connectorIdParam ? parseInt(connectorIdParam, 10) : NaN;
const router = useRouter();
const params = useParams();
const searchSpaceId = params.search_space_id as string;
// Ensure connectorId is parsed safely
const connectorIdParam = params.connector_id as string;
const connectorId = connectorIdParam ? parseInt(connectorIdParam, 10) : NaN;
// Use the custom hook to manage state and logic
const {
connectorsLoading,
connector,
isSaving,
editForm,
patForm, // Needed for GitHub child component
handleSaveChanges,
// GitHub specific props for the child component
editMode,
setEditMode, // Pass down if needed by GitHub component
originalPat,
currentSelectedRepos,
fetchedRepos,
setFetchedRepos,
newSelectedRepos,
setNewSelectedRepos,
isFetchingRepos,
handleFetchRepositories,
handleRepoSelectionChange,
} = useConnectorEditPage(connectorId, searchSpaceId);
// Use the custom hook to manage state and logic
const {
connectorsLoading,
connector,
isSaving,
editForm,
patForm, // Needed for GitHub child component
handleSaveChanges,
// GitHub specific props for the child component
editMode,
setEditMode, // Pass down if needed by GitHub component
originalPat,
currentSelectedRepos,
fetchedRepos,
setFetchedRepos,
newSelectedRepos,
setNewSelectedRepos,
isFetchingRepos,
handleFetchRepositories,
handleRepoSelectionChange,
} = useConnectorEditPage(connectorId, searchSpaceId);
// Redirect if connectorId is not a valid number after parsing
useEffect(() => {
if (isNaN(connectorId)) {
toast.error("Invalid Connector ID.");
router.push(`/dashboard/${searchSpaceId}/connectors`);
}
}, [connectorId, router, searchSpaceId]);
// Redirect if connectorId is not a valid number after parsing
useEffect(() => {
if (isNaN(connectorId)) {
toast.error("Invalid Connector ID.");
router.push(`/dashboard/${searchSpaceId}/connectors`);
}
}, [connectorId, router, searchSpaceId]);
// Loading State
if (connectorsLoading || !connector) {
// Handle NaN case before showing skeleton
if (isNaN(connectorId)) return null;
return <EditConnectorLoadingSkeleton />;
}
// Loading State
if (connectorsLoading || !connector) {
// Handle NaN case before showing skeleton
if (isNaN(connectorId)) return null;
return <EditConnectorLoadingSkeleton />;
}
// Main Render using data/handlers from the hook
return (
<div className="container mx-auto py-8 max-w-3xl">
<Button
variant="ghost"
className="mb-6"
onClick={() => router.push(`/dashboard/${searchSpaceId}/connectors`)}
>
<ArrowLeft className="mr-2 h-4 w-4" /> Back to Connectors
</Button>
// Main Render using data/handlers from the hook
return (
<div className="container mx-auto py-8 max-w-3xl">
<Button
variant="ghost"
className="mb-6"
onClick={() => router.push(`/dashboard/${searchSpaceId}/connectors`)}
>
<ArrowLeft className="mr-2 h-4 w-4" /> Back to Connectors
</Button>
<motion.div
initial={{ opacity: 0, y: 20 }}
animate={{ opacity: 1, y: 0 }}
transition={{ duration: 0.5 }}
>
<Card className="border-2 border-border">
<CardHeader>
<CardTitle className="text-2xl font-bold flex items-center gap-2">
{getConnectorIcon(connector.connector_type)}
Edit {getConnectorTypeDisplay(connector.connector_type)} Connector
</CardTitle>
<CardDescription>
Modify connector name and configuration.
</CardDescription>
</CardHeader>
<motion.div
initial={{ opacity: 0, y: 20 }}
animate={{ opacity: 1, y: 0 }}
transition={{ duration: 0.5 }}
>
<Card className="border-2 border-border">
<CardHeader>
<CardTitle className="text-2xl font-bold flex items-center gap-2">
{getConnectorIcon(connector.connector_type)}
Edit {getConnectorTypeDisplay(connector.connector_type)} Connector
</CardTitle>
<CardDescription>
Modify connector name and configuration.
</CardDescription>
</CardHeader>
<Form {...editForm}>
{/* Pass hook's handleSaveChanges */}
<form
onSubmit={editForm.handleSubmit(handleSaveChanges)}
className="space-y-6"
>
<CardContent className="space-y-6">
{/* Pass form control from hook */}
<EditConnectorNameForm control={editForm.control} />
<Form {...editForm}>
{/* Pass hook's handleSaveChanges */}
<form
onSubmit={editForm.handleSubmit(handleSaveChanges)}
className="space-y-6"
>
<CardContent className="space-y-6">
{/* Pass form control from hook */}
<EditConnectorNameForm control={editForm.control} />
<hr />
<hr />
<h3 className="text-lg font-semibold">Configuration</h3>
<h3 className="text-lg font-semibold">Configuration</h3>
{/* == GitHub == */}
{connector.connector_type === "GITHUB_CONNECTOR" && (
<EditGitHubConnectorConfig
// Pass relevant state and handlers from hook
editMode={editMode}
setEditMode={setEditMode} // Pass setter if child manages mode
originalPat={originalPat}
currentSelectedRepos={currentSelectedRepos}
fetchedRepos={fetchedRepos}
newSelectedRepos={newSelectedRepos}
isFetchingRepos={isFetchingRepos}
patForm={patForm}
handleFetchRepositories={handleFetchRepositories}
handleRepoSelectionChange={handleRepoSelectionChange}
setNewSelectedRepos={setNewSelectedRepos}
setFetchedRepos={setFetchedRepos}
/>
)}
{/* == GitHub == */}
{connector.connector_type === "GITHUB_CONNECTOR" && (
<EditGitHubConnectorConfig
// Pass relevant state and handlers from hook
editMode={editMode}
setEditMode={setEditMode} // Pass setter if child manages mode
originalPat={originalPat}
currentSelectedRepos={currentSelectedRepos}
fetchedRepos={fetchedRepos}
newSelectedRepos={newSelectedRepos}
isFetchingRepos={isFetchingRepos}
patForm={patForm}
handleFetchRepositories={handleFetchRepositories}
handleRepoSelectionChange={handleRepoSelectionChange}
setNewSelectedRepos={setNewSelectedRepos}
setFetchedRepos={setFetchedRepos}
/>
)}
{/* == Slack == */}
{connector.connector_type === "SLACK_CONNECTOR" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="SLACK_BOT_TOKEN"
fieldLabel="Slack Bot Token"
fieldDescription="Update the Slack Bot Token if needed."
placeholder="Begins with xoxb-..."
/>
)}
{/* == Notion == */}
{connector.connector_type === "NOTION_CONNECTOR" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="NOTION_INTEGRATION_TOKEN"
fieldLabel="Notion Integration Token"
fieldDescription="Update the Notion Integration Token if needed."
placeholder="Begins with secret_..."
/>
)}
{/* == Serper == */}
{connector.connector_type === "SERPER_API" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="SERPER_API_KEY"
fieldLabel="Serper API Key"
fieldDescription="Update the Serper API Key if needed."
/>
)}
{/* == Tavily == */}
{connector.connector_type === "TAVILY_API" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="TAVILY_API_KEY"
fieldLabel="Tavily API Key"
fieldDescription="Update the Tavily API Key if needed."
/>
)}
{/* == Slack == */}
{connector.connector_type === "SLACK_CONNECTOR" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="SLACK_BOT_TOKEN"
fieldLabel="Slack Bot Token"
fieldDescription="Update the Slack Bot Token if needed."
placeholder="Begins with xoxb-..."
/>
)}
{/* == Notion == */}
{connector.connector_type === "NOTION_CONNECTOR" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="NOTION_INTEGRATION_TOKEN"
fieldLabel="Notion Integration Token"
fieldDescription="Update the Notion Integration Token if needed."
placeholder="Begins with secret_..."
/>
)}
{/* == Serper == */}
{connector.connector_type === "SERPER_API" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="SERPER_API_KEY"
fieldLabel="Serper API Key"
fieldDescription="Update the Serper API Key if needed."
/>
)}
{/* == Tavily == */}
{connector.connector_type === "TAVILY_API" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="TAVILY_API_KEY"
fieldLabel="Tavily API Key"
fieldDescription="Update the Tavily API Key if needed."
/>
)}
{/* == Linear == */}
{connector.connector_type === "LINEAR_CONNECTOR" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="LINEAR_API_KEY"
fieldLabel="Linear API Key"
fieldDescription="Update your Linear API Key if needed."
placeholder="Begins with lin_api_..."
/>
)}
{/* == Linear == */}
{connector.connector_type === "LINEAR_CONNECTOR" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="LINEAR_API_KEY"
fieldLabel="Linear API Key"
fieldDescription="Update your Linear API Key if needed."
placeholder="Begins with lin_api_..."
/>
)}
{/* == Linkup == */}
{connector.connector_type === "LINKUP_API" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="LINKUP_API_KEY"
fieldLabel="Linkup API Key"
fieldDescription="Update your Linkup API Key if needed."
placeholder="Begins with linkup_..."
/>
)}
{/* == Jira == */}
{connector.connector_type === "JIRA_CONNECTOR" && (
<div className="space-y-4">
<EditSimpleTokenForm
control={editForm.control}
fieldName="JIRA_BASE_URL"
fieldLabel="Jira Base URL"
fieldDescription="Update your Jira instance URL if needed."
placeholder="https://yourcompany.atlassian.net"
/>
<EditSimpleTokenForm
control={editForm.control}
fieldName="JIRA_EMAIL"
fieldLabel="Jira Email"
fieldDescription="Update your Atlassian account email if needed."
placeholder="your.email@company.com"
/>
<EditSimpleTokenForm
control={editForm.control}
fieldName="JIRA_API_TOKEN"
fieldLabel="Jira API Token"
fieldDescription="Update your Jira API Token if needed."
placeholder="Your Jira API Token"
/>
</div>
)}
{/* == Discord == */}
{connector.connector_type === "DISCORD_CONNECTOR" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="DISCORD_BOT_TOKEN"
fieldLabel="Discord Bot Token"
fieldDescription="Update the Discord Bot Token if needed."
placeholder="Bot token..."
/>
)}
{/* == Linkup == */}
{connector.connector_type === "LINKUP_API" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="LINKUP_API_KEY"
fieldLabel="Linkup API Key"
fieldDescription="Update your Linkup API Key if needed."
placeholder="Begins with linkup_..."
/>
)}
</CardContent>
<CardFooter className="border-t pt-6">
<Button
type="submit"
disabled={isSaving}
className="w-full sm:w-auto"
>
{isSaving ? (
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
) : (
<Check className="mr-2 h-4 w-4" />
)}
Save Changes
</Button>
</CardFooter>
</form>
</Form>
</Card>
</motion.div>
</div>
);
{/* == Discord == */}
{connector.connector_type === "DISCORD_CONNECTOR" && (
<EditSimpleTokenForm
control={editForm.control}
fieldName="DISCORD_BOT_TOKEN"
fieldLabel="Discord Bot Token"
fieldDescription="Update the Discord Bot Token if needed."
placeholder="Bot token..."
/>
)}
</CardContent>
<CardFooter className="border-t pt-6">
<Button
type="submit"
disabled={isSaving}
className="w-full sm:w-auto"
>
{isSaving ? (
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
) : (
<Check className="mr-2 h-4 w-4" />
)}
Save Changes
</Button>
</CardFooter>
</form>
</Form>
</Card>
</motion.div>
</div>
);
}

122
surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx
View file

@ -9,7 +9,10 @@ import * as z from "zod";
import { toast } from "sonner";
import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
import { useSearchSourceConnectors, SearchSourceConnector } from "@/hooks/useSearchSourceConnectors";
import {
useSearchSourceConnectors,
SearchSourceConnector,
} from "@/hooks/useSearchSourceConnectors";
import {
Form,
FormControl,
@ -28,11 +31,7 @@ import {
CardHeader,
CardTitle,
} from "@/components/ui/card";
import {
Alert,
AlertDescription,
AlertTitle,
} from "@/components/ui/alert";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
// Define the form schema with Zod
const apiConnectorFormSchema = z.object({
@ -47,13 +46,15 @@ const apiConnectorFormSchema = z.object({
// Helper function to get connector type display name
const getConnectorTypeDisplay = (type: string): string => {
const typeMap: Record<string, string> = {
"SERPER_API": "Serper API",
"TAVILY_API": "Tavily API",
"SLACK_CONNECTOR": "Slack Connector",
"NOTION_CONNECTOR": "Notion Connector",
"GITHUB_CONNECTOR": "GitHub Connector",
"DISCORD_CONNECTOR": "Discord Connector",
"LINKUP_API": "Linkup",
SERPER_API: "Serper API",
TAVILY_API: "Tavily API",
SLACK_CONNECTOR: "Slack Connector",
NOTION_CONNECTOR: "Notion Connector",
GITHUB_CONNECTOR: "GitHub Connector",
LINEAR_CONNECTOR: "Linear Connector",
JIRA_CONNECTOR: "Jira Connector",
DISCORD_CONNECTOR: "Discord Connector",
LINKUP_API: "Linkup",
// Add other connector types here as needed
};
return typeMap[type] || type;
@ -67,9 +68,11 @@ export default function EditConnectorPage() {
const params = useParams();
const searchSpaceId = params.search_space_id as string;
const connectorId = parseInt(params.connector_id as string, 10);
const { connectors, updateConnector } = useSearchSourceConnectors();
const [connector, setConnector] = useState<SearchSourceConnector | null>(null);
const [connector, setConnector] = useState<SearchSourceConnector | null>(
null,
);
const [isLoading, setIsLoading] = useState(true);
const [isSubmitting, setIsSubmitting] = useState(false);
// console.log("connector", connector);
@ -85,24 +88,24 @@ export default function EditConnectorPage() {
// Get API key field name based on connector type
const getApiKeyFieldName = (connectorType: string): string => {
const fieldMap: Record<string, string> = {
"SERPER_API": "SERPER_API_KEY",
"TAVILY_API": "TAVILY_API_KEY",
"SLACK_CONNECTOR": "SLACK_BOT_TOKEN",
"NOTION_CONNECTOR": "NOTION_INTEGRATION_TOKEN",
"GITHUB_CONNECTOR": "GITHUB_PAT",
"DISCORD_CONNECTOR": "DISCORD_BOT_TOKEN",
"LINKUP_API": "LINKUP_API_KEY"
SERPER_API: "SERPER_API_KEY",
TAVILY_API: "TAVILY_API_KEY",
SLACK_CONNECTOR: "SLACK_BOT_TOKEN",
NOTION_CONNECTOR: "NOTION_INTEGRATION_TOKEN",
GITHUB_CONNECTOR: "GITHUB_PAT",
DISCORD_CONNECTOR: "DISCORD_BOT_TOKEN",
LINKUP_API: "LINKUP_API_KEY",
};
return fieldMap[connectorType] || "";
};
// Find connector in the list
useEffect(() => {
const currentConnector = connectors.find(c => c.id === connectorId);
const currentConnector = connectors.find((c) => c.id === connectorId);
if (currentConnector) {
setConnector(currentConnector);
// Check if connector type is supported
const apiKeyField = getApiKeyFieldName(currentConnector.connector_type);
if (apiKeyField) {
@ -115,7 +118,7 @@ export default function EditConnectorPage() {
toast.error("This connector type is not supported for editing");
router.push(`/dashboard/${searchSpaceId}/connectors`);
}
setIsLoading(false);
} else if (!isLoading && connectors.length > 0) {
// If connectors are loaded but this one isn't found
@ -127,11 +130,11 @@ export default function EditConnectorPage() {
// Handle form submission
const onSubmit = async (values: ApiConnectorFormValues) => {
if (!connector) return;
setIsSubmitting(true);
try {
const apiKeyField = getApiKeyFieldName(connector.connector_type);
// Only update the API key if a new one was provided
const updatedConfig = { ...connector.config };
if (values.api_key) {
@ -150,7 +153,9 @@ export default function EditConnectorPage() {
router.push(`/dashboard/${searchSpaceId}/connectors`);
} catch (error) {
console.error("Error updating connector:", error);
toast.error(error instanceof Error ? error.message : "Failed to update connector");
toast.error(
error instanceof Error ? error.message : "Failed to update connector",
);
} finally {
setIsSubmitting(false);
}
@ -186,24 +191,30 @@ export default function EditConnectorPage() {
<Card className="border-2 border-border">
<CardHeader>
<CardTitle className="text-2xl font-bold">
Edit {connector ? getConnectorTypeDisplay(connector.connector_type) : ""} Connector
Edit{" "}
{connector
? getConnectorTypeDisplay(connector.connector_type)
: ""}{" "}
Connector
</CardTitle>
<CardDescription>
Update your connector settings.
</CardDescription>
<CardDescription>Update your connector settings.</CardDescription>
</CardHeader>
<CardContent>
<Alert className="mb-6 bg-muted">
<Info className="h-4 w-4" />
<AlertTitle>API Key Security</AlertTitle>
<AlertDescription>
Your API key is stored securely. For security reasons, we don't display your existing API key.
If you don't update the API key field, your existing key will be preserved.
Your API key is stored securely. For security reasons, we don't
display your existing API key. If you don't update the API key
field, your existing key will be preserved.
</AlertDescription>
</Alert>
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-6">
<form
onSubmit={form.handleSubmit(onSubmit)}
className="space-y-6"
>
<FormField
control={form.control}
name="name"
@ -227,10 +238,10 @@ export default function EditConnectorPage() {
render={({ field }) => (
<FormItem>
<FormLabel>
{connector?.connector_type === "SLACK_CONNECTOR"
? "Slack Bot Token"
: connector?.connector_type === "NOTION_CONNECTOR"
? "Notion Integration Token"
{connector?.connector_type === "SLACK_CONNECTOR"
? "Slack Bot Token"
: connector?.connector_type === "NOTION_CONNECTOR"
? "Notion Integration Token"
: connector?.connector_type === "GITHUB_CONNECTOR"
? "GitHub Personal Access Token (PAT)"
: connector?.connector_type === "LINKUP_API"
@ -238,27 +249,28 @@ export default function EditConnectorPage() {
: "API Key"}
</FormLabel>
<FormControl>
<Input
type="password"
<Input
type="password"
placeholder={
connector?.connector_type === "SLACK_CONNECTOR"
? "Enter new Slack Bot Token (optional)"
: connector?.connector_type === "NOTION_CONNECTOR"
connector?.connector_type === "SLACK_CONNECTOR"
? "Enter new Slack Bot Token (optional)"
: connector?.connector_type === "NOTION_CONNECTOR"
? "Enter new Notion Token (optional)"
: connector?.connector_type === "GITHUB_CONNECTOR"
: connector?.connector_type ===
"GITHUB_CONNECTOR"
? "Enter new GitHub PAT (optional)"
: connector?.connector_type === "LINKUP_API"
? "Enter new Linkup API Key (optional)"
: "Enter new API key (optional)"
}
{...field}
}
{...field}
/>
</FormControl>
<FormDescription>
{connector?.connector_type === "SLACK_CONNECTOR"
? "Enter a new Slack Bot Token or leave blank to keep your existing token."
: connector?.connector_type === "NOTION_CONNECTOR"
? "Enter a new Notion Integration Token or leave blank to keep your existing token."
{connector?.connector_type === "SLACK_CONNECTOR"
? "Enter a new Slack Bot Token or leave blank to keep your existing token."
: connector?.connector_type === "NOTION_CONNECTOR"
? "Enter a new Notion Integration Token or leave blank to keep your existing token."
: connector?.connector_type === "GITHUB_CONNECTOR"
? "Enter a new GitHub PAT or leave blank to keep your existing token."
: connector?.connector_type === "LINKUP_API"
@ -271,8 +283,8 @@ export default function EditConnectorPage() {
/>
<div className="flex justify-end">
<Button
type="submit"
<Button
type="submit"
disabled={isSubmitting}
className="w-full sm:w-auto"
>
@ -296,4 +308,4 @@ export default function EditConnectorPage() {
</motion.div>
</div>
);
}
}

472
surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx Normal file
View file

@ -0,0 +1,472 @@
"use client";
import { useState } from "react";
import { useRouter, useParams } from "next/navigation";
import { motion } from "framer-motion";
import { zodResolver } from "@hookform/resolvers/zod";
import { useForm } from "react-hook-form";
import * as z from "zod";
import { toast } from "sonner";
import { ArrowLeft, Check, Info, Loader2 } from "lucide-react";
import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors";
import {
Form,
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import { Button } from "@/components/ui/button";
import {
Card,
CardContent,
CardDescription,
CardFooter,
CardHeader,
CardTitle,
} from "@/components/ui/card";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import {
Accordion,
AccordionContent,
AccordionItem,
AccordionTrigger,
} from "@/components/ui/accordion";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
// Define the form schema with Zod
const jiraConnectorFormSchema = z.object({
name: z.string().min(3, {
message: "Connector name must be at least 3 characters.",
}),
base_url: z
.string()
.url({
message:
"Please enter a valid Jira URL (e.g., https://yourcompany.atlassian.net)",
})
.refine(
(url) => {
return url.includes("atlassian.net") || url.includes("jira");
},
{
message: "Please enter a valid Jira instance URL",
},
),
email: z.string().email({
message: "Please enter a valid email address.",
}),
api_token: z.string().min(10, {
message: "Jira API Token is required and must be valid.",
}),
});
// Define the type for the form values
type JiraConnectorFormValues = z.infer<typeof jiraConnectorFormSchema>;
export default function JiraConnectorPage() {
const router = useRouter();
const params = useParams();
const searchSpaceId = params.search_space_id as string;
const [isSubmitting, setIsSubmitting] = useState(false);
const { createConnector } = useSearchSourceConnectors();
// Initialize the form
const form = useForm<JiraConnectorFormValues>({
resolver: zodResolver(jiraConnectorFormSchema),
defaultValues: {
name: "Jira Connector",
base_url: "",
email: "",
api_token: "",
},
});
// Handle form submission
const onSubmit = async (values: JiraConnectorFormValues) => {
setIsSubmitting(true);
try {
await createConnector({
name: values.name,
connector_type: "JIRA_CONNECTOR",
config: {
JIRA_BASE_URL: values.base_url,
JIRA_EMAIL: values.email,
JIRA_API_TOKEN: values.api_token,
},
is_indexable: true,
last_indexed_at: null,
});
toast.success("Jira connector created successfully!");
// Navigate back to connectors page
router.push(`/dashboard/${searchSpaceId}/connectors`);
} catch (error) {
console.error("Error creating connector:", error);
toast.error(
error instanceof Error ? error.message : "Failed to create connector",
);
} finally {
setIsSubmitting(false);
}
};
return (
<div className="container mx-auto py-8 max-w-3xl">
<Button
variant="ghost"
className="mb-6"
onClick={() =>
router.push(`/dashboard/${searchSpaceId}/connectors/add`)
}
>
<ArrowLeft className="mr-2 h-4 w-4" />
Back to Connectors
</Button>
<motion.div
initial={{ opacity: 0, y: 20 }}
animate={{ opacity: 1, y: 0 }}
transition={{ duration: 0.5 }}
>
<Tabs defaultValue="connect" className="w-full">
<TabsList className="grid w-full grid-cols-2 mb-6">
<TabsTrigger value="connect">Connect</TabsTrigger>
<TabsTrigger value="documentation">Documentation</TabsTrigger>
</TabsList>
<TabsContent value="connect">
<Card className="border-2 border-border">
<CardHeader>
<CardTitle className="text-2xl font-bold">
Connect Jira Instance
</CardTitle>
<CardDescription>
Integrate with Jira to search and retrieve information from
your issues, tickets, and comments. This connector can index
your Jira content for search.
</CardDescription>
</CardHeader>
<CardContent>
<Alert className="mb-6 bg-muted">
<Info className="h-4 w-4" />
<AlertTitle>Jira Personal Access Token Required</AlertTitle>
<AlertDescription>
You'll need a Jira Personal Access Token to use this
connector. You can create one from{" "}
<a
href="https://id.atlassian.com/manage-profile/security/api-tokens"
target="_blank"
rel="noopener noreferrer"
className="font-medium underline underline-offset-4"
>
Atlassian Account Settings
</a>
</AlertDescription>
</Alert>
<Form {...form}>
<form
onSubmit={form.handleSubmit(onSubmit)}
className="space-y-6"
>
<FormField
control={form.control}
name="name"
render={({ field }) => (
<FormItem>
<FormLabel>Connector Name</FormLabel>
<FormControl>
<Input placeholder="My Jira Connector" {...field} />
</FormControl>
<FormDescription>
A friendly name to identify this connector.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="base_url"
render={({ field }) => (
<FormItem>
<FormLabel>Jira Instance URL</FormLabel>
<FormControl>
<Input
placeholder="https://yourcompany.atlassian.net"
{...field}
/>
</FormControl>
<FormDescription>
Your Jira instance URL. For Atlassian Cloud, this is
typically https://yourcompany.atlassian.net
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="email"
render={({ field }) => (
<FormItem>
<FormLabel>Email Address</FormLabel>
<FormControl>
<Input
type="email"
placeholder="your.email@company.com"
{...field}
/>
</FormControl>
<FormDescription>
Your Atlassian account email address.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="api_token"
render={({ field }) => (
<FormItem>
<FormLabel>API Token</FormLabel>
<FormControl>
<Input
type="password"
placeholder="Your Jira API Token"
{...field}
/>
</FormControl>
<FormDescription>
Your Jira API Token will be encrypted and stored securely.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<div className="flex justify-end">
<Button
type="submit"
disabled={isSubmitting}
className="w-full sm:w-auto"
>
{isSubmitting ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
Connecting...
</>
) : (
<>
<Check className="mr-2 h-4 w-4" />
Connect Jira
</>
)}
</Button>
</div>
</form>
</Form>
</CardContent>
<CardFooter className="flex flex-col items-start border-t bg-muted/50 px-6 py-4">
<h4 className="text-sm font-medium">
What you get with Jira integration:
</h4>
<ul className="mt-2 list-disc pl-5 text-sm text-muted-foreground">
<li>Search through all your Jira issues and tickets</li>
<li>
Access issue descriptions, comments, and full discussion
threads
</li>
<li>
Connect your team's project management directly to your
search space
</li>
<li>
Keep your search results up-to-date with latest Jira content
</li>
<li>
Index your Jira issues for enhanced search capabilities
</li>
<li>
Search by issue keys, status, priority, and assignee
information
</li>
</ul>
</CardFooter>
</Card>
</TabsContent>
<TabsContent value="documentation">
<Card className="border-2 border-border">
<CardHeader>
<CardTitle className="text-2xl font-bold">
Jira Connector Documentation
</CardTitle>
<CardDescription>
Learn how to set up and use the Jira connector to index your
project management data.
</CardDescription>
</CardHeader>
<CardContent className="space-y-6">
<div>
<h3 className="text-xl font-semibold mb-2">How it works</h3>
<p className="text-muted-foreground">
The Jira connector uses the Jira REST API with Basic Authentication
to fetch all issues and comments that your account has
access to within your Jira instance.
</p>
<ul className="mt-2 list-disc pl-5 text-muted-foreground">
<li>
For follow up indexing runs, the connector retrieves
issues and comments that have been updated since the last
indexing attempt.
</li>
<li>
Indexing is configured to run periodically, so updates
should appear in your search results within minutes.
</li>
</ul>
</div>
<Accordion type="single" collapsible className="w-full">
<AccordionItem value="authorization">
<AccordionTrigger className="text-lg font-medium">
Authorization
</AccordionTrigger>
<AccordionContent className="space-y-4">
<Alert className="bg-muted">
<Info className="h-4 w-4" />
<AlertTitle>Read-Only Access is Sufficient</AlertTitle>
<AlertDescription>
You only need read access for this connector to work.
The API Token will only be used to read your Jira data.
</AlertDescription>
</Alert>
<div className="space-y-6">
<div>
<h4 className="font-medium mb-2">
Step 1: Create an API Token
</h4>
<ol className="list-decimal pl-5 space-y-3">
<li>Log in to your Atlassian account</li>
<li>
Navigate to{" "}
<a
href="https://id.atlassian.com/manage-profile/security/api-tokens"
target="_blank"
rel="noopener noreferrer"
className="font-medium underline underline-offset-4"
>
https://id.atlassian.com/manage-profile/security/api-tokens
</a>
</li>
<li>
Click <strong>Create API token</strong>
</li>
<li>
Enter a label for your token (like "SurfSense
Connector")
</li>
<li>
Click <strong>Create</strong>
</li>
<li>
Copy the generated token as it will only be shown
once
</li>
</ol>
</div>
<div>
<h4 className="font-medium mb-2">
Step 2: Grant necessary access
</h4>
<p className="text-muted-foreground mb-3">
The API Token will have access to all projects and
issues that your user account can see. Make sure your
account has appropriate permissions for the projects
you want to index.
</p>
<Alert className="bg-muted">
<Info className="h-4 w-4" />
<AlertTitle>Data Privacy</AlertTitle>
<AlertDescription>
Only issues, comments, and basic metadata will be
indexed. Jira attachments and linked files are not
indexed by this connector.
</AlertDescription>
</Alert>
</div>
</div>
</AccordionContent>
</AccordionItem>
<AccordionItem value="indexing">
<AccordionTrigger className="text-lg font-medium">
Indexing
</AccordionTrigger>
<AccordionContent className="space-y-4">
<ol className="list-decimal pl-5 space-y-3">
<li>
Navigate to the Connector Dashboard and select the{" "}
<strong>Jira</strong> Connector.
</li>
<li>
Enter your <strong>Jira Instance URL</strong> (e.g.,
https://yourcompany.atlassian.net)
</li>
<li>
Place your <strong>Personal Access Token</strong> in
the form field.
</li>
<li>
Click <strong>Connect</strong> to establish the
connection.
</li>
<li>
Once connected, your Jira issues will be indexed
automatically.
</li>
</ol>
<Alert className="bg-muted">
<Info className="h-4 w-4" />
<AlertTitle>What Gets Indexed</AlertTitle>
<AlertDescription>
<p className="mb-2">
The Jira connector indexes the following data:
</p>
<ul className="list-disc pl-5">
<li>Issue keys and summaries (e.g., PROJ-123)</li>
<li>Issue descriptions</li>
<li>Issue comments and discussion threads</li>
<li>
Issue status, priority, and type information
</li>
<li>Assignee and reporter information</li>
<li>Project information</li>
</ul>
</AlertDescription>
</Alert>
</AccordionContent>
</AccordionItem>
</Accordion>
</CardContent>
</Card>
</TabsContent>
</Tabs>
</motion.div>
</div>
);
}

165
surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx
View file

@ -1,8 +1,17 @@
"use client";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Card, CardContent, CardFooter, CardHeader } from "@/components/ui/card";
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
import {
Card,
CardContent,
CardFooter,
CardHeader,
} from "@/components/ui/card";
import {
Collapsible,
CollapsibleContent,
CollapsibleTrigger,
} from "@/components/ui/collapsible";
import {
IconBrandDiscord,
IconBrandGithub,
@ -67,23 +76,26 @@ const connectorCategories: ConnectorCategory[] = [
{
id: "slack-connector",
title: "Slack",
description: "Connect to your Slack workspace to access messages and channels.",
description:
"Connect to your Slack workspace to access messages and channels.",
icon: <IconBrandSlack className="h-6 w-6" />,
status: "available",
},
{
id: "ms-teams",
title: "Microsoft Teams",
description: "Connect to Microsoft Teams to access your team's conversations.",
description:
"Connect to Microsoft Teams to access your team's conversations.",
icon: <IconBrandWindows className="h-6 w-6" />,
status: "coming-soon",
},
{
id: "discord-connector",
title: "Discord",
description: "Connect to Discord servers to access messages and channels.",
description:
"Connect to Discord servers to access messages and channels.",
icon: <IconBrandDiscord className="h-6 w-6" />,
status: "available"
status: "available",
},
],
},
@ -94,16 +106,18 @@ const connectorCategories: ConnectorCategory[] = [
{
id: "linear-connector",
title: "Linear",
description: "Connect to Linear to search issues, comments and project data.",
description:
"Connect to Linear to search issues, comments and project data.",
icon: <IconLayoutKanban className="h-6 w-6" />,
status: "available",
},
{
id: "jira-connector",
title: "Jira",
description: "Connect to Jira to search issues, tickets and project data.",
description:
"Connect to Jira to search issues, tickets and project data.",
icon: <IconTicket className="h-6 w-6" />,
status: "coming-soon",
status: "available",
},
],
},
@ -114,14 +128,16 @@ const connectorCategories: ConnectorCategory[] = [
{
id: "notion-connector",
title: "Notion",
description: "Connect to your Notion workspace to access pages and databases.",
description:
"Connect to your Notion workspace to access pages and databases.",
icon: <IconBrandNotion className="h-6 w-6" />,
status: "available",
},
{
id: "github-connector",
title: "GitHub",
description: "Connect a GitHub PAT to index code and docs from accessible repositories.",
description:
"Connect a GitHub PAT to index code and docs from accessible repositories.",
icon: <IconBrandGithub className="h-6 w-6" />,
status: "available",
},
@ -141,7 +157,8 @@ const connectorCategories: ConnectorCategory[] = [
{
id: "zoom",
title: "Zoom",
description: "Connect to Zoom to access meeting recordings and transcripts.",
description:
"Connect to Zoom to access meeting recordings and transcripts.",
icon: <IconBrandZoom className="h-6 w-6" />,
status: "coming-soon",
},
@ -152,7 +169,7 @@ const connectorCategories: ConnectorCategory[] = [
// Animation variants
const fadeIn = {
hidden: { opacity: 0 },
visible: { opacity: 1, transition: { duration: 0.4 } }
visible: { opacity: 1, transition: { duration: 0.4 } },
};
const staggerContainer = {
@ -160,43 +177,49 @@ const staggerContainer = {
visible: {
opacity: 1,
transition: {
staggerChildren: 0.1
}
}
staggerChildren: 0.1,
},
},
};
const cardVariants = {
hidden: { opacity: 0, y: 20 },
visible: {
opacity: 1,
visible: {
opacity: 1,
y: 0,
transition: {
transition: {
type: "spring",
stiffness: 260,
damping: 20
}
damping: 20,
},
},
hover: {
hover: {
scale: 1.02,
boxShadow: "0 10px 15px -3px rgba(0, 0, 0, 0.1), 0 4px 6px -2px rgba(0, 0, 0, 0.05)",
transition: {
boxShadow:
"0 10px 15px -3px rgba(0, 0, 0, 0.1), 0 4px 6px -2px rgba(0, 0, 0, 0.05)",
transition: {
type: "spring",
stiffness: 400,
damping: 10
}
}
damping: 10,
},
},
};
export default function ConnectorsPage() {
const params = useParams();
const searchSpaceId = params.search_space_id as string;
const [expandedCategories, setExpandedCategories] = useState<string[]>(["search-engines", "knowledge-bases", "project-management", "team-chats"]);
const [expandedCategories, setExpandedCategories] = useState<string[]>([
"search-engines",
"knowledge-bases",
"project-management",
"team-chats",
]);
const toggleCategory = (categoryId: string) => {
setExpandedCategories(prev =>
prev.includes(categoryId)
? prev.filter(id => id !== categoryId)
: [...prev, categoryId]
setExpandedCategories((prev) =>
prev.includes(categoryId)
? prev.filter((id) => id !== categoryId)
: [...prev, categoryId],
);
};
@ -205,9 +228,9 @@ export default function ConnectorsPage() {
<motion.div
initial={{ opacity: 0, y: 30 }}
animate={{ opacity: 1, y: 0 }}
transition={{
transition={{
duration: 0.6,
ease: [0.22, 1, 0.36, 1]
ease: [0.22, 1, 0.36, 1],
}}
className="mb-12 text-center"
>
@ -215,18 +238,19 @@ export default function ConnectorsPage() {
Connect Your Tools
</h1>
<p className="text-muted-foreground mt-3 text-lg max-w-2xl mx-auto">
Integrate with your favorite services to enhance your research capabilities.
Integrate with your favorite services to enhance your research
capabilities.
</p>
</motion.div>
<motion.div
<motion.div
className="space-y-8"
initial="hidden"
animate="visible"
variants={staggerContainer}
>
{connectorCategories.map((category) => (
<motion.div
<motion.div
key={category.id}
variants={fadeIn}
className="rounded-lg border bg-card text-card-foreground shadow-sm"
@ -239,9 +263,17 @@ export default function ConnectorsPage() {
<div className="flex items-center justify-between space-x-4 p-4">
<h3 className="text-xl font-semibold">{category.title}</h3>
<CollapsibleTrigger asChild>
<Button variant="ghost" size="sm" className="w-9 p-0 hover:bg-muted">
<Button
variant="ghost"
size="sm"
className="w-9 p-0 hover:bg-muted"
>
<motion.div
animate={{ rotate: expandedCategories.includes(category.id) ? 180 : 0 }}
animate={{
rotate: expandedCategories.includes(category.id)
? 180
: 0,
}}
transition={{ duration: 0.3, ease: "easeInOut" }}
>
<IconChevronDown className="h-5 w-5" />
@ -250,10 +282,10 @@ export default function ConnectorsPage() {
</Button>
</CollapsibleTrigger>
</div>
<CollapsibleContent>
<AnimatePresence>
<motion.div
<motion.div
className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3 p-4"
variants={staggerContainer}
initial="hidden"
@ -279,50 +311,75 @@ export default function ConnectorsPage() {
</div>
<div>
<div className="flex items-center gap-2">
<h3 className="font-medium">{connector.title}</h3>
<h3 className="font-medium">
{connector.title}
</h3>
{connector.status === "coming-soon" && (
<Badge variant="outline" className="text-xs bg-amber-100 dark:bg-amber-950 text-amber-800 dark:text-amber-300 border-amber-200 dark:border-amber-800">
<Badge
variant="outline"
className="text-xs bg-amber-100 dark:bg-amber-950 text-amber-800 dark:text-amber-300 border-amber-200 dark:border-amber-800"
>
Coming soon
</Badge>
)}
{connector.status === "connected" && (
<Badge variant="outline" className="text-xs bg-green-100 dark:bg-green-950 text-green-800 dark:text-green-300 border-green-200 dark:border-green-800">
<Badge
variant="outline"
className="text-xs bg-green-100 dark:bg-green-950 text-green-800 dark:text-green-300 border-green-200 dark:border-green-800"
>
Connected
</Badge>
)}
</div>
</div>
</CardHeader>
<CardContent className="pb-4">
<p className="text-sm text-muted-foreground">
{connector.description}
</p>
</CardContent>
<CardFooter className="mt-auto pt-2">
{connector.status === 'available' && (
<Link href={`/dashboard/${searchSpaceId}/connectors/add/${connector.id}`} className="w-full">
<Button variant="default" className="w-full group">
{connector.status === "available" && (
<Link
href={`/dashboard/${searchSpaceId}/connectors/add/${connector.id}`}
className="w-full"
>
<Button
variant="default"
className="w-full group"
>
<span>Connect</span>
<motion.div
className="ml-1"
initial={{ x: 0 }}
whileHover={{ x: 3 }}
transition={{ type: "spring", stiffness: 400, damping: 10 }}
transition={{
type: "spring",
stiffness: 400,
damping: 10,
}}
>
<IconChevronRight className="h-4 w-4" />
</motion.div>
</Button>
</Link>
)}
{connector.status === 'coming-soon' && (
<Button variant="outline" disabled className="w-full opacity-70">
{connector.status === "coming-soon" && (
<Button
variant="outline"
disabled
className="w-full opacity-70"
>
Coming Soon
</Button>
)}
{connector.status === 'connected' && (
<Button variant="outline" className="w-full border-green-500 text-green-600 hover:bg-green-50 dark:hover:bg-green-950">
{connector.status === "connected" && (
<Button
variant="outline"
className="w-full border-green-500 text-green-600 hover:bg-green-50 dark:hover:bg-green-950"
>
Manage
</Button>
)}

2026
surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
View file

File diff suppressed because it is too large Load diff

3512
surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx
View file

File diff suppressed because it is too large Load diff

202
surfsense_web/components/chat/ConnectorComponents.tsx
View file

@ -1,6 +1,6 @@
import React from 'react';
import {
ChevronDown,
import React from "react";
import {
ChevronDown,
Plus,
Search,
Globe,
@ -12,78 +12,99 @@ import {
Webhook,
MessageCircle,
FileText,
} from 'lucide-react';
import { IconBrandNotion, IconBrandSlack, IconBrandYoutube, IconBrandGithub, IconLayoutKanban, IconLinkPlus, IconBrandDiscord } from "@tabler/icons-react";
import { Button } from '@/components/ui/button';
import { Connector, ResearchMode } from './types';
} from "lucide-react";
import {
IconBrandNotion,
IconBrandSlack,
IconBrandYoutube,
IconBrandGithub,
IconLayoutKanban,
IconLinkPlus,
IconBrandDiscord,
IconTicket,
} from "@tabler/icons-react";
import { Button } from "@/components/ui/button";
import { Connector, ResearchMode } from "./types";
// Helper function to get connector icon
export const getConnectorIcon = (connectorType: string) => {
const iconProps = { className: "h-4 w-4" };
switch(connectorType) {
case 'LINKUP_API':
switch (connectorType) {
case "LINKUP_API":
return <IconLinkPlus {...iconProps} />;
case 'LINEAR_CONNECTOR':
case "LINEAR_CONNECTOR":
return <IconLayoutKanban {...iconProps} />;
case 'GITHUB_CONNECTOR':
case "GITHUB_CONNECTOR":
return <IconBrandGithub {...iconProps} />;
case 'YOUTUBE_VIDEO':
case "YOUTUBE_VIDEO":
return <IconBrandYoutube {...iconProps} />;
case 'CRAWLED_URL':
case "CRAWLED_URL":
return <Globe {...iconProps} />;
case 'FILE':
return <File {...iconProps} />;
case 'EXTENSION':
return <Webhook {...iconProps} />;
case 'SERPER_API':
case 'TAVILY_API':
case "FILE":
return <File {...iconProps} />;
case "EXTENSION":
return <Webhook {...iconProps} />;
case "SERPER_API":
case "TAVILY_API":
return <Link {...iconProps} />;
case 'SLACK_CONNECTOR':
case "SLACK_CONNECTOR":
return <IconBrandSlack {...iconProps} />;
case 'NOTION_CONNECTOR':
case "NOTION_CONNECTOR":
return <IconBrandNotion {...iconProps} />;
case 'DISCORD_CONNECTOR':
case "DISCORD_CONNECTOR":
return <IconBrandDiscord {...iconProps} />;
case 'DEEP':
case "JIRA_CONNECTOR":
return <IconTicket {...iconProps} />;
case "DEEP":
return <Sparkles {...iconProps} />;
case 'DEEPER':
case "DEEPER":
return <Microscope {...iconProps} />;
case 'DEEPEST':
case "DEEPEST":
return <Telescope {...iconProps} />;
default:
return <Search {...iconProps} />;
}
};
export const researcherOptions: { value: ResearchMode; label: string; icon: React.ReactNode }[] = [
export const researcherOptions: {
value: ResearchMode;
label: string;
icon: React.ReactNode;
}[] = [
{
value: 'QNA',
label: 'Q/A',
icon: getConnectorIcon('GENERAL')
value: "QNA",
label: "Q/A",
icon: getConnectorIcon("GENERAL"),
},
{
value: 'REPORT_GENERAL',
label: 'General',
icon: getConnectorIcon('GENERAL')
value: "REPORT_GENERAL",
label: "General",
icon: getConnectorIcon("GENERAL"),
},
{
value: 'REPORT_DEEP',
label: 'Deep',
icon: getConnectorIcon('DEEP')
value: "REPORT_DEEP",
label: "Deep",
icon: getConnectorIcon("DEEP"),
},
{
value: 'REPORT_DEEPER',
label: 'Deeper',
icon: getConnectorIcon('DEEPER')
value: "REPORT_DEEPER",
label: "Deeper",
icon: getConnectorIcon("DEEPER"),
},
]
];
/**
* Displays a small icon for a connector type
*/
export const ConnectorIcon = ({ type, index = 0 }: { type: string; index?: number }) => (
<div
export const ConnectorIcon = ({
type,
index = 0,
}: {
type: string;
index?: number;
}) => (
<div
className="w-4 h-4 rounded-full flex items-center justify-center bg-muted border border-background"
style={{ zIndex: 10 - index }}
>
@ -109,24 +130,30 @@ type ConnectorButtonProps = {
/**
* Button that displays selected connectors and opens connector selection dialog
*/
export const ConnectorButton = ({ selectedConnectors, onClick, connectorSources }: ConnectorButtonProps) => {
export const ConnectorButton = ({
selectedConnectors,
onClick,
connectorSources,
}: ConnectorButtonProps) => {
const totalConnectors = connectorSources.length;
const selectedCount = selectedConnectors.length;
const progressPercentage = (selectedCount / totalConnectors) * 100;
// Get the name of a single selected connector
const getSingleConnectorName = () => {
const connector = connectorSources.find(c => c.type === selectedConnectors[0]);
return connector?.name || '';
const connector = connectorSources.find(
(c) => c.type === selectedConnectors[0],
);
return connector?.name || "";
};
// Get display text based on selection count
const getDisplayText = () => {
if (selectedCount === totalConnectors) return "All Connectors";
if (selectedCount === 1) return getSingleConnectorName();
return `${selectedCount} Connectors`;
};
// Render the empty state (no connectors selected)
const renderEmptyState = () => (
<>
@ -134,7 +161,7 @@ export const ConnectorButton = ({ selectedConnectors, onClick, connectorSources
<span className="text-muted-foreground">Select Connectors</span>
</>
);
// Render the selected connectors preview
const renderSelectedConnectors = () => (
<>
@ -143,32 +170,36 @@ export const ConnectorButton = ({ selectedConnectors, onClick, connectorSources
{selectedConnectors.slice(0, 3).map((type, index) => (
<ConnectorIcon key={type} type={type} index={index} />
))}
{/* Show count indicator if more than 3 connectors are selected */}
{selectedCount > 3 && <ConnectorCountBadge count={selectedCount - 3} />}
</div>
{/* Display text */}
<span className="font-medium">{getDisplayText()}</span>
</>
);
return (
<Button
variant="outline"
className="h-8 px-2 text-xs font-medium rounded-md border-border relative overflow-hidden group"
onClick={onClick}
aria-label={selectedCount === 0 ? "Select Connectors" : `${selectedCount} connectors selected`}
aria-label={
selectedCount === 0
? "Select Connectors"
: `${selectedCount} connectors selected`
}
>
{/* Progress indicator */}
<div
className="absolute bottom-0 left-0 h-1 bg-primary"
style={{
<div
className="absolute bottom-0 left-0 h-1 bg-primary"
style={{
width: `${progressPercentage}%`,
transition: 'width 0.3s ease'
}}
transition: "width 0.3s ease",
}}
/>
<div className="flex items-center gap-1.5 z-10 relative">
{selectedCount === 0 ? renderEmptyState() : renderSelectedConnectors()}
<ChevronDown className="h-3 w-3 ml-0.5 text-muted-foreground opacity-70" />
@ -183,29 +214,32 @@ type ResearchModeControlProps = {
onChange: (value: ResearchMode) => void;
};
export const ResearchModeControl = ({ value, onChange }: ResearchModeControlProps) => {
export const ResearchModeControl = ({
value,
onChange,
}: ResearchModeControlProps) => {
// Determine if we're in Q/A mode or Report mode
const isQnaMode = value === 'QNA';
const isReportMode = value.startsWith('REPORT_');
const isQnaMode = value === "QNA";
const isReportMode = value.startsWith("REPORT_");
// Get the current report sub-mode
const getCurrentReportMode = () => {
if (!isReportMode) return 'GENERAL';
return value.replace('REPORT_', '') as 'GENERAL' | 'DEEP' | 'DEEPER';
if (!isReportMode) return "GENERAL";
return value.replace("REPORT_", "") as "GENERAL" | "DEEP" | "DEEPER";
};
const reportSubOptions = [
{ value: 'GENERAL', label: 'General', icon: getConnectorIcon('GENERAL') },
{ value: 'DEEP', label: 'Deep', icon: getConnectorIcon('DEEP') },
{ value: 'DEEPER', label: 'Deeper', icon: getConnectorIcon('DEEPER') },
{ value: "GENERAL", label: "General", icon: getConnectorIcon("GENERAL") },
{ value: "DEEP", label: "Deep", icon: getConnectorIcon("DEEP") },
{ value: "DEEPER", label: "Deeper", icon: getConnectorIcon("DEEPER") },
];
const handleModeToggle = (mode: 'QNA' | 'REPORT') => {
if (mode === 'QNA') {
onChange('QNA');
const handleModeToggle = (mode: "QNA" | "REPORT") => {
if (mode === "QNA") {
onChange("QNA");
} else {
// Default to GENERAL for Report mode
onChange('REPORT_GENERAL');
onChange("REPORT_GENERAL");
}
};
@ -219,11 +253,11 @@ export const ResearchModeControl = ({ value, onChange }: ResearchModeControlProp
<div className="flex h-8 rounded-md border border-border overflow-hidden">
<button
className={`flex h-full items-center gap-1 px-3 text-xs font-medium transition-colors whitespace-nowrap ${
isQnaMode
? 'bg-primary text-primary-foreground'
: 'hover:bg-muted text-muted-foreground hover:text-foreground'
isQnaMode
? "bg-primary text-primary-foreground"
: "hover:bg-muted text-muted-foreground hover:text-foreground"
}`}
onClick={() => handleModeToggle('QNA')}
onClick={() => handleModeToggle("QNA")}
aria-pressed={isQnaMode}
>
<MessageCircle className="h-3 w-3" />
@ -231,11 +265,11 @@ export const ResearchModeControl = ({ value, onChange }: ResearchModeControlProp
</button>
<button
className={`flex h-full items-center gap-1 px-3 text-xs font-medium transition-colors whitespace-nowrap ${
isReportMode
? 'bg-primary text-primary-foreground'
: 'hover:bg-muted text-muted-foreground hover:text-foreground'
isReportMode
? "bg-primary text-primary-foreground"
: "hover:bg-muted text-muted-foreground hover:text-foreground"
}`}
onClick={() => handleModeToggle('REPORT')}
onClick={() => handleModeToggle("REPORT")}
aria-pressed={isReportMode}
>
<FileText className="h-3 w-3" />
@ -250,9 +284,9 @@ export const ResearchModeControl = ({ value, onChange }: ResearchModeControlProp
<button
key={option.value}
className={`flex h-full items-center gap-1 px-2 text-xs font-medium transition-colors whitespace-nowrap ${
getCurrentReportMode() === option.value
? 'bg-primary text-primary-foreground'
: 'hover:bg-muted text-muted-foreground hover:text-foreground'
getCurrentReportMode() === option.value
? "bg-primary text-primary-foreground"
: "hover:bg-muted text-muted-foreground hover:text-foreground"
}`}
onClick={() => handleReportSubModeChange(option.value)}
aria-pressed={getCurrentReportMode() === option.value}
@ -265,4 +299,4 @@ export const ResearchModeControl = ({ value, onChange }: ResearchModeControlProp
)}
</div>
);
};
};

25
surfsense_web/lib/connectors/utils.ts
View file

@ -1,14 +1,15 @@
// Helper function to get connector type display name
export const getConnectorTypeDisplay = (type: string): string => {
const typeMap: Record<string, string> = {
"SERPER_API": "Serper API",
"TAVILY_API": "Tavily API",
"SLACK_CONNECTOR": "Slack",
"NOTION_CONNECTOR": "Notion",
"GITHUB_CONNECTOR": "GitHub",
"LINEAR_CONNECTOR": "Linear",
"DISCORD_CONNECTOR": "Discord",
"LINKUP_API": "Linkup",
};
return typeMap[type] || type;
};
const typeMap: Record<string, string> = {
SERPER_API: "Serper API",
TAVILY_API: "Tavily API",
SLACK_CONNECTOR: "Slack",
NOTION_CONNECTOR: "Notion",
GITHUB_CONNECTOR: "GitHub",
LINEAR_CONNECTOR: "Linear",
JIRA_CONNECTOR: "Jira",
DISCORD_CONNECTOR: "Discord",
LINKUP_API: "Linkup",
};
return typeMap[type] || type;
};