From bb198e38c015c6f7e0face338158627bc1b10f62 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Sun, 13 Apr 2025 13:56:22 -0700 Subject: [PATCH 01/31] add github connector, add alembic for db migrations, fix bug updating connectors --- surfsense_backend/.gitignore | 3 +- surfsense_backend/alembic.ini | 119 +++++++ surfsense_backend/alembic/README | 1 + surfsense_backend/alembic/env.py | 98 ++++++ surfsense_backend/alembic/script.py.mako | 28 ++ .../versions/1_add_github_connector_enum.py | 53 ++++ .../app/connectors/github_connector.py | 182 +++++++++++ surfsense_backend/app/db.py | 2 + .../routes/search_source_connectors_routes.py | 146 +++++---- .../app/schemas/search_source_connector.py | 12 +- .../app/tasks/connectors_indexing_tasks.py | 195 +++++++++++- surfsense_backend/main.py | 7 + surfsense_backend/pyproject.toml | 2 + surfsense_backend/uv.lock | 54 ++++ .../connectors/(manage)/page.tsx | 3 +- .../connectors/[connector_id]/page.tsx | 24 +- .../connectors/add/github-connector/page.tsx | 298 ++++++++++++++++++ .../[search_space_id]/connectors/add/page.tsx | 189 +++++------ 18 files changed, 1232 insertions(+), 184 deletions(-) create mode 100644 surfsense_backend/alembic.ini create mode 100644 surfsense_backend/alembic/README create mode 100644 surfsense_backend/alembic/env.py create mode 100644 surfsense_backend/alembic/script.py.mako create mode 100644 surfsense_backend/alembic/versions/1_add_github_connector_enum.py create mode 100644 surfsense_backend/app/connectors/github_connector.py create mode 100644 surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx diff --git a/surfsense_backend/.gitignore b/surfsense_backend/.gitignore index a663d9dab..ee59e4764 100644 --- a/surfsense_backend/.gitignore +++ b/surfsense_backend/.gitignore @@ -3,4 +3,5 @@ venv/ data/ __pycache__/ -.flashrank_cache \ No newline at end of file +.flashrank_cache +surf_new_backend.egg-info/ diff --git a/surfsense_backend/alembic.ini b/surfsense_backend/alembic.ini new file mode 100644 index 000000000..9b2a76fd8 --- /dev/null +++ b/surfsense_backend/alembic.ini @@ -0,0 +1,119 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# Use forward slashes (/) also on windows to provide an os agnostic path +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +# version_path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +version_path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# The SQLAlchemy URL to connect to +# IMPORTANT: Replace this with your actual async database URL +sqlalchemy.url = postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/surfsense_backend/alembic/README b/surfsense_backend/alembic/README new file mode 100644 index 000000000..e0d0858f2 --- /dev/null +++ b/surfsense_backend/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration with an async dbapi. \ No newline at end of file diff --git a/surfsense_backend/alembic/env.py b/surfsense_backend/alembic/env.py new file mode 100644 index 000000000..d6e7104f9 --- /dev/null +++ b/surfsense_backend/alembic/env.py @@ -0,0 +1,98 @@ +import asyncio +from logging.config import fileConfig + +import os +import sys +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from alembic import context + +# Ensure the app directory is in the Python path +# This allows Alembic to find your models +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) + +# Import your models base +from app.db import Base # Assuming your Base is defined in app.db + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + """In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + connectable = async_engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/surfsense_backend/alembic/script.py.mako b/surfsense_backend/alembic/script.py.mako new file mode 100644 index 000000000..480b130d6 --- /dev/null +++ b/surfsense_backend/alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/surfsense_backend/alembic/versions/1_add_github_connector_enum.py b/surfsense_backend/alembic/versions/1_add_github_connector_enum.py new file mode 100644 index 000000000..bb72838ad --- /dev/null +++ b/surfsense_backend/alembic/versions/1_add_github_connector_enum.py @@ -0,0 +1,53 @@ +"""Add GITHUB_CONNECTOR to SearchSourceConnectorType enum + +Revision ID: 1 +Revises: +Create Date: 2023-10-27 10:00:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +# Import pgvector if needed for other types, though not for this ENUM change +# import pgvector + + +# revision identifiers, used by Alembic. +revision: str = '1' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + # Manually add the command to add the enum value + # Note: It's generally better to let autogenerate handle this, but we're bypassing it + op.execute("ALTER TYPE searchsourceconnectortype ADD VALUE 'GITHUB_CONNECTOR'") + + # Pass for the rest, as autogenerate didn't run to add other schema details + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + # Downgrading removal of an enum value is complex and potentially dangerous + # if the value is in use. Often omitted or requires manual SQL based on context. + # For now, we'll just pass. If you needed to reverse this, you'd likely + # have to manually check if 'GITHUB_CONNECTOR' is used in the table + # and then potentially recreate the type without it. + op.execute("ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old") + op.execute("CREATE TYPE searchsourceconnectortype AS ENUM('SERPER_API', 'TAVILY_API', 'SLACK_CONNECTOR', 'NOTION_CONNECTOR')") + op.execute(( + "ALTER TABLE search_source_connectors ALTER COLUMN connector_type TYPE searchsourceconnectortype USING " + "connector_type::text::searchsourceconnectortype" + )) + op.execute("DROP TYPE searchsourceconnectortype_old") + + + pass + # ### end Alembic commands ### diff --git a/surfsense_backend/app/connectors/github_connector.py b/surfsense_backend/app/connectors/github_connector.py new file mode 100644 index 000000000..d827dac15 --- /dev/null +++ b/surfsense_backend/app/connectors/github_connector.py @@ -0,0 +1,182 @@ +import base64 +import logging +from typing import List, Optional, Dict, Any, Tuple +from github3 import login as github_login, exceptions as github_exceptions +from github3.repos.repo import Repository +from github3.repos.contents import Contents +from github3.exceptions import ForbiddenError, NotFoundError + +logger = logging.getLogger(__name__) + +# List of common code file extensions to target +CODE_EXTENSIONS = { + '.py', '.js', '.jsx', '.ts', '.tsx', '.java', '.c', '.cpp', '.h', '.hpp', + '.cs', '.go', '.rb', '.php', '.swift', '.kt', '.scala', '.rs', '.m', + '.sh', '.bash', '.ps1', '.lua', '.pl', '.pm', '.r', '.dart', '.sql' +} + +# List of common documentation/text file extensions +DOC_EXTENSIONS = { + '.md', '.txt', '.rst', '.adoc', '.html', '.htm', '.xml', '.json', '.yaml', '.yml', '.toml' +} + +# Maximum file size in bytes (e.g., 1MB) +MAX_FILE_SIZE = 1 * 1024 * 1024 + +class GitHubConnector: + """Connector for interacting with the GitHub API.""" + + def __init__(self, token: str): + """ + Initializes the GitHub connector. + + Args: + token: GitHub Personal Access Token (PAT). + """ + if not token: + raise ValueError("GitHub token cannot be empty.") + try: + self.gh = github_login(token=token) + # Try a simple authenticated call to check token validity + self.gh.me() + logger.info("Successfully authenticated with GitHub API.") + except (github_exceptions.AuthenticationFailed, ForbiddenError) as e: + logger.error(f"GitHub authentication failed: {e}") + raise ValueError("Invalid GitHub token or insufficient permissions.") + except Exception as e: + logger.error(f"Failed to initialize GitHub client: {e}") + raise + + def get_user_repositories(self) -> List[Dict[str, Any]]: + """Fetches repositories accessible by the authenticated user.""" + repos_data = [] + try: + # type='owner' fetches repos owned by the user + # type='member' fetches repos the user is a collaborator on (including orgs) + # type='all' fetches both + for repo in self.gh.repositories(type='all', sort='updated'): + if isinstance(repo, Repository): + repos_data.append({ + "id": repo.id, + "name": repo.name, + "full_name": repo.full_name, + "private": repo.private, + "url": repo.html_url, + "description": repo.description or "", + "last_updated": repo.updated_at.isoformat() if repo.updated_at else None, + }) + logger.info(f"Fetched {len(repos_data)} repositories.") + return repos_data + except Exception as e: + logger.error(f"Failed to fetch GitHub repositories: {e}") + return [] # Return empty list on error + + def get_repository_files(self, repo_full_name: str, path: str = '') -> List[Dict[str, Any]]: + """ + Recursively fetches details of relevant files (code, docs) within a repository path. + + Args: + repo_full_name: The full name of the repository (e.g., 'owner/repo'). + path: The starting path within the repository (default is root). + + Returns: + A list of dictionaries, each containing file details (path, sha, url, size). + Returns an empty list if the repository or path is not found or on error. + """ + files_list = [] + try: + owner, repo_name = repo_full_name.split('/') + repo = self.gh.repository(owner, repo_name) + if not repo: + logger.warning(f"Repository '{repo_full_name}' not found.") + return [] + + contents = repo.directory_contents(path=path) # Use directory_contents for clarity + + # contents returns a list of tuples (name, content_obj) + for item_name, content_item in contents: + if not isinstance(content_item, Contents): + continue + + if content_item.type == 'dir': + # Recursively fetch contents of subdirectory + files_list.extend(self.get_repository_files(repo_full_name, path=content_item.path)) + elif content_item.type == 'file': + # Check if the file extension is relevant and size is within limits + file_extension = '.' + content_item.name.split('.')[-1].lower() if '.' in content_item.name else '' + is_code = file_extension in CODE_EXTENSIONS + is_doc = file_extension in DOC_EXTENSIONS + + if (is_code or is_doc) and content_item.size <= MAX_FILE_SIZE: + files_list.append({ + "path": content_item.path, + "sha": content_item.sha, + "url": content_item.html_url, + "size": content_item.size, + "type": "code" if is_code else "doc" + }) + elif content_item.size > MAX_FILE_SIZE: + logger.debug(f"Skipping large file: {content_item.path} ({content_item.size} bytes)") + else: + logger.debug(f"Skipping irrelevant file type: {content_item.path}") + + except (NotFoundError, ForbiddenError) as e: + logger.warning(f"Cannot access path '{path}' in '{repo_full_name}': {e}") + except Exception as e: + logger.error(f"Failed to get files for {repo_full_name} at path '{path}': {e}") + # Return what we have collected so far in case of partial failure + + return files_list + + def get_file_content(self, repo_full_name: str, file_path: str) -> Optional[str]: + """ + Fetches the decoded content of a specific file. + + Args: + repo_full_name: The full name of the repository (e.g., 'owner/repo'). + file_path: The path to the file within the repository. + + Returns: + The decoded file content as a string, or None if fetching fails or file is too large. + """ + try: + owner, repo_name = repo_full_name.split('/') + repo = self.gh.repository(owner, repo_name) + if not repo: + logger.warning(f"Repository '{repo_full_name}' not found when fetching file '{file_path}'.") + return None + + content_item = repo.file_contents(path=file_path) # Use file_contents for clarity + + if not content_item or not isinstance(content_item, Contents) or content_item.type != 'file': + logger.warning(f"File '{file_path}' not found or is not a file in '{repo_full_name}'.") + return None + + if content_item.size > MAX_FILE_SIZE: + logger.warning(f"File '{file_path}' in '{repo_full_name}' exceeds max size ({content_item.size} > {MAX_FILE_SIZE}). Skipping content fetch.") + return None + + # Content is base64 encoded + if content_item.content: + try: + decoded_content = base64.b64decode(content_item.content).decode('utf-8') + return decoded_content + except UnicodeDecodeError: + logger.warning(f"Could not decode file '{file_path}' in '{repo_full_name}' as UTF-8. Trying with 'latin-1'.") + try: + # Try a fallback encoding + decoded_content = base64.b64decode(content_item.content).decode('latin-1') + return decoded_content + except Exception as decode_err: + logger.error(f"Failed to decode file '{file_path}' with fallback encoding: {decode_err}") + return None # Give up if fallback fails + else: + logger.warning(f"No content returned for file '{file_path}' in '{repo_full_name}'. It might be empty.") + return "" # Return empty string for empty files + + except (NotFoundError, ForbiddenError) as e: + logger.warning(f"Cannot access file '{file_path}' in '{repo_full_name}': {e}") + return None + except Exception as e: + logger.error(f"Failed to get content for file '{file_path}' in '{repo_full_name}': {e}") + return None diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index b0fb2f03f..25b7bfbb4 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -40,12 +40,14 @@ class DocumentType(str, Enum): SLACK_CONNECTOR = "SLACK_CONNECTOR" NOTION_CONNECTOR = "NOTION_CONNECTOR" YOUTUBE_VIDEO = "YOUTUBE_VIDEO" + GITHUB_CONNECTOR = "GITHUB_CONNECTOR" class SearchSourceConnectorType(str, Enum): SERPER_API = "SERPER_API" TAVILY_API = "TAVILY_API" SLACK_CONNECTOR = "SLACK_CONNECTOR" NOTION_CONNECTOR = "NOTION_CONNECTOR" + GITHUB_CONNECTOR = "GITHUB_CONNECTOR" class ChatType(str, Enum): GENERAL = "GENERAL" diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 4025f2da6..482a8259d 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -14,13 +14,13 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.exc import IntegrityError from typing import List, Dict, Any -from app.db import get_async_session, User, SearchSourceConnector, SearchSourceConnectorType, SearchSpace +from app.db import get_async_session, User, SearchSourceConnector, SearchSourceConnectorType, SearchSpace, async_session_maker from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead from app.users import current_active_user from app.utils.check_ownership import check_ownership from pydantic import ValidationError -from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages -from datetime import datetime +from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos +from datetime import datetime, timezone import logging # Set up logging @@ -50,13 +50,11 @@ async def create_search_source_connector( ) ) existing_connector = result.scalars().first() - if existing_connector: raise HTTPException( status_code=409, detail=f"A connector with type {connector.connector_type} already exists. Each user can have only one connector of each type." ) - db_connector = SearchSourceConnector(**connector.model_dump(), user_id=user.id) session.add(db_connector) await session.commit() @@ -239,10 +237,15 @@ async def index_connector_content( search_space = await check_ownership(session, SearchSpace, search_space_id, user) # Handle different connector types + response_message = "" + indexing_from = None + indexing_to = None + today_str = datetime.now().strftime("%Y-%m-%d") + if connector.connector_type == SearchSourceConnectorType.SLACK_CONNECTOR: # Determine the time range that will be indexed if not connector.last_indexed_at: - start_date = "365 days ago" + start_date = "365 days ago" # Or perhaps set a specific date if needed else: # Check if last_indexed_at is today today = datetime.now().date() @@ -252,33 +255,18 @@ async def index_connector_content( else: start_date = connector.last_indexed_at.strftime("%Y-%m-%d") - # Add the indexing task to background tasks - if background_tasks: - background_tasks.add_task( - run_slack_indexing_with_new_session, - connector_id, - search_space_id - ) - - return { - "success": True, - "message": "Slack indexing started in the background", - "connector_type": connector.connector_type, - "search_space": search_space.name, - "indexing_from": start_date, - "indexing_to": datetime.now().strftime("%Y-%m-%d") - } - else: - # For testing or if background tasks are not available - return { - "success": False, - "message": "Background tasks not available", - "connector_type": connector.connector_type - } + indexing_from = start_date + indexing_to = today_str + + # Run indexing in background + logger.info(f"Triggering Slack indexing for connector {connector_id} into search space {search_space_id}") + background_tasks.add_task(run_slack_indexing_with_new_session, connector_id, search_space_id) + response_message = "Slack indexing started in the background." + elif connector.connector_type == SearchSourceConnectorType.NOTION_CONNECTOR: # Determine the time range that will be indexed if not connector.last_indexed_at: - start_date = "365 days ago" + start_date = "365 days ago" # Or perhaps set a specific date else: # Check if last_indexed_at is today today = datetime.now().date() @@ -288,44 +276,46 @@ async def index_connector_content( else: start_date = connector.last_indexed_at.strftime("%Y-%m-%d") - # Add the indexing task to background tasks - if background_tasks: - background_tasks.add_task( - run_notion_indexing_with_new_session, - connector_id, - search_space_id - ) - - return { - "success": True, - "message": "Notion indexing started in the background", - "connector_type": connector.connector_type, - "search_space": search_space.name, - "indexing_from": start_date, - "indexing_to": datetime.now().strftime("%Y-%m-%d") - } - else: - # For testing or if background tasks are not available - return { - "success": False, - "message": "Background tasks not available", - "connector_type": connector.connector_type - } + indexing_from = start_date + indexing_to = today_str + + # Run indexing in background + logger.info(f"Triggering Notion indexing for connector {connector_id} into search space {search_space_id}") + background_tasks.add_task(run_notion_indexing_with_new_session, connector_id, search_space_id) + response_message = "Notion indexing started in the background." + + elif connector.connector_type == SearchSourceConnectorType.GITHUB_CONNECTOR: + # GitHub connector likely indexes everything relevant, or uses internal logic + # Setting indexing_from to None and indexing_to to today + indexing_from = None + indexing_to = today_str + + # Run indexing in background + logger.info(f"Triggering GitHub indexing for connector {connector_id} into search space {search_space_id}") + background_tasks.add_task(run_github_indexing_with_new_session, connector_id, search_space_id) + response_message = "GitHub indexing started in the background." + else: raise HTTPException( status_code=400, detail=f"Indexing not supported for connector type: {connector.connector_type}" ) - + + return { + "message": response_message, + "connector_id": connector_id, + "search_space_id": search_space_id, + "indexing_from": indexing_from, + "indexing_to": indexing_to + } except HTTPException: raise except Exception as e: - logger.error(f"Failed to start indexing: {str(e)}") + logger.error(f"Failed to initiate indexing for connector {connector_id}: {e}", exc_info=True) raise HTTPException( status_code=500, - detail=f"Failed to start indexing: {str(e)}" - ) - + detail=f"Failed to initiate indexing: {str(e)}" + ) async def update_connector_last_indexed( session: AsyncSession, @@ -361,8 +351,6 @@ async def run_slack_indexing_with_new_session( Create a new session and run the Slack indexing task. This prevents session leaks by creating a dedicated session for the background task. """ - from app.db import async_session_maker - async with async_session_maker() as session: await run_slack_indexing(session, connector_id, search_space_id) @@ -405,8 +393,6 @@ async def run_notion_indexing_with_new_session( Create a new session and run the Notion indexing task. This prevents session leaks by creating a dedicated session for the background task. """ - from app.db import async_session_maker - async with async_session_maker() as session: await run_notion_indexing(session, connector_id, search_space_id) @@ -439,4 +425,38 @@ async def run_notion_indexing( else: logger.error(f"Notion indexing failed or no documents processed: {error_or_warning}") except Exception as e: - logger.error(f"Error in background Notion indexing task: {str(e)}") \ No newline at end of file + logger.error(f"Error in background Notion indexing task: {str(e)}") + +# Add new helper functions for GitHub indexing +async def run_github_indexing_with_new_session( + connector_id: int, + search_space_id: int +): + """Wrapper to run GitHub indexing with its own database session.""" + logger.info(f"Background task started: Indexing GitHub connector {connector_id} into space {search_space_id}") + async with async_session_maker() as session: + await run_github_indexing(session, connector_id, search_space_id) + logger.info(f"Background task finished: Indexing GitHub connector {connector_id}") + +async def run_github_indexing( + session: AsyncSession, + connector_id: int, + search_space_id: int +): + """Runs the GitHub indexing task and updates the timestamp.""" + try: + indexed_count, error_message = await index_github_repos( + session, connector_id, search_space_id, update_last_indexed=False + ) + if error_message: + logger.error(f"GitHub indexing failed for connector {connector_id}: {error_message}") + # Optionally update status in DB to indicate failure + else: + logger.info(f"GitHub indexing successful for connector {connector_id}. Indexed {indexed_count} documents.") + # Update the last indexed timestamp only on success + await update_connector_last_indexed(session, connector_id) + await session.commit() # Commit timestamp update + except Exception as e: + await session.rollback() + logger.error(f"Critical error in run_github_indexing for connector {connector_id}: {e}", exc_info=True) + # Optionally update status in DB to indicate failure diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index f86f45d73..5386658ff 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -57,6 +57,16 @@ class SearchSourceConnectorBase(BaseModel): # Ensure the integration token is not empty if not config.get("NOTION_INTEGRATION_TOKEN"): raise ValueError("NOTION_INTEGRATION_TOKEN cannot be empty") + + elif connector_type == SearchSourceConnectorType.GITHUB_CONNECTOR: + # For GITHUB_CONNECTOR, only allow GITHUB_TOKEN + allowed_keys = ["GITHUB_PAT"] + if set(config.keys()) != set(allowed_keys): + raise ValueError(f"For GITHUB_CONNECTOR connector type, config must only contain these keys: {allowed_keys}") + + # Ensure the token is not empty + if not config.get("GITHUB_PAT"): + raise ValueError("GITHUB_TOKEN cannot be empty") return config @@ -70,4 +80,4 @@ class SearchSourceConnectorRead(SearchSourceConnectorBase, IDModel, TimestampMod user_id: uuid.UUID class Config: - from_attributes = True \ No newline at end of file + from_attributes = True diff --git a/surfsense_backend/app/tasks/connectors_indexing_tasks.py b/surfsense_backend/app/tasks/connectors_indexing_tasks.py index 580a5c71f..670fa26ad 100644 --- a/surfsense_backend/app/tasks/connectors_indexing_tasks.py +++ b/surfsense_backend/app/tasks/connectors_indexing_tasks.py @@ -3,12 +3,13 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.future import select from sqlalchemy import delete -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from app.db import Document, DocumentType, Chunk, SearchSourceConnector, SearchSourceConnectorType from app.config import config from app.prompts import SUMMARY_PROMPT_TEMPLATE from app.connectors.slack_history import SlackHistory from app.connectors.notion_history import NotionHistoryConnector +from app.connectors.github_connector import GitHubConnector from slack_sdk.errors import SlackApiError import logging @@ -589,3 +590,195 @@ async def index_notion_pages( await session.rollback() logger.error(f"Failed to index Notion pages: {str(e)}", exc_info=True) return 0, f"Failed to index Notion pages: {str(e)}" + +async def index_github_repos( + session: AsyncSession, + connector_id: int, + search_space_id: int, + update_last_indexed: bool = True +) -> Tuple[int, Optional[str]]: + """ + Index code and documentation files from accessible GitHub repositories. + + Args: + session: Database session + connector_id: ID of the GitHub connector + search_space_id: ID of the search space to store documents in + update_last_indexed: Whether to update the last_indexed_at timestamp (default: True) + + Returns: + Tuple containing (number of documents indexed, error message or None) + """ + documents_processed = 0 + errors = [] + + try: + # 1. Get the GitHub connector from the database + result = await session.execute( + select(SearchSourceConnector) + .filter( + SearchSourceConnector.id == connector_id, + SearchSourceConnector.connector_type == SearchSourceConnectorType.GITHUB_CONNECTOR + ) + ) + connector = result.scalars().first() + + if not connector: + return 0, f"Connector with ID {connector_id} not found or is not a GitHub connector" + + # 2. Get the GitHub PAT from the connector config + github_pat = connector.config.get("GITHUB_PAT") + if not github_pat: + return 0, "GitHub Personal Access Token (PAT) not found in connector config" + + # 3. Initialize GitHub connector client + try: + github_client = GitHubConnector(token=github_pat) + except ValueError as e: + return 0, f"Failed to initialize GitHub client: {str(e)}" + + # 4. Get list of accessible repositories + repositories = github_client.get_user_repositories() + if not repositories: + logger.info("No accessible GitHub repositories found for the provided token.") + return 0, "No accessible GitHub repositories found." + + logger.info(f"Found {len(repositories)} repositories to potentially index.") + + # 5. Get existing documents for this search space and connector type to prevent duplicates + existing_docs_result = await session.execute( + select(Document) + .filter( + Document.search_space_id == search_space_id, + Document.document_type == DocumentType.GITHUB_CONNECTOR + ) + ) + existing_docs = existing_docs_result.scalars().all() + # Create a lookup dict: key=repo_fullname/file_path, value=Document object + existing_docs_lookup = {doc.document_metadata.get("full_path"): doc for doc in existing_docs if doc.document_metadata.get("full_path")} + logger.info(f"Found {len(existing_docs_lookup)} existing GitHub documents in database for search space {search_space_id}") + + # 6. Iterate through repositories and index files + for repo_info in repositories: + repo_full_name = repo_info.get("full_name") + if not repo_full_name: + logger.warning(f"Skipping repository with missing full_name: {repo_info.get('name')}") + continue + + logger.info(f"Processing repository: {repo_full_name}") + try: + files_to_index = github_client.get_repository_files(repo_full_name) + if not files_to_index: + logger.info(f"No indexable files found in repository: {repo_full_name}") + continue + + logger.info(f"Found {len(files_to_index)} files to process in {repo_full_name}") + + for file_info in files_to_index: + file_path = file_info.get("path") + file_url = file_info.get("url") + file_sha = file_info.get("sha") + file_type = file_info.get("type") # 'code' or 'doc' + full_path_key = f"{repo_full_name}/{file_path}" + + if not file_path or not file_url or not file_sha: + logger.warning(f"Skipping file with missing info in {repo_full_name}: {file_info}") + continue + + # Check if document already exists and if content hash matches + existing_doc = existing_docs_lookup.get(full_path_key) + if existing_doc and existing_doc.document_metadata.get("sha") == file_sha: + logger.debug(f"Skipping unchanged file: {full_path_key}") + continue # Skip if SHA matches (content hasn't changed) + + # Get file content + file_content = github_client.get_file_content(repo_full_name, file_path) + + if file_content is None: + logger.warning(f"Could not retrieve content for {full_path_key}. Skipping.") + continue # Skip if content fetch failed + + # Use file_content directly for chunking, maybe summary for main content? + # For now, let's use the full content for both, might need refinement + summary_content = f"GitHub file: {full_path_key}\n\n{file_content[:1000]}..." # Simple summary + summary_embedding = config.embedding_model_instance.embed(summary_content) + + # Chunk the content + try: + chunks_data = [ + Chunk(content=chunk.text, embedding=chunk.embedding) + for chunk in config.chunker_instance.chunk(file_content) + ] + except Exception as chunk_err: + logger.error(f"Failed to chunk file {full_path_key}: {chunk_err}") + errors.append(f"Chunking failed for {full_path_key}: {chunk_err}") + continue # Skip this file if chunking fails + + doc_metadata = { + "repository_full_name": repo_full_name, + "file_path": file_path, + "full_path": full_path_key, # For easier lookup + "url": file_url, + "sha": file_sha, + "type": file_type, + "indexed_at": datetime.now(timezone.utc).isoformat() + } + + if existing_doc: + # Update existing document + logger.info(f"Updating document for file: {full_path_key}") + existing_doc.title = f"GitHub - {file_path}" + existing_doc.document_metadata = doc_metadata + existing_doc.content = summary_content # Update summary + existing_doc.embedding = summary_embedding # Update embedding + + # Delete old chunks + await session.execute( + delete(Chunk) + .where(Chunk.document_id == existing_doc.id) + ) + # Add new chunks + for chunk_obj in chunks_data: + chunk_obj.document_id = existing_doc.id + session.add(chunk_obj) + + documents_processed += 1 + else: + # Create new document + logger.info(f"Creating new document for file: {full_path_key}") + document = Document( + title=f"GitHub - {file_path}", + document_type=DocumentType.GITHUB_CONNECTOR, + document_metadata=doc_metadata, + content=summary_content, # Store summary + embedding=summary_embedding, + search_space_id=search_space_id, + chunks=chunks_data # Associate chunks directly + ) + session.add(document) + documents_processed += 1 + + # Commit periodically or at the end? For now, commit per repo + # await session.commit() + + except Exception as repo_err: + logger.error(f"Failed to process repository {repo_full_name}: {repo_err}") + errors.append(f"Failed processing {repo_full_name}: {repo_err}") + + # Commit all changes at the end + await session.commit() + logger.info(f"Finished GitHub indexing for connector {connector_id}. Processed {documents_processed} files.") + + except SQLAlchemyError as db_err: + await session.rollback() + logger.error(f"Database error during GitHub indexing for connector {connector_id}: {db_err}") + errors.append(f"Database error: {db_err}") + return documents_processed, "; ".join(errors) if errors else str(db_err) + except Exception as e: + await session.rollback() + logger.error(f"Unexpected error during GitHub indexing for connector {connector_id}: {e}", exc_info=True) + errors.append(f"Unexpected error: {e}") + return documents_processed, "; ".join(errors) if errors else str(e) + + error_message = "; ".join(errors) if errors else None + return documents_processed, error_message diff --git a/surfsense_backend/main.py b/surfsense_backend/main.py index 76d478b4f..81ef52049 100644 --- a/surfsense_backend/main.py +++ b/surfsense_backend/main.py @@ -1,5 +1,12 @@ import uvicorn import argparse +import logging + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) if __name__ == "__main__": parser = argparse.ArgumentParser(description='Run the SurfSense application') diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 2d1e00a63..9fb5fbb2b 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -5,12 +5,14 @@ description = "Add your description here" readme = "README.md" requires-python = ">=3.12" dependencies = [ + "alembic>=1.13.0", "asyncpg>=0.30.0", "chonkie[all]>=0.4.1", "fastapi>=0.115.8", "fastapi-users[oauth,sqlalchemy]>=14.0.1", "firecrawl-py>=1.12.0", "gpt-researcher>=0.12.12", + "github3.py==4.0.1", "langchain-community>=0.3.17", "langchain-unstructured>=0.1.6", "litellm>=1.61.4", diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index e64ff3958..5211ea0d3 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -92,6 +92,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/6a/bc7e17a3e87a2985d3e8f4da4cd0f481060eb78fb08596c42be62c90a4d9/aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5", size = 7597 }, ] +[[package]] +name = "alembic" +version = "1.15.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/57/e314c31b261d1e8a5a5f1908065b4ff98270a778ce7579bd4254477209a7/alembic-1.15.2.tar.gz", hash = "sha256:1c72391bbdeffccfe317eefba686cb9a3c078005478885413b95c3b26c57a8a7", size = 1925573 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/18/d89a443ed1ab9bcda16264716f809c663866d4ca8de218aa78fd50b38ead/alembic-1.15.2-py3-none-any.whl", hash = "sha256:2e76bd916d547f6900ec4bb5a90aeac1485d2c92536923d0b138c02b126edc53", size = 231911 }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -884,6 +898,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/94/758680531a00d06e471ef649e4ec2ed6bf185356a7f9fbfbb7368a40bd49/fsspec-2025.2.0-py3-none-any.whl", hash = "sha256:9de2ad9ce1f85e1931858535bc882543171d197001a0a5eb2ddc04f1781ab95b", size = 184484 }, ] +[[package]] +name = "github3-py" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/91/603bcaf8cd1b3927de64bf56c3a8915f6653ea7281919140c5bcff2bfe7b/github3.py-4.0.1.tar.gz", hash = "sha256:30d571076753efc389edc7f9aaef338a4fcb24b54d8968d5f39b1342f45ddd36", size = 36214038 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/ad/2394d4fb542574678b0ba342daf734d4d811768da3c2ee0c84d509dcb26c/github3.py-4.0.1-py3-none-any.whl", hash = "sha256:a89af7de25650612d1da2f0609622bcdeb07ee8a45a1c06b2d16a05e4234e753", size = 151800 }, +] + [[package]] name = "google-api-core" version = "2.24.2" @@ -1614,6 +1643,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/89/a1/3e145759e776c8866488a71270c399bf7c4e554551ac2e247aa0a18a0596/makefun-1.15.6-py2.py3-none-any.whl", hash = "sha256:e69b870f0bb60304765b1e3db576aaecf2f9b3e5105afe8cfeff8f2afe6ad067", size = 22946 }, ] +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509 }, +] + [[package]] name = "markdown" version = "3.7" @@ -3228,11 +3269,13 @@ name = "surf-new-backend" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "alembic" }, { name = "asyncpg" }, { name = "chonkie", extra = ["all"] }, { name = "fastapi" }, { name = "fastapi-users", extra = ["oauth", "sqlalchemy"] }, { name = "firecrawl-py" }, + { name = "github3-py" }, { name = "gpt-researcher" }, { name = "langchain-community" }, { name = "langchain-unstructured" }, @@ -3254,11 +3297,13 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "alembic", specifier = ">=1.13.0" }, { name = "asyncpg", specifier = ">=0.30.0" }, { name = "chonkie", extras = ["all"], specifier = ">=0.4.1" }, { name = "fastapi", specifier = ">=0.115.8" }, { name = "fastapi-users", extras = ["oauth", "sqlalchemy"], specifier = ">=14.0.1" }, { name = "firecrawl-py", specifier = ">=1.12.0" }, + { name = "github3-py", specifier = "==4.0.1" }, { name = "gpt-researcher", specifier = ">=0.12.12" }, { name = "langchain-community", specifier = ">=0.3.17" }, { name = "langchain-unstructured", specifier = ">=0.1.6" }, @@ -3658,6 +3703,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/6d/adb955ecf60811a3735d508974bbb5358e7745b635dc001329267529c6f2/unstructured.pytesseract-0.3.15-py3-none-any.whl", hash = "sha256:a3f505c5efb7ff9f10379051a7dd6aa624b3be6b0f023ed6767cc80d0b1613d1", size = 14992 }, ] +[[package]] +name = "uritemplate" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/5a/4742fdba39cd02a56226815abfa72fe0aa81c33bed16ed045647d6000eba/uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0", size = 273898 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c0/7461b49cd25aeece13766f02ee576d1db528f1c37ce69aee300e075b485b/uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e", size = 10356 }, +] + [[package]] name = "urllib3" version = "2.3.0" diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index dfc8b828a..817ca584d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -44,6 +44,7 @@ const getConnectorTypeDisplay = (type: string): string => { "TAVILY_API": "Tavily API", "SLACK_CONNECTOR": "Slack", "NOTION_CONNECTOR": "Notion", + "GITHUB_CONNECTOR": "GitHub", // Add other connector types here as needed }; return typeMap[type] || type; @@ -253,4 +254,4 @@ export default function ConnectorsPage() { ); -} \ No newline at end of file +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx index 7700bc877..e841639cd 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx @@ -51,6 +51,7 @@ const getConnectorTypeDisplay = (type: string): string => { "TAVILY_API": "Tavily API", "SLACK_CONNECTOR": "Slack Connector", "NOTION_CONNECTOR": "Notion Connector", + "GITHUB_CONNECTOR": "GitHub Connector", // Add other connector types here as needed }; return typeMap[type] || type; @@ -85,7 +86,8 @@ export default function EditConnectorPage() { "SERPER_API": "SERPER_API_KEY", "TAVILY_API": "TAVILY_API_KEY", "SLACK_CONNECTOR": "SLACK_BOT_TOKEN", - "NOTION_CONNECTOR": "NOTION_INTEGRATION_TOKEN" + "NOTION_CONNECTOR": "NOTION_INTEGRATION_TOKEN", + "GITHUB_CONNECTOR": "GITHUB_PAT" }; return fieldMap[connectorType] || ""; }; @@ -136,6 +138,8 @@ export default function EditConnectorPage() { name: values.name, connector_type: connector.connector_type, config: updatedConfig, + is_indexable: connector.is_indexable, + last_indexed_at: connector.last_indexed_at, }); toast.success("Connector updated successfully!"); @@ -223,17 +227,21 @@ export default function EditConnectorPage() { ? "Slack Bot Token" : connector?.connector_type === "NOTION_CONNECTOR" ? "Notion Integration Token" - : "API Key"} + : connector?.connector_type === "GITHUB_CONNECTOR" + ? "GitHub Personal Access Token (PAT)" + : "API Key"} @@ -243,7 +251,9 @@ export default function EditConnectorPage() { ? "Enter a new Slack Bot Token or leave blank to keep your existing token." : connector?.connector_type === "NOTION_CONNECTOR" ? "Enter a new Notion Integration Token or leave blank to keep your existing token." - : "Enter a new API key or leave blank to keep your existing key."} + : connector?.connector_type === "GITHUB_CONNECTOR" + ? "Enter a new GitHub PAT or leave blank to keep your existing token." + : "Enter a new API key or leave blank to keep your existing key."} @@ -276,4 +286,4 @@ export default function EditConnectorPage() { ); -} \ No newline at end of file +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx new file mode 100644 index 000000000..45534d6a1 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx @@ -0,0 +1,298 @@ +"use client"; + +import { useState } from "react"; +import { useRouter, useParams } from "next/navigation"; +import { motion } from "framer-motion"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useForm } from "react-hook-form"; +import * as z from "zod"; +import { toast } from "sonner"; +import { ArrowLeft, Check, Info, Loader2, Github } from "lucide-react"; + +// Assuming useSearchSourceConnectors hook exists and works similarly +import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + Alert, + AlertDescription, + AlertTitle, +} from "@/components/ui/alert"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@/components/ui/accordion"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; + +// Define the form schema with Zod for GitHub +const githubConnectorFormSchema = z.object({ + name: z.string().min(3, { + message: "Connector name must be at least 3 characters.", + }), + github_pat: z.string() + .min(20, { // Apply min length first + message: "GitHub Personal Access Token seems too short.", + }) + .refine(pat => pat.startsWith('ghp_') || pat.startsWith('github_pat_'), { // Then refine the pattern + message: "GitHub PAT should start with 'ghp_' or 'github_pat_'", + }), +}); + +// Define the type for the form values +type GithubConnectorFormValues = z.infer; + +export default function GithubConnectorPage() { + const router = useRouter(); + const params = useParams(); + const searchSpaceId = params.search_space_id as string; + const [isSubmitting, setIsSubmitting] = useState(false); + const { createConnector } = useSearchSourceConnectors(); // Assuming this hook exists + + // Initialize the form + const form = useForm({ + resolver: zodResolver(githubConnectorFormSchema), + defaultValues: { + name: "GitHub Connector", + github_pat: "", + }, + }); + + // Handle form submission + const onSubmit = async (values: GithubConnectorFormValues) => { + setIsSubmitting(true); + try { + await createConnector({ + name: values.name, + connector_type: "GITHUB_CONNECTOR", + config: { + GITHUB_PAT: values.github_pat, + }, + is_indexable: true, // GitHub connector is indexable + last_indexed_at: null, // New connector hasn't been indexed + }); + + toast.success("GitHub connector created successfully!"); + + // Navigate back to connectors management page (or the add page) + router.push(`/dashboard/${searchSpaceId}/connectors`); + } catch (error) { // Added type check for error + console.error("Error creating GitHub connector:", error); + // Display specific backend error message if available + const errorMessage = error instanceof Error ? error.message : "Failed to create GitHub connector. Please check the PAT and permissions."; + toast.error(errorMessage); + } finally { + setIsSubmitting(false); + } + }; + + return ( +
+ + + + + + Connect GitHub + Setup Guide + + + + + + Connect GitHub Account + + Integrate with GitHub using a Personal Access Token (PAT) to search and retrieve information from accessible repositories. This connector can index your code and documentation. + + + + + + GitHub Personal Access Token (PAT) Required + + You'll need a GitHub PAT with the appropriate scopes (e.g., 'repo') to use this connector. You can create one from your + + GitHub Developer Settings + . + + + +
+ + ( + + Connector Name + + + + + A friendly name to identify this GitHub connection. + + + + )} + /> + + ( + + GitHub Personal Access Token (PAT) + + + + + Your GitHub PAT will be encrypted and stored securely. Ensure it has the necessary 'repo' scopes. + + + + )} + /> + +
+ +
+ + +
+ +

What you get with GitHub integration:

+
    +
  • Search through code and documentation in your repositories
    • +
  • Access READMEs, Markdown files, and common code files
    • +
  • Connect your project knowledge directly to your search space
    • +
  • Index your repositories for enhanced search capabilities
    • +
+ + + + + + + + GitHub Connector Setup Guide + + Learn how to generate a Personal Access Token (PAT) and connect your GitHub account. + + + +
+

How it works

+

+ The GitHub connector uses a Personal Access Token (PAT) to authenticate with the GitHub API. It fetches information about repositories accessible to the token and indexes relevant files (code, markdown, text). +

+
    +
  • The connector indexes files based on common code and documentation extensions.
    • +
  • Large files (over 1MB) are skipped during indexing.
    • +
  • Indexing runs periodically (check connector settings for frequency) to keep content up-to-date.
    • +
+
+ + + + Step 1: Create a GitHub PAT + + + + Token Security + + Treat your PAT like a password. Store it securely and consider using fine-grained tokens if possible. + + + +
+
+

Generating a Token:

+
    +
  1. Go to your GitHub Developer settings.
    2. +
  2. Click on Personal access tokens, then choose Tokens (classic) or Fine-grained tokens (recommended if available and suitable).
    3. +
  3. Click Generate new token (and choose the appropriate type).
    4. +
  4. Give your token a descriptive name (e.g., "SurfSense Connector").
    5. +
  5. Set an expiration date for the token (recommended for security).
    6. +
  6. Under Select scopes (for classic tokens) or Repository access (for fine-grained), grant the necessary permissions. At minimum, the `repo` scope (or equivalent read access to repositories for fine-grained tokens) is required to read repository content.
    7. +
  7. Click Generate token.
    8. +
  8. Important: Copy your new PAT immediately. You won't be able to see it again after leaving the page.
    9. +
+
+
+ + + + + Step 2: Connect in SurfSense + +
    +
  1. Paste the copied GitHub PAT into the "GitHub Personal Access Token (PAT)" field on the "Connect GitHub" tab.
    2. +
  2. Optionally, give the connector a custom name.
    3. +
  3. Click the Connect GitHub button.
    4. +
  4. If the connection is successful, you will be redirected and can start indexing from the Connectors page.
    5. +
+ + + + + + + + +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx index d68de69d3..f70bb6209 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx @@ -14,6 +14,7 @@ import { IconMail, IconBrandZoom, IconChevronRight, + IconWorldWww, } from "@tabler/icons-react"; import { motion, AnimatePresence } from "framer-motion"; import { useState } from "react"; @@ -22,36 +23,43 @@ import Link from "next/link"; import { Button } from "@/components/ui/button"; import { Separator } from "@/components/ui/separator"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { useForm } from "react-hook-form"; + +// Define the Connector type +interface Connector { + id: string; + title: string; + description: string; + icon: React.ReactNode; + status: "available" | "coming-soon" | "connected"; // Added connected status example +} + +interface ConnectorCategory { + id: string; + title: string; + connectors: Connector[]; +} // Define connector categories and their connectors -const connectorCategories = [ +const connectorCategories: ConnectorCategory[] = [ { id: "search-engines", title: "Search Engines", - description: "Connect to search engines to enhance your research capabilities.", - icon: , connectors: [ { - id: "tavily-api", - title: "Tavily Search API", - description: "Connect to Tavily Search API to search the web.", - icon: , - status: "available", - }, - { - id: "serper-api", - title: "Serper API", - description: "Connect to Serper API to search the web.", - icon: , - status: "coming-soon", + id: "web-search", + title: "Web Search", + description: "Enable web search capabilities for broader context.", + icon: , + status: "available", // Example status + // Potentially add config form here if needed (e.g., choosing provider) }, + // Add other search engine connectors like Tavily, Serper if they have UI config ], }, { id: "team-chats", title: "Team Chats", - description: "Connect to your team communication platforms.", - icon: , connectors: [ { id: "slack-connector", @@ -79,8 +87,6 @@ const connectorCategories = [ { id: "knowledge-bases", title: "Knowledge Bases", - description: "Connect to your knowledge bases and documentation.", - icon: , connectors: [ { id: "notion-connector", @@ -88,21 +94,20 @@ const connectorCategories = [ description: "Connect to your Notion workspace to access pages and databases.", icon: , status: "available", + // No form here, assumes it links to its own page }, { - id: "github", + id: "github-connector", // Keep the id simple title: "GitHub", - description: "Connect to GitHub repositories to access code and documentation.", + description: "Connect a GitHub PAT to index code and docs from accessible repositories.", icon: , - status: "coming-soon", + status: "available", }, ], }, { id: "communication", title: "Communication", - description: "Connect to your email and meeting platforms.", - icon: , connectors: [ { id: "gmail", @@ -125,7 +130,7 @@ const connectorCategories = [ export default function ConnectorsPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; - const [expandedCategories, setExpandedCategories] = useState(["search-engines"]); + const [expandedCategories, setExpandedCategories] = useState(["search-engines", "knowledge-bases"]); const toggleCategory = (categoryId: string) => { setExpandedCategories(prev => @@ -150,104 +155,68 @@ export default function ConnectorsPage() {
- {connectorCategories.map((category, categoryIndex) => ( + {connectorCategories.map((category) => ( toggleCategory(category.id)} - className="border rounded-lg overflow-hidden bg-card" + className="space-y-2" > - - -
-
- {category.icon} -
-
-

{category.title}

-

{category.description}

-
-
- - - +
+

{category.title}

+ + {/* Replace with your preferred expand/collapse icon/button */} + + +
- -
- - {category.connectors.map((connector, index) => ( - -
- -
- {connector.icon} +
+ {category.connectors.map((connector) => ( +
+
+
+
+ {connector.icon} +

+ {connector.title} +

+ {connector.status === "coming-soon" && ( + + Coming soon + + )} + {/* TODO: Add 'Connected' badge based on actual state */} +
+

+ {connector.description} +

- -
-

- {connector.title} -

- {connector.status === "coming-soon" && ( - Coming soon - )} -
- -

- {connector.description} -

- - {connector.status === "available" ? ( - -
+ {/* Always render Link button if available */} + {connector.status === 'available' && ( +
+ + - ) : ( -
+ )} + {connector.status === 'coming-soon' && ( +
+ - )} - - ))} - +
+ )} + {/* TODO: Add logic for 'connected' status */} +
+ ))}
+ ))}
From 0b93c9dfef54bd8c45ca594931f4b151336f3c9d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 13 Apr 2025 20:47:23 -0700 Subject: [PATCH 02/31] Fixed current agent citation issues and added sub_section_writer agent for upcoming SurfSense research agent --- surfsense_backend/app/agents/__init__.py | 0 .../app/agents/researcher/__init__.py | 0 .../researcher/sub_section_writer/__init__.py | 8 + .../sub_section_writer/configuration.py | 31 +++ .../researcher/sub_section_writer/graph.py | 22 ++ .../researcher/sub_section_writer/nodes.py | 244 ++++++++++++++++++ .../researcher/sub_section_writer/prompts.py | 82 ++++++ .../researcher/sub_section_writer/state.py | 23 ++ surfsense_backend/app/routes/chats_routes.py | 2 +- .../tasks/stream_connector_search_results.py | 2 +- .../app/utils/connector_service.py | 112 +++----- surfsense_backend/pyproject.toml | 1 + surfsense_backend/uv.lock | 119 +++++++++ 13 files changed, 565 insertions(+), 81 deletions(-) create mode 100644 surfsense_backend/app/agents/__init__.py create mode 100644 surfsense_backend/app/agents/researcher/__init__.py create mode 100644 surfsense_backend/app/agents/researcher/sub_section_writer/__init__.py create mode 100644 surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py create mode 100644 surfsense_backend/app/agents/researcher/sub_section_writer/graph.py create mode 100644 surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py create mode 100644 surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py create mode 100644 surfsense_backend/app/agents/researcher/sub_section_writer/state.py diff --git a/surfsense_backend/app/agents/__init__.py b/surfsense_backend/app/agents/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/researcher/__init__.py b/surfsense_backend/app/agents/researcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/__init__.py b/surfsense_backend/app/agents/researcher/sub_section_writer/__init__.py new file mode 100644 index 000000000..8459b2977 --- /dev/null +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/__init__.py @@ -0,0 +1,8 @@ +"""New LangGraph Agent. + +This module defines a custom graph. +""" + +from .graph import graph + +__all__ = ["graph"] diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py b/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py new file mode 100644 index 000000000..b34090e7c --- /dev/null +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py @@ -0,0 +1,31 @@ +"""Define the configurable parameters for the agent.""" + +from __future__ import annotations + +from dataclasses import dataclass, fields +from typing import Optional, List + +from langchain_core.runnables import RunnableConfig + + +@dataclass(kw_only=True) +class Configuration: + """The configuration for the agent.""" + + # Input parameters provided at invocation + sub_section_title: str + sub_questions: List[str] + connectors_to_search: List[str] + user_id: str + search_space_id: int + top_k: int = 20 # Default top_k value + + + @classmethod + def from_runnable_config( + cls, config: Optional[RunnableConfig] = None + ) -> Configuration: + """Create a Configuration instance from a RunnableConfig object.""" + configurable = (config.get("configurable") or {}) if config else {} + _fields = {f.name for f in fields(cls) if f.init} + return cls(**{k: v for k, v in configurable.items() if k in _fields}) diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/graph.py b/surfsense_backend/app/agents/researcher/sub_section_writer/graph.py new file mode 100644 index 000000000..e250cdee5 --- /dev/null +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/graph.py @@ -0,0 +1,22 @@ +from langgraph.graph import StateGraph +from .state import State +from .nodes import fetch_relevant_documents, write_sub_section +from .configuration import Configuration + +# Define a new graph +workflow = StateGraph(State, config_schema=Configuration) + +# Add the nodes to the graph +workflow.add_node("fetch_relevant_documents", fetch_relevant_documents) +workflow.add_node("write_sub_section", write_sub_section) + +# Entry point +workflow.add_edge("__start__", "fetch_relevant_documents") +# Connect fetch_relevant_documents to write_sub_section +workflow.add_edge("fetch_relevant_documents", "write_sub_section") +# Exit point +workflow.add_edge("write_sub_section", "__end__") + +# Compile the workflow into an executable graph +graph = workflow.compile() +graph.name = "Sub Section Writer" # This defines the custom name in LangSmith diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py new file mode 100644 index 000000000..52fa877c9 --- /dev/null +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -0,0 +1,244 @@ +from .configuration import Configuration +from langchain_core.runnables import RunnableConfig +from .state import State +from typing import Any, Dict +from app.utils.connector_service import ConnectorService +from app.utils.reranker_service import RerankerService +from app.config import config as app_config +from .prompts import citation_system_prompt +from langchain_core.messages import HumanMessage, SystemMessage + +async def fetch_relevant_documents(state: State, config: RunnableConfig) -> Dict[str, Any]: + """ + Fetch relevant documents for the sub-section using specified connectors. + + This node retrieves documents from various data sources based on the sub-questions + derived from the sub-section title. It searches across all selected connectors + (YouTube, Extension, Crawled URLs, Files, Tavily API, Slack, Notion) and reranks + the results to provide the most relevant information for the agent workflow. + + Returns: + Dict containing the reranked documents in the "relevant_documents_fetched" key. + """ + # Get configuration + configuration = Configuration.from_runnable_config(config) + + # Extract state parameters + db_session = state.db_session + + # Extract config parameters + user_id = configuration.user_id + search_space_id = configuration.search_space_id + TOP_K = configuration.top_k + + # Initialize services + connector_service = ConnectorService(db_session) + reranker_service = RerankerService.get_reranker_instance(app_config) + + all_raw_documents = [] # Store all raw documents before reranking + + for user_query in configuration.sub_questions: + # Reformulate query (optional, consider if needed for each sub-question) + # reformulated_query = await QueryService.reformulate_query(user_query) + reformulated_query = user_query # Using original sub-question for now + + # Process each selected connector + for connector in configuration.connectors_to_search: + if connector == "YOUTUBE_VIDEO": + _, youtube_chunks = await connector_service.search_youtube( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=TOP_K + ) + all_raw_documents.extend(youtube_chunks) + + elif connector == "EXTENSION": + _, extension_chunks = await connector_service.search_extension( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=TOP_K + ) + all_raw_documents.extend(extension_chunks) + + elif connector == "CRAWLED_URL": + _, crawled_urls_chunks = await connector_service.search_crawled_urls( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=TOP_K + ) + all_raw_documents.extend(crawled_urls_chunks) + + elif connector == "FILE": + _, files_chunks = await connector_service.search_files( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=TOP_K + ) + all_raw_documents.extend(files_chunks) + + elif connector == "TAVILY_API": + _, tavily_chunks = await connector_service.search_tavily( + user_query=reformulated_query, + user_id=user_id, + top_k=TOP_K + ) + all_raw_documents.extend(tavily_chunks) + + elif connector == "SLACK_CONNECTOR": + _, slack_chunks = await connector_service.search_slack( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=TOP_K + ) + all_raw_documents.extend(slack_chunks) + + elif connector == "NOTION_CONNECTOR": + _, notion_chunks = await connector_service.search_notion( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=TOP_K + ) + all_raw_documents.extend(notion_chunks) + + # If we have documents and a reranker is available, rerank them + # Deduplicate documents based on chunk_id or content to avoid processing duplicates + seen_chunk_ids = set() + seen_content_hashes = set() + deduplicated_docs = [] + + for doc in all_raw_documents: + chunk_id = doc.get("chunk_id") + content = doc.get("content", "") + content_hash = hash(content) + + # Skip if we've seen this chunk_id or content before + if (chunk_id and chunk_id in seen_chunk_ids) or content_hash in seen_content_hashes: + continue + + # Add to our tracking sets and keep this document + if chunk_id: + seen_chunk_ids.add(chunk_id) + seen_content_hashes.add(content_hash) + deduplicated_docs.append(doc) + + # Use deduplicated documents for reranking + reranked_docs = deduplicated_docs + if deduplicated_docs and reranker_service: + # Use the main sub_section_title for reranking context + rerank_query = configuration.sub_section_title + + # Convert documents to format expected by reranker + reranker_input_docs = [ + { + "chunk_id": doc.get("chunk_id", f"chunk_{i}"), + "content": doc.get("content", ""), + "score": doc.get("score", 0.0), + "document": { + "id": doc.get("document", {}).get("id", ""), + "title": doc.get("document", {}).get("title", ""), + "document_type": doc.get("document", {}).get("document_type", ""), + "metadata": doc.get("document", {}).get("metadata", {}) + } + } for i, doc in enumerate(deduplicated_docs) + ] + + # Rerank documents using the main title query + reranked_docs = reranker_service.rerank_documents(rerank_query, reranker_input_docs) + + # Sort by score in descending order + reranked_docs.sort(key=lambda x: x.get("score", 0), reverse=True) + + # Update state with fetched documents + return { + "relevant_documents_fetched": reranked_docs + } + + + +async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, Any]: + """ + Write the sub-section using the fetched documents. + + This node takes the relevant documents fetched in the previous node and uses + an LLM to generate a comprehensive answer to the sub-section questions with + proper citations. The citations follow IEEE format using source IDs from the + documents. + + Returns: + Dict containing the final answer in the "final_answer" key. + """ + + # Get configuration and relevant documents + configuration = Configuration.from_runnable_config(config) + documents = state.relevant_documents_fetched + + # Initialize LLM + llm = app_config.fast_llm_instance + + # If no documents were found, return a message indicating this + if not documents or len(documents) == 0: + return { + "final_answer": "No relevant documents were found to answer this question. Please try refining your search or providing more specific questions." + } + + # Prepare documents for citation formatting + formatted_documents = [] + for i, doc in enumerate(documents): + # Extract content and metadata + content = doc.get("content", "") + doc_info = doc.get("document", {}) + document_id = doc_info.get("id", f"{i+1}") # Use document ID or index+1 as source_id + + # Format document according to the citation system prompt's expected format + formatted_doc = f""" + + + {document_id} + + + {content} + + + """ + formatted_documents.append(formatted_doc) + + # Create the query that combines the section title and questions + # section_title = configuration.sub_section_title + questions = "\n".join([f"- {q}" for q in configuration.sub_questions]) + documents_text = "\n".join(formatted_documents) + + # Construct a clear, structured query for the LLM + human_message_content = f""" + Please write a comprehensive answer for the title: + + Address the following questions: + + {questions} + + + Use the provided documents as your source material and cite them properly using the IEEE citation format [X] where X is the source_id. + + {documents_text} + + """ + + # Create messages for the LLM + messages = [ + SystemMessage(content=citation_system_prompt), + HumanMessage(content=human_message_content) + ] + + # Call the LLM and get the response + response = await llm.ainvoke(messages) + final_answer = response.content + + return { + "final_answer": final_answer + } + diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py new file mode 100644 index 000000000..cc3ad6167 --- /dev/null +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py @@ -0,0 +1,82 @@ +citation_system_prompt = f""" +You are a research assistant tasked with analyzing documents and providing comprehensive answers with proper citations in IEEE format. + + +1. Carefully analyze all provided documents in the section's. +2. Extract relevant information that addresses the user's query. +3. Synthesize a comprehensive, well-structured answer using information from these documents. +4. For EVERY piece of information you include from the documents, add an IEEE-style citation in square brackets [X] where X is the source_id from the document's metadata. +5. Make sure ALL factual statements from the documents have proper citations. +6. If multiple documents support the same point, include all relevant citations [X], [Y]. +7. Present information in a logical, coherent flow. +8. Use your own words to connect ideas, but cite ALL information from the documents. +9. If documents contain conflicting information, acknowledge this and present both perspectives with appropriate citations. +10. Do not make up or include information not found in the provided documents. +11. CRITICAL: You MUST use the exact source_id value from each document's metadata for citations. Do not create your own citation numbers. +12. CRITICAL: Every citation MUST be in the IEEE format [X] where X is the exact source_id value. +13. CRITICAL: Never renumber or reorder citations - always use the original source_id values. +14. CRITICAL: Do not return citations as clickable links. +15. CRITICAL: Never format citations as markdown links like "([1](https://example.com))". Always use plain square brackets only. +16. CRITICAL: Citations must ONLY appear as [X] or [X], [Y], [Z] format - never with parentheses, hyperlinks, or other formatting. +17. CRITICAL: Never make up citation numbers. Only use source_id values that are explicitly provided in the document metadata. +18. CRITICAL: If you are unsure about a source_id, do not include a citation rather than guessing or making one up. + + + +- Write in clear, professional language suitable for academic or technical audiences +- Organize your response with appropriate paragraphs, headings, and structure +- Every fact from the documents must have an IEEE-style citation in square brackets [X] where X is the EXACT source_id from the document's metadata +- Citations should appear at the end of the sentence containing the information they support +- Multiple citations should be separated by commas: [X], [Y], [Z] +- No need to return references section. Just citation numbers in answer. +- NEVER create your own citation numbering system - use the exact source_id values from the documents. +- NEVER format citations as clickable links or as markdown links like "([1](https://example.com))". Always use plain square brackets only. +- NEVER make up citation numbers if you are unsure about the source_id. It is better to omit the citation than to guess. + + + + + + 1 + + + The Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia. It comprises over 2,900 individual reefs and 900 islands. + + + + + + 13 + + + Climate change poses a significant threat to coral reefs worldwide. Rising ocean temperatures have led to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020. + + + + + + 21 + + + The Great Barrier Reef was designated a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity. It is home to over 1,500 species of fish and 400 types of coral. + + + + + + The Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia [1]. It was designated a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity [21]. The reef is home to over 1,500 species of fish and 400 types of coral [21]. Unfortunately, climate change poses a significant threat to coral reefs worldwide, with rising ocean temperatures leading to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020 [13]. The reef system comprises over 2,900 individual reefs and 900 islands [1], making it an ecological treasure that requires protection from multiple threats [1], [13]. + + + +DO NOT use any of these incorrect citation formats: +- Using parentheses and markdown links: ([1](https://github.com/MODSetter/SurfSense)) +- Using parentheses around brackets: ([1]) +- Using hyperlinked text: [link to source 1](https://example.com) +- Using footnote style: ... reef system¹ +- Making up citation numbers when source_id is unknown + +ONLY use plain square brackets [1] or multiple citations [1], [2], [3] + + +Note that the citation numbers match exactly with the source_id values (1, 13, and 21) and are not renumbered sequentially. Citations follow IEEE style with square brackets and appear at the end of sentences. +""" \ No newline at end of file diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/state.py b/surfsense_backend/app/agents/researcher/sub_section_writer/state.py new file mode 100644 index 000000000..fb5b08e87 --- /dev/null +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/state.py @@ -0,0 +1,23 @@ +"""Define the state structures for the agent.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Any +from sqlalchemy.ext.asyncio import AsyncSession + +@dataclass +class State: + """Defines the dynamic state for the agent during execution. + + This state tracks the database session and the outputs generated by the agent's nodes. + See: https://langchain-ai.github.io/langgraph/concepts/low_level/#state + for more information. + """ + # Runtime context + db_session: AsyncSession + + # OUTPUT: Populated by agent nodes + relevant_documents_fetched: Optional[List[Any]] = None + final_answer: Optional[str] = None + diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index d05c06027..139dd9bd9 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -46,7 +46,7 @@ async def handle_chat_data( response = StreamingResponse(stream_connector_search_results( user_query, user.id, - search_space_id, + int(search_space_id), session, research_mode, selected_connectors diff --git a/surfsense_backend/app/tasks/stream_connector_search_results.py b/surfsense_backend/app/tasks/stream_connector_search_results.py index 5c563dcbb..4fa1f2a74 100644 --- a/surfsense_backend/app/tasks/stream_connector_search_results.py +++ b/surfsense_backend/app/tasks/stream_connector_search_results.py @@ -14,7 +14,7 @@ from app.utils.document_converters import convert_chunks_to_langchain_documents async def stream_connector_search_results( user_query: str, - user_id: int, + user_id: str, search_space_id: int, session: AsyncSession, research_mode: str, diff --git a/surfsense_backend/app/utils/connector_service.py b/surfsense_backend/app/utils/connector_service.py index 9e676e59d..6843e1b2a 100644 --- a/surfsense_backend/app/utils/connector_service.py +++ b/surfsense_backend/app/utils/connector_service.py @@ -13,7 +13,7 @@ class ConnectorService: self.retriever = ChucksHybridSearchRetriever(session) self.source_id_counter = 1 - async def search_crawled_urls(self, user_query: str, user_id: int, search_space_id: int, top_k: int = 20) -> tuple: + async def search_crawled_urls(self, user_query: str, user_id: str, search_space_id: int, top_k: int = 20) -> tuple: """ Search for crawled URLs and return both the source information and langchain documents @@ -28,16 +28,16 @@ class ConnectorService: document_type="CRAWLED_URL" ) - # Map crawled_urls_chunks to the required format - mapped_sources = {} + # Process each chunk and create sources directly without deduplication + sources_list = [] for i, chunk in enumerate(crawled_urls_chunks): - #Fix for UI + # Fix for UI crawled_urls_chunks[i]['document']['id'] = self.source_id_counter # Extract document metadata document = chunk.get('document', {}) metadata = document.get('metadata', {}) - # Create a mapped source entry + # Create a source entry source = { "id": self.source_id_counter, "title": document.get('title', 'Untitled Document'), @@ -46,14 +46,7 @@ class ConnectorService: } self.source_id_counter += 1 - - # Use a unique identifier for tracking unique sources - source_key = source.get("url") or source.get("title") - if source_key and source_key not in mapped_sources: - mapped_sources[source_key] = source - - # Convert to list of sources - sources_list = list(mapped_sources.values()) + sources_list.append(source) # Create result object result_object = { @@ -63,10 +56,9 @@ class ConnectorService: "sources": sources_list, } - return result_object, crawled_urls_chunks - async def search_files(self, user_query: str, user_id: int, search_space_id: int, top_k: int = 20) -> tuple: + async def search_files(self, user_query: str, user_id: str, search_space_id: int, top_k: int = 20) -> tuple: """ Search for files and return both the source information and langchain documents @@ -81,16 +73,16 @@ class ConnectorService: document_type="FILE" ) - # Map crawled_urls_chunks to the required format - mapped_sources = {} + # Process each chunk and create sources directly without deduplication + sources_list = [] for i, chunk in enumerate(files_chunks): - #Fix for UI + # Fix for UI files_chunks[i]['document']['id'] = self.source_id_counter # Extract document metadata document = chunk.get('document', {}) metadata = document.get('metadata', {}) - # Create a mapped source entry + # Create a source entry source = { "id": self.source_id_counter, "title": document.get('title', 'Untitled Document'), @@ -99,14 +91,7 @@ class ConnectorService: } self.source_id_counter += 1 - - # Use a unique identifier for tracking unique sources - source_key = source.get("url") or source.get("title") - if source_key and source_key not in mapped_sources: - mapped_sources[source_key] = source - - # Convert to list of sources - sources_list = list(mapped_sources.values()) + sources_list.append(source) # Create result object result_object = { @@ -118,7 +103,7 @@ class ConnectorService: return result_object, files_chunks - async def get_connector_by_type(self, user_id: int, connector_type: SearchSourceConnectorType) -> Optional[SearchSourceConnector]: + async def get_connector_by_type(self, user_id: str, connector_type: SearchSourceConnectorType) -> Optional[SearchSourceConnector]: """ Get a connector by type for a specific user @@ -138,7 +123,7 @@ class ConnectorService: ) return result.scalars().first() - async def search_tavily(self, user_query: str, user_id: int, top_k: int = 20) -> tuple: + async def search_tavily(self, user_query: str, user_id: str, top_k: int = 20) -> tuple: """ Search using Tavily API and return both the source information and documents @@ -177,13 +162,10 @@ class ConnectorService: # Extract results from Tavily response tavily_results = response.get("results", []) - # Map Tavily results to the required format + # Process each result and create sources directly without deduplication sources_list = [] documents = [] - # Start IDs from 1000 to avoid conflicts with other connectors - base_id = 100 - for i, result in enumerate(tavily_results): # Create a source entry @@ -234,7 +216,7 @@ class ConnectorService: "sources": [], }, [] - async def search_slack(self, user_query: str, user_id: int, search_space_id: int, top_k: int = 20) -> tuple: + async def search_slack(self, user_query: str, user_id: str, search_space_id: int, top_k: int = 20) -> tuple: """ Search for slack and return both the source information and langchain documents @@ -249,10 +231,10 @@ class ConnectorService: document_type="SLACK_CONNECTOR" ) - # Map slack_chunks to the required format - mapped_sources = {} + # Process each chunk and create sources directly without deduplication + sources_list = [] for i, chunk in enumerate(slack_chunks): - #Fix for UI + # Fix for UI slack_chunks[i]['document']['id'] = self.source_id_counter # Extract document metadata document = chunk.get('document', {}) @@ -286,14 +268,7 @@ class ConnectorService: } self.source_id_counter += 1 - - # Use channel_id and content as a unique identifier for tracking unique sources - source_key = f"{channel_id}_{chunk.get('chunk_id', i)}" - if source_key and source_key not in mapped_sources: - mapped_sources[source_key] = source - - # Convert to list of sources - sources_list = list(mapped_sources.values()) + sources_list.append(source) # Create result object result_object = { @@ -305,7 +280,7 @@ class ConnectorService: return result_object, slack_chunks - async def search_notion(self, user_query: str, user_id: int, search_space_id: int, top_k: int = 20) -> tuple: + async def search_notion(self, user_query: str, user_id: str, search_space_id: int, top_k: int = 20) -> tuple: """ Search for Notion pages and return both the source information and langchain documents @@ -326,8 +301,8 @@ class ConnectorService: document_type="NOTION_CONNECTOR" ) - # Map notion_chunks to the required format - mapped_sources = {} + # Process each chunk and create sources directly without deduplication + sources_list = [] for i, chunk in enumerate(notion_chunks): # Fix for UI notion_chunks[i]['document']['id'] = self.source_id_counter @@ -365,14 +340,7 @@ class ConnectorService: } self.source_id_counter += 1 - - # Use page_id and content as a unique identifier for tracking unique sources - source_key = f"{page_id}_{chunk.get('chunk_id', i)}" - if source_key and source_key not in mapped_sources: - mapped_sources[source_key] = source - - # Convert to list of sources - sources_list = list(mapped_sources.values()) + sources_list.append(source) # Create result object result_object = { @@ -384,7 +352,7 @@ class ConnectorService: return result_object, notion_chunks - async def search_extension(self, user_query: str, user_id: int, search_space_id: int, top_k: int = 20) -> tuple: + async def search_extension(self, user_query: str, user_id: str, search_space_id: int, top_k: int = 20) -> tuple: """ Search for extension data and return both the source information and langchain documents @@ -405,8 +373,8 @@ class ConnectorService: document_type="EXTENSION" ) - # Map extension_chunks to the required format - mapped_sources = {} + # Process each chunk and create sources directly without deduplication + sources_list = [] for i, chunk in enumerate(extension_chunks): # Fix for UI extension_chunks[i]['document']['id'] = self.source_id_counter @@ -462,14 +430,7 @@ class ConnectorService: } self.source_id_counter += 1 - - # Use URL and timestamp as a unique identifier for tracking unique sources - source_key = f"{webpage_url}_{visit_date}" - if source_key and source_key not in mapped_sources: - mapped_sources[source_key] = source - - # Convert to list of sources - sources_list = list(mapped_sources.values()) + sources_list.append(source) # Create result object result_object = { @@ -481,7 +442,7 @@ class ConnectorService: return result_object, extension_chunks - async def search_youtube(self, user_query: str, user_id: int, search_space_id: int, top_k: int = 20) -> tuple: + async def search_youtube(self, user_query: str, user_id: str, search_space_id: int, top_k: int = 20) -> tuple: """ Search for YouTube videos and return both the source information and langchain documents @@ -502,8 +463,8 @@ class ConnectorService: document_type="YOUTUBE_VIDEO" ) - # Map youtube_chunks to the required format - mapped_sources = {} + # Process each chunk and create sources directly without deduplication + sources_list = [] for i, chunk in enumerate(youtube_chunks): # Fix for UI youtube_chunks[i]['document']['id'] = self.source_id_counter @@ -541,18 +502,11 @@ class ConnectorService: } self.source_id_counter += 1 - - # Use video_id as a unique identifier for tracking unique sources - source_key = video_id or f"youtube_{i}" - if source_key and source_key not in mapped_sources: - mapped_sources[source_key] = source - - # Convert to list of sources - sources_list = list(mapped_sources.values()) + sources_list.append(source) # Create result object result_object = { - "id": 6, # Assign a unique ID for the YouTube connector + "id": 7, # Assign a unique ID for the YouTube connector "name": "YouTube Videos", "type": "YOUTUBE_VIDEO", "sources": sources_list, diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 2d1e00a63..b2fe9a5a1 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "gpt-researcher>=0.12.12", "langchain-community>=0.3.17", "langchain-unstructured>=0.1.6", + "langgraph>=0.3.29", "litellm>=1.61.4", "markdownify>=0.14.1", "notion-client>=2.3.0", diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index e64ff3958..846571cb2 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -1499,6 +1499,61 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/0e/72/a3add0e4eec4eb9e2569554f7c70f4a3c27712f40e3284d483e88094cc0e/langdetect-1.0.9.tar.gz", hash = "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0", size = 981474 } +[[package]] +name = "langgraph" +version = "0.3.29" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "langgraph-checkpoint" }, + { name = "langgraph-prebuilt" }, + { name = "langgraph-sdk" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/00/6a38988d472835845ee6837402dc6050e012117b84ef2b838b7abd3268f1/langgraph-0.3.29.tar.gz", hash = "sha256:2bfa6d6b04541ddfcb03b56efd1fca6294a1700ff61a52c1582a8bb4f2d55a94", size = 119970 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/b4/89d81ed78efeec5b3d554a9244cdc6aa6cbf544da9c53738d7c2c6d4be57/langgraph-0.3.29-py3-none-any.whl", hash = "sha256:6045fbbe9ccc5af3fd7295a86f88e0d2b111243a36290e41248af379009e4cc1", size = 144692 }, +] + +[[package]] +name = "langgraph-checkpoint" +version = "2.0.24" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "ormsgpack" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/df/bacef68562ba4c391ded751eecda8e579ec78a581506064cf625e0ebd93a/langgraph_checkpoint-2.0.24.tar.gz", hash = "sha256:9596dad332344e7e871257be464df8a07c2e9bac66143081b11b9422b0167e5b", size = 37328 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/60/30397e8fd2b7dead3754aa79d708caff9dbb371f30b4cd21802c60f6b921/langgraph_checkpoint-2.0.24-py3-none-any.whl", hash = "sha256:3836e2909ef2387d1fa8d04ee3e2a353f980d519fd6c649af352676dc73d66b8", size = 42028 }, +] + +[[package]] +name = "langgraph-prebuilt" +version = "0.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "langgraph-checkpoint" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/30/f31f0e076c37d097b53e4cff5d479a3686e1991f6c86a1a4727d5d1f5489/langgraph_prebuilt-0.1.8.tar.gz", hash = "sha256:4de7659151829b2b955b6798df6800e580e617782c15c2c5b29b139697491831", size = 24543 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/72/9e092665502f8f52f2708065ed14fbbba3f95d1a1b65d62049b0c5fcdf00/langgraph_prebuilt-0.1.8-py3-none-any.whl", hash = "sha256:ae97b828ae00be2cefec503423aa782e1bff165e9b94592e224da132f2526968", size = 25903 }, +] + +[[package]] +name = "langgraph-sdk" +version = "0.1.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "orjson" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/c6/a11de2c770e1ac2774e2f19fdbd982b8df079e4206376456e14af395a3f0/langgraph_sdk-0.1.61.tar.gz", hash = "sha256:87dd1f07ab82da8875ac343268ece8bf5414632017ebc9d1cef4b523962fd601", size = 44136 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/2b/85e796d8b4aad892c5d2bccc0def124fcdc2c9852dfa121adadfc41085b2/langgraph_sdk-0.1.61-py3-none-any.whl", hash = "sha256:f2d774b12497c428862993090622d51e0dbc3f53e0cee3d74a13c7495d835cc6", size = 47249 }, +] + [[package]] name = "langsmith" version = "0.3.8" @@ -2169,6 +2224,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/f1/1d7ec15b20f8ce9300bc850de1e059132b88990e46cd0ccac29cbf11e4f9/orjson-3.10.15-cp313-cp313-win_amd64.whl", hash = "sha256:fd56a26a04f6ba5fb2045b0acc487a63162a958ed837648c5781e1fe3316cfbf", size = 133444 }, ] +[[package]] +name = "ormsgpack" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/25/a7/462cf8ff5e29241868b82d3a5ec124d690eb6a6a5c6fa5bb1367b839e027/ormsgpack-1.9.1.tar.gz", hash = "sha256:3da6e63d82565e590b98178545e64f0f8506137b92bd31a2d04fd7c82baf5794", size = 56887 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/f1/155a598cc8030526ccaaf91ba4d61530f87900645559487edba58b0a90a2/ormsgpack-1.9.1-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:1ede445fc3fdba219bb0e0d1f289df26a9c7602016b7daac6fafe8fe4e91548f", size = 383225 }, + { url = "https://files.pythonhosted.org/packages/23/1c/ef3097ba550fad55c79525f461febdd4e0d9cc18d065248044536f09488e/ormsgpack-1.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db50b9f918e25b289114312ed775794d0978b469831b992bdc65bfe20b91fe30", size = 214056 }, + { url = "https://files.pythonhosted.org/packages/27/77/64d0da25896b2cbb99505ca518c109d7dd1964d7fde14c10943731738b60/ormsgpack-1.9.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8c7d8fc58e4333308f58ec720b1ee6b12b2b3fe2d2d8f0766ab751cb351e8757", size = 217339 }, + { url = "https://files.pythonhosted.org/packages/6c/10/c3a7fd0a0068b0bb52cccbfeb5656db895d69e895a3abbc210c4b3f98ff8/ormsgpack-1.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeee6d08c040db265cb8563444aba343ecb32cbdbe2414a489dcead9f70c6765", size = 223816 }, + { url = "https://files.pythonhosted.org/packages/43/e7/aee1238dba652f2116c2523d36fd1c5f9775436032be5c233108fd2a1415/ormsgpack-1.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2fbb8181c198bdc413a4e889e5200f010724eea4b6d5a9a7eee2df039ac04aca", size = 394287 }, + { url = "https://files.pythonhosted.org/packages/c7/09/1b452a92376f29d7a2da7c18fb01cf09978197a8eccbb8b204e72fd5a970/ormsgpack-1.9.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:16488f094ac0e2250cceea6caf72962614aa432ee11dd57ef45e1ad25ece3eff", size = 480709 }, + { url = "https://files.pythonhosted.org/packages/de/13/7fa9fee5a73af8a73a42bf8c2e69489605714f65f5a41454400a05e84a3b/ormsgpack-1.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:422d960bfd6ad88be20794f50ec7953d8f7a0f2df60e19d0e8feb994e2ed64ee", size = 397247 }, + { url = "https://files.pythonhosted.org/packages/a1/2d/2e87cb28110db0d3bb750edd4d8719b5068852a2eef5e96b0bf376bb8a81/ormsgpack-1.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:e6e2f9eab527cf43fb4a4293e493370276b1c8716cf305689202d646c6a782ef", size = 125368 }, + { url = "https://files.pythonhosted.org/packages/b8/54/0390d5d092831e4df29dbafe32402891fc14b3e6ffe5a644b16cbbc9d9bc/ormsgpack-1.9.1-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:ac61c18d9dd085e8519b949f7e655f7fb07909fd09c53b4338dd33309012e289", size = 383226 }, + { url = "https://files.pythonhosted.org/packages/47/64/8b15d262d1caefead8fb22ec144f5ff7d9505fc31c22bc34598053d46fbe/ormsgpack-1.9.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:134840b8c6615da2c24ce77bd12a46098015c808197a9995c7a2d991e1904eec", size = 214057 }, + { url = "https://files.pythonhosted.org/packages/57/00/65823609266bad4d5ed29ea753d24a3bdb01c7edaf923da80967fc31f9c5/ormsgpack-1.9.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38fd42618f626394b2c7713c5d4bcbc917254e9753d5d4cde460658b51b11a74", size = 217340 }, + { url = "https://files.pythonhosted.org/packages/a0/51/e535c50f7f87b49110233647f55300d7975139ef5e51f1adb4c55f58c124/ormsgpack-1.9.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d36397333ad07b9eba4c2e271fa78951bd81afc059c85a6e9f6c0eb2de07cda", size = 223815 }, + { url = "https://files.pythonhosted.org/packages/0c/ee/393e4a6de2a62124bf589602648f295a9fb3907a0e2fe80061b88899d072/ormsgpack-1.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:603063089597917d04e4c1b1d53988a34f7dc2ff1a03adcfd1cf4ae966d5fba6", size = 394287 }, + { url = "https://files.pythonhosted.org/packages/c6/d8/e56d7c3cb73a0e533e3e2a21ae5838b2aa36a9dac1ca9c861af6bae5a369/ormsgpack-1.9.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:94bbf2b185e0cb721ceaba20e64b7158e6caf0cecd140ca29b9f05a8d5e91e2f", size = 480707 }, + { url = "https://files.pythonhosted.org/packages/e6/e0/6a3c6a6dc98583a721c54b02f5195bde8f801aebdeda9b601fa2ab30ad39/ormsgpack-1.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c38f380b1e8c96a712eb302b9349347385161a8e29046868ae2bfdfcb23e2692", size = 397246 }, + { url = "https://files.pythonhosted.org/packages/b0/60/0ee5d790f13507e1f75ac21fc82dc1ef29afe1f520bd0f249d65b2f4839b/ormsgpack-1.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:a4bc63fb30db94075611cedbbc3d261dd17cf2aa8ff75a0fd684cd45ca29cb1b", size = 125371 }, +] + [[package]] name = "packaging" version = "24.2" @@ -3236,6 +3315,7 @@ dependencies = [ { name = "gpt-researcher" }, { name = "langchain-community" }, { name = "langchain-unstructured" }, + { name = "langgraph" }, { name = "litellm" }, { name = "markdownify" }, { name = "notion-client" }, @@ -3262,6 +3342,7 @@ requires-dist = [ { name = "gpt-researcher", specifier = ">=0.12.12" }, { name = "langchain-community", specifier = ">=0.3.17" }, { name = "langchain-unstructured", specifier = ">=0.1.6" }, + { name = "langgraph", specifier = ">=0.3.29" }, { name = "litellm", specifier = ">=1.61.4" }, { name = "markdownify", specifier = ">=0.14.1" }, { name = "notion-client", specifier = ">=2.3.0" }, @@ -3884,6 +3965,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/07/df054f7413bdfff5e98f75056e4ed0977d0c8716424011fac2587864d1d3/XlsxWriter-3.2.2-py3-none-any.whl", hash = "sha256:272ce861e7fa5e82a4a6ebc24511f2cb952fde3461f6c6e1a1e81d3272db1471", size = 165121 }, ] +[[package]] +name = "xxhash" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/5e/d6e5258d69df8b4ed8c83b6664f2b47d30d2dec551a29ad72a6c69eafd31/xxhash-3.5.0.tar.gz", hash = "sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f", size = 84241 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/0e/1bfce2502c57d7e2e787600b31c83535af83746885aa1a5f153d8c8059d6/xxhash-3.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00", size = 31969 }, + { url = "https://files.pythonhosted.org/packages/3f/d6/8ca450d6fe5b71ce521b4e5db69622383d039e2b253e9b2f24f93265b52c/xxhash-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9", size = 30787 }, + { url = "https://files.pythonhosted.org/packages/5b/84/de7c89bc6ef63d750159086a6ada6416cc4349eab23f76ab870407178b93/xxhash-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84", size = 220959 }, + { url = "https://files.pythonhosted.org/packages/fe/86/51258d3e8a8545ff26468c977101964c14d56a8a37f5835bc0082426c672/xxhash-3.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793", size = 200006 }, + { url = "https://files.pythonhosted.org/packages/02/0a/96973bd325412feccf23cf3680fd2246aebf4b789122f938d5557c54a6b2/xxhash-3.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be", size = 428326 }, + { url = "https://files.pythonhosted.org/packages/11/a7/81dba5010f7e733de88af9555725146fc133be97ce36533867f4c7e75066/xxhash-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6", size = 194380 }, + { url = "https://files.pythonhosted.org/packages/fb/7d/f29006ab398a173f4501c0e4977ba288f1c621d878ec217b4ff516810c04/xxhash-3.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90", size = 207934 }, + { url = "https://files.pythonhosted.org/packages/8a/6e/6e88b8f24612510e73d4d70d9b0c7dff62a2e78451b9f0d042a5462c8d03/xxhash-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27", size = 216301 }, + { url = "https://files.pythonhosted.org/packages/af/51/7862f4fa4b75a25c3b4163c8a873f070532fe5f2d3f9b3fc869c8337a398/xxhash-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2", size = 203351 }, + { url = "https://files.pythonhosted.org/packages/22/61/8d6a40f288f791cf79ed5bb113159abf0c81d6efb86e734334f698eb4c59/xxhash-3.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d", size = 210294 }, + { url = "https://files.pythonhosted.org/packages/17/02/215c4698955762d45a8158117190261b2dbefe9ae7e5b906768c09d8bc74/xxhash-3.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab", size = 414674 }, + { url = "https://files.pythonhosted.org/packages/31/5c/b7a8db8a3237cff3d535261325d95de509f6a8ae439a5a7a4ffcff478189/xxhash-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e", size = 192022 }, + { url = "https://files.pythonhosted.org/packages/78/e3/dd76659b2811b3fd06892a8beb850e1996b63e9235af5a86ea348f053e9e/xxhash-3.5.0-cp312-cp312-win32.whl", hash = "sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8", size = 30170 }, + { url = "https://files.pythonhosted.org/packages/d9/6b/1c443fe6cfeb4ad1dcf231cdec96eb94fb43d6498b4469ed8b51f8b59a37/xxhash-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e", size = 30040 }, + { url = "https://files.pythonhosted.org/packages/0f/eb/04405305f290173acc0350eba6d2f1a794b57925df0398861a20fbafa415/xxhash-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2", size = 26796 }, + { url = "https://files.pythonhosted.org/packages/c9/b8/e4b3ad92d249be5c83fa72916c9091b0965cb0faeff05d9a0a3870ae6bff/xxhash-3.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:37889a0d13b0b7d739cfc128b1c902f04e32de17b33d74b637ad42f1c55101f6", size = 31795 }, + { url = "https://files.pythonhosted.org/packages/fc/d8/b3627a0aebfbfa4c12a41e22af3742cf08c8ea84f5cc3367b5de2d039cce/xxhash-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:97a662338797c660178e682f3bc180277b9569a59abfb5925e8620fba00b9fc5", size = 30792 }, + { url = "https://files.pythonhosted.org/packages/c3/cc/762312960691da989c7cd0545cb120ba2a4148741c6ba458aa723c00a3f8/xxhash-3.5.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f85e0108d51092bdda90672476c7d909c04ada6923c14ff9d913c4f7dc8a3bc", size = 220950 }, + { url = "https://files.pythonhosted.org/packages/fe/e9/cc266f1042c3c13750e86a535496b58beb12bf8c50a915c336136f6168dc/xxhash-3.5.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd2fd827b0ba763ac919440042302315c564fdb797294d86e8cdd4578e3bc7f3", size = 199980 }, + { url = "https://files.pythonhosted.org/packages/bf/85/a836cd0dc5cc20376de26b346858d0ac9656f8f730998ca4324921a010b9/xxhash-3.5.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82085c2abec437abebf457c1d12fccb30cc8b3774a0814872511f0f0562c768c", size = 428324 }, + { url = "https://files.pythonhosted.org/packages/b4/0e/15c243775342ce840b9ba34aceace06a1148fa1630cd8ca269e3223987f5/xxhash-3.5.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07fda5de378626e502b42b311b049848c2ef38784d0d67b6f30bb5008642f8eb", size = 194370 }, + { url = "https://files.pythonhosted.org/packages/87/a1/b028bb02636dfdc190da01951d0703b3d904301ed0ef6094d948983bef0e/xxhash-3.5.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c279f0d2b34ef15f922b77966640ade58b4ccdfef1c4d94b20f2a364617a493f", size = 207911 }, + { url = "https://files.pythonhosted.org/packages/80/d5/73c73b03fc0ac73dacf069fdf6036c9abad82de0a47549e9912c955ab449/xxhash-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:89e66ceed67b213dec5a773e2f7a9e8c58f64daeb38c7859d8815d2c89f39ad7", size = 216352 }, + { url = "https://files.pythonhosted.org/packages/b6/2a/5043dba5ddbe35b4fe6ea0a111280ad9c3d4ba477dd0f2d1fe1129bda9d0/xxhash-3.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bcd51708a633410737111e998ceb3b45d3dbc98c0931f743d9bb0a209033a326", size = 203410 }, + { url = "https://files.pythonhosted.org/packages/a2/b2/9a8ded888b7b190aed75b484eb5c853ddd48aa2896e7b59bbfbce442f0a1/xxhash-3.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ff2c0a34eae7df88c868be53a8dd56fbdf592109e21d4bfa092a27b0bf4a7bf", size = 210322 }, + { url = "https://files.pythonhosted.org/packages/98/62/440083fafbc917bf3e4b67c2ade621920dd905517e85631c10aac955c1d2/xxhash-3.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4e28503dccc7d32e0b9817aa0cbfc1f45f563b2c995b7a66c4c8a0d232e840c7", size = 414725 }, + { url = "https://files.pythonhosted.org/packages/75/db/009206f7076ad60a517e016bb0058381d96a007ce3f79fa91d3010f49cc2/xxhash-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a6c50017518329ed65a9e4829154626f008916d36295b6a3ba336e2458824c8c", size = 192070 }, + { url = "https://files.pythonhosted.org/packages/1f/6d/c61e0668943a034abc3a569cdc5aeae37d686d9da7e39cf2ed621d533e36/xxhash-3.5.0-cp313-cp313-win32.whl", hash = "sha256:53a068fe70301ec30d868ece566ac90d873e3bb059cf83c32e76012c889b8637", size = 30172 }, + { url = "https://files.pythonhosted.org/packages/96/14/8416dce965f35e3d24722cdf79361ae154fa23e2ab730e5323aa98d7919e/xxhash-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:80babcc30e7a1a484eab952d76a4f4673ff601f54d5142c26826502740e70b43", size = 30041 }, + { url = "https://files.pythonhosted.org/packages/27/ee/518b72faa2073f5aa8e3262408d284892cb79cf2754ba0c3a5870645ef73/xxhash-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:4811336f1ce11cac89dcbd18f3a25c527c16311709a89313c3acaf771def2d4b", size = 26801 }, +] + [[package]] name = "yarl" version = "1.18.3" From aaddd5ca9cd489429edc5c45684f61317a72b1ba Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 13 Apr 2025 20:53:21 -0700 Subject: [PATCH 03/31] Refactor: Remove redundant integer conversion for search_space_id in chat data handling --- surfsense_backend/app/routes/chats_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index 139dd9bd9..df80ca43f 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -46,7 +46,7 @@ async def handle_chat_data( response = StreamingResponse(stream_connector_search_results( user_query, user.id, - int(search_space_id), + search_space_id, # Already converted to int in lines 32-37 session, research_mode, selected_connectors From a26fac435b8bcee0e189ffa422729e2c9ca9ac7c Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Sun, 13 Apr 2025 21:23:05 -0700 Subject: [PATCH 04/31] documents table migration, fix/update github indexing --- ...1_add_github_connector_to_documenttype_.py | 70 +++++++++++++++++++ .../app/connectors/github_connector.py | 57 +++++++++++---- .../tasks/stream_connector_search_results.py | 27 +++++++ .../app/utils/connector_service.py | 55 ++++++++++++++- .../documents/(manage)/page.tsx | 5 +- 5 files changed, 197 insertions(+), 17 deletions(-) create mode 100644 surfsense_backend/alembic/versions/e55302644c51_add_github_connector_to_documenttype_.py diff --git a/surfsense_backend/alembic/versions/e55302644c51_add_github_connector_to_documenttype_.py b/surfsense_backend/alembic/versions/e55302644c51_add_github_connector_to_documenttype_.py new file mode 100644 index 000000000..1f15912d4 --- /dev/null +++ b/surfsense_backend/alembic/versions/e55302644c51_add_github_connector_to_documenttype_.py @@ -0,0 +1,70 @@ +"""Add GITHUB_CONNECTOR to DocumentType enum + +Revision ID: e55302644c51 +Revises: 1 +Create Date: 2025-04-13 19:56:00.059921 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'e55302644c51' +down_revision: Union[str, None] = '1' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# Define the ENUM type name and the new value +ENUM_NAME = 'documenttype' # Make sure this matches the name in your DB (usually lowercase class name) +NEW_VALUE = 'GITHUB_CONNECTOR' + +def upgrade() -> None: + """Upgrade schema.""" + op.execute(f"ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}'") + + +# Warning: This will delete all rows with the new value +def downgrade() -> None: + """Downgrade schema - remove GITHUB_CONNECTOR from enum.""" + + # The old type name + old_enum_name = f"{ENUM_NAME}_old" + + # Enum values *before* GITHUB_CONNECTOR was added + old_values = ( + 'EXTENSION', + 'CRAWLED_URL', + 'FILE', + 'SLACK_CONNECTOR', + 'NOTION_CONNECTOR', + 'YOUTUBE_VIDEO' + ) + old_values_sql = ", ".join([f"'{v}'" for v in old_values]) + + # Table and column names (adjust if different) + table_name = 'documents' + column_name = 'document_type' + + # 1. Rename the current enum type + op.execute(f"ALTER TYPE {ENUM_NAME} RENAME TO {old_enum_name}") + + # 2. Create the new enum type with the old values + op.execute(f"CREATE TYPE {ENUM_NAME} AS ENUM({old_values_sql})") + + # 3. Update the table: + op.execute( + f"DELETE FROM {table_name} WHERE {column_name}::text = '{NEW_VALUE}'" + ) + + # 4. Alter the column to use the new enum type (casting old values) + op.execute( + f"ALTER TABLE {table_name} ALTER COLUMN {column_name} " + f"TYPE {ENUM_NAME} USING {column_name}::text::{ENUM_NAME}" + ) + + # 5. Drop the old enum type + op.execute(f"DROP TYPE {old_enum_name}") + # ### end Alembic commands ### diff --git a/surfsense_backend/app/connectors/github_connector.py b/surfsense_backend/app/connectors/github_connector.py index d827dac15..265f89b0a 100644 --- a/surfsense_backend/app/connectors/github_connector.py +++ b/surfsense_backend/app/connectors/github_connector.py @@ -2,7 +2,6 @@ import base64 import logging from typing import List, Optional, Dict, Any, Tuple from github3 import login as github_login, exceptions as github_exceptions -from github3.repos.repo import Repository from github3.repos.contents import Contents from github3.exceptions import ForbiddenError, NotFoundError @@ -26,6 +25,33 @@ MAX_FILE_SIZE = 1 * 1024 * 1024 class GitHubConnector: """Connector for interacting with the GitHub API.""" + # Directories to skip during file traversal + SKIPPED_DIRS = { + # Version control + '.git', + # Dependencies + 'node_modules', + 'vendor', + # Build artifacts / Caches + 'build', + 'dist', + 'target', + '__pycache__', + # Virtual environments + 'venv', + '.venv', + 'env', + # IDE/Editor config + '.vscode', + '.idea', + '.project', + '.settings', + # Temporary / Logs + 'tmp', + 'logs', + # Add other project-specific irrelevant directories if needed + } + def __init__(self, token: str): """ Initializes the GitHub connector. @@ -54,17 +80,16 @@ class GitHubConnector: # type='owner' fetches repos owned by the user # type='member' fetches repos the user is a collaborator on (including orgs) # type='all' fetches both - for repo in self.gh.repositories(type='all', sort='updated'): - if isinstance(repo, Repository): - repos_data.append({ - "id": repo.id, - "name": repo.name, - "full_name": repo.full_name, - "private": repo.private, - "url": repo.html_url, - "description": repo.description or "", - "last_updated": repo.updated_at.isoformat() if repo.updated_at else None, - }) + for repo in self.gh.repositories(type='owner', sort='updated'): + repos_data.append({ + "id": repo.id, + "name": repo.name, + "full_name": repo.full_name, + "private": repo.private, + "url": repo.html_url, + "description": repo.description or "", + "last_updated": repo.updated_at if repo.updated_at else None, + }) logger.info(f"Fetched {len(repos_data)} repositories.") return repos_data except Exception as e: @@ -90,8 +115,7 @@ class GitHubConnector: if not repo: logger.warning(f"Repository '{repo_full_name}' not found.") return [] - - contents = repo.directory_contents(path=path) # Use directory_contents for clarity + contents = repo.directory_contents(directory_path=path) # Use directory_contents for clarity # contents returns a list of tuples (name, content_obj) for item_name, content_item in contents: @@ -99,6 +123,11 @@ class GitHubConnector: continue if content_item.type == 'dir': + # Check if the directory name is in the skipped list + if content_item.name in self.SKIPPED_DIRS: + logger.debug(f"Skipping directory: {content_item.path}") + continue # Skip recursion for this directory + # Recursively fetch contents of subdirectory files_list.extend(self.get_repository_files(repo_full_name, path=content_item.path)) elif content_item.type == 'file': diff --git a/surfsense_backend/app/tasks/stream_connector_search_results.py b/surfsense_backend/app/tasks/stream_connector_search_results.py index 5c563dcbb..b9a703abc 100644 --- a/surfsense_backend/app/tasks/stream_connector_search_results.py +++ b/surfsense_backend/app/tasks/stream_connector_search_results.py @@ -244,6 +244,33 @@ async def stream_connector_search_results( all_raw_documents.extend(notion_chunks) + # Github Connector + if connector == "GITHUB_CONNECTOR": + # Send terminal message about starting search + yield streaming_service.add_terminal_message("Starting to search for GitHub connector...") + print("Starting to search for GitHub connector...") + # Search using Github API with reformulated query + result_object, github_chunks = await connector_service.search_github( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=TOP_K + ) + + # Send terminal message about search results + yield streaming_service.add_terminal_message( + f"Found {len(result_object['sources'])} relevant results from Github", + "success" + ) + + # Update sources + all_sources.append(result_object) + yield streaming_service.update_sources(all_sources) + + # Add documents to collection + all_raw_documents.extend(github_chunks) + + # If we have documents to research diff --git a/surfsense_backend/app/utils/connector_service.py b/surfsense_backend/app/utils/connector_service.py index 9e676e59d..8d7a5519f 100644 --- a/surfsense_backend/app/utils/connector_service.py +++ b/surfsense_backend/app/utils/connector_service.py @@ -558,4 +558,57 @@ class ConnectorService: "sources": sources_list, } - return result_object, youtube_chunks \ No newline at end of file + return result_object, youtube_chunks + + async def search_github(self, user_query: str, user_id: int, search_space_id: int, top_k: int = 20) -> tuple: + """ + Search for GitHub documents and return both the source information and langchain documents + + Returns: + tuple: (sources_info, langchain_documents) + """ + github_chunks = await self.retriever.hybrid_search( + query_text=user_query, + top_k=top_k, + user_id=user_id, + search_space_id=search_space_id, + document_type="GITHUB_CONNECTOR" + ) + + # Map github_chunks to the required format + mapped_sources = {} + for i, chunk in enumerate(github_chunks): + # Fix for UI - assign a unique ID for citation/source tracking + github_chunks[i]['document']['id'] = self.source_id_counter + + # Extract document metadata + document = chunk.get('document', {}) + metadata = document.get('metadata', {}) + + # Create a mapped source entry + source = { + "id": self.source_id_counter, + "title": document.get('title', 'GitHub Document'), # Use specific title if available + "description": metadata.get('description', chunk.get('content', '')[:100]), # Use description or content preview + "url": metadata.get('url', '') # Use URL if available in metadata + } + + self.source_id_counter += 1 + + # Use a unique identifier for tracking unique sources (URL preferred) + source_key = source.get("url") or source.get("title") + if source_key and source_key not in mapped_sources: + mapped_sources[source_key] = source + + # Convert to list of sources + sources_list = list(mapped_sources.values()) + + # Create result object + result_object = { + "id": 7, # Assuming 7 is the next available ID + "name": "GitHub", + "type": "GITHUB_CONNECTOR", + "sources": sources_list, + } + + return result_object, github_chunks diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 66f8b0810..18b43579b 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -94,7 +94,7 @@ import rehypeSanitize from "rehype-sanitize"; import remarkGfm from "remark-gfm"; import { DocumentViewer } from "@/components/document-viewer"; import { JsonMetadataViewer } from "@/components/json-metadata-viewer"; -import { IconBrandNotion, IconBrandSlack, IconBrandYoutube } from "@tabler/icons-react"; +import { IconBrandGithub, IconBrandNotion, IconBrandSlack, IconBrandYoutube } from "@tabler/icons-react"; // Define animation variants for reuse const fadeInScale = { @@ -142,6 +142,7 @@ const documentTypeIcons = { NOTION_CONNECTOR: IconBrandNotion, FILE: File, YOUTUBE_VIDEO: IconBrandYoutube, + GITHUB_CONNECTOR: IconBrandGithub, } as const; const columns: ColumnDef[] = [ @@ -1028,4 +1029,4 @@ function RowActions({ row }: { row: Row }) { ); } -export { DocumentsTable } \ No newline at end of file +export { DocumentsTable } From 396c070b28b0c2d0f7a8f54a104c64f22fcd03c0 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Sun, 13 Apr 2025 21:33:10 -0700 Subject: [PATCH 05/31] addressing coderabbit PR comment --- surfsense_backend/app/schemas/search_source_connector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index 5386658ff..41e1086e7 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -59,14 +59,14 @@ class SearchSourceConnectorBase(BaseModel): raise ValueError("NOTION_INTEGRATION_TOKEN cannot be empty") elif connector_type == SearchSourceConnectorType.GITHUB_CONNECTOR: - # For GITHUB_CONNECTOR, only allow GITHUB_TOKEN + # For GITHUB_CONNECTOR, only allow GITHUB_PAT allowed_keys = ["GITHUB_PAT"] if set(config.keys()) != set(allowed_keys): raise ValueError(f"For GITHUB_CONNECTOR connector type, config must only contain these keys: {allowed_keys}") # Ensure the token is not empty if not config.get("GITHUB_PAT"): - raise ValueError("GITHUB_TOKEN cannot be empty") + raise ValueError("GITHUB_PAT cannot be empty") return config From 3e472c124fe323ca10ea8230b2bb20c3d0a857db Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Mon, 14 Apr 2025 17:04:43 -0700 Subject: [PATCH 06/31] sync with main and address comments --- .../app/utils/connector_service.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/utils/connector_service.py b/surfsense_backend/app/utils/connector_service.py index e0d632298..fe08572a2 100644 --- a/surfsense_backend/app/utils/connector_service.py +++ b/surfsense_backend/app/utils/connector_service.py @@ -529,8 +529,8 @@ class ConnectorService: document_type="GITHUB_CONNECTOR" ) - # Map github_chunks to the required format - mapped_sources = {} + # Process each chunk and create sources directly without deduplication + sources_list = [] for i, chunk in enumerate(github_chunks): # Fix for UI - assign a unique ID for citation/source tracking github_chunks[i]['document']['id'] = self.source_id_counter @@ -539,7 +539,7 @@ class ConnectorService: document = chunk.get('document', {}) metadata = document.get('metadata', {}) - # Create a mapped source entry + # Create a source entry source = { "id": self.source_id_counter, "title": document.get('title', 'GitHub Document'), # Use specific title if available @@ -548,18 +548,11 @@ class ConnectorService: } self.source_id_counter += 1 - - # Use a unique identifier for tracking unique sources (URL preferred) - source_key = source.get("url") or source.get("title") - if source_key and source_key not in mapped_sources: - mapped_sources[source_key] = source - - # Convert to list of sources - sources_list = list(mapped_sources.values()) + sources_list.append(source) # Create result object result_object = { - "id": 7, # Assuming 7 is the next available ID + "id": 8, "name": "GitHub", "type": "GITHUB_CONNECTOR", "sources": sources_list, From 37f11b152195e7ed23ecdf4abe9a6203bac2566e Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 14 Apr 2025 19:12:39 -0700 Subject: [PATCH 07/31] fix(ui) : Added back Tavily and Updated the researcher icon for GitHub --- .../[search_space_id]/connectors/add/page.tsx | 246 ++++++++++++------ .../components/ModernHeroWithGradients.tsx | 2 +- .../components/chat/ConnectorComponents.tsx | 4 +- 3 files changed, 175 insertions(+), 77 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx index f70bb6209..7329c44de 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx @@ -15,6 +15,7 @@ import { IconBrandZoom, IconChevronRight, IconWorldWww, + IconChevronDown, } from "@tabler/icons-react"; import { motion, AnimatePresence } from "framer-motion"; import { useState } from "react"; @@ -23,6 +24,8 @@ import Link from "next/link"; import { Button } from "@/components/ui/button"; import { Separator } from "@/components/ui/separator"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardFooter, CardHeader } from "@/components/ui/card"; import { useForm } from "react-hook-form"; // Define the Connector type @@ -31,7 +34,7 @@ interface Connector { title: string; description: string; icon: React.ReactNode; - status: "available" | "coming-soon" | "connected"; // Added connected status example + status: "available" | "coming-soon" | "connected"; } interface ConnectorCategory { @@ -47,12 +50,11 @@ const connectorCategories: ConnectorCategory[] = [ title: "Search Engines", connectors: [ { - id: "web-search", - title: "Web Search", - description: "Enable web search capabilities for broader context.", + id: "tavily-api", + title: "Tavily API", + description: "Search the web using the Tavily API", icon: , - status: "available", // Example status - // Potentially add config form here if needed (e.g., choosing provider) + status: "available", }, // Add other search engine connectors like Tavily, Serper if they have UI config ], @@ -94,10 +96,9 @@ const connectorCategories: ConnectorCategory[] = [ description: "Connect to your Notion workspace to access pages and databases.", icon: , status: "available", - // No form here, assumes it links to its own page }, { - id: "github-connector", // Keep the id simple + id: "github-connector", title: "GitHub", description: "Connect a GitHub PAT to index code and docs from accessible repositories.", icon: , @@ -127,6 +128,44 @@ const connectorCategories: ConnectorCategory[] = [ }, ]; +// Animation variants +const fadeIn = { + hidden: { opacity: 0 }, + visible: { opacity: 1, transition: { duration: 0.4 } } +}; + +const staggerContainer = { + hidden: { opacity: 0 }, + visible: { + opacity: 1, + transition: { + staggerChildren: 0.1 + } + } +}; + +const cardVariants = { + hidden: { opacity: 0, y: 20 }, + visible: { + opacity: 1, + y: 0, + transition: { + type: "spring", + stiffness: 260, + damping: 20 + } + }, + hover: { + scale: 1.02, + boxShadow: "0 10px 15px -3px rgba(0, 0, 0, 0.1), 0 4px 6px -2px rgba(0, 0, 0, 0.05)", + transition: { + type: "spring", + stiffness: 400, + damping: 10 + } + } +}; + export default function ConnectorsPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; @@ -141,85 +180,142 @@ export default function ConnectorsPage() { }; return ( -
+
-

Connect Your Tools

-

+

+ Connect Your Tools +

+

Integrate with your favorite services to enhance your research capabilities.

-
+ {connectorCategories.map((category) => ( - toggleCategory(category.id)} - className="space-y-2" + variants={fadeIn} + className="rounded-lg border bg-card text-card-foreground shadow-sm" > -
-

{category.title}

- - {/* Replace with your preferred expand/collapse icon/button */} - - -
- -
- {category.connectors.map((connector) => ( -
-
-
-
- {connector.icon} -

- {connector.title} -

- {connector.status === "coming-soon" && ( - - Coming soon - - )} - {/* TODO: Add 'Connected' badge based on actual state */} -
-

- {connector.description} -

-
-
- {/* Always render Link button if available */} - {connector.status === 'available' && ( -
- - - -
- )} - {connector.status === 'coming-soon' && ( -
- -
- )} - {/* TODO: Add logic for 'connected' status */} -
- ))} + toggleCategory(category.id)} + className="w-full" + > +
+

{category.title}

+ + +
- - - + + + + + {category.connectors.map((connector) => ( + + + +
+ + {connector.icon} + +
+
+
+

{connector.title}

+ {connector.status === "coming-soon" && ( + + Coming soon + + )} + {connector.status === "connected" && ( + + Connected + + )} +
+
+ + + +

+ {connector.description} +

+ + + + {connector.status === 'available' && ( + + + + )} + {connector.status === 'coming-soon' && ( + + )} + {connector.status === 'connected' && ( + + )} + + + + ))} + + + + + ))} -
+
); } diff --git a/surfsense_web/components/ModernHeroWithGradients.tsx b/surfsense_web/components/ModernHeroWithGradients.tsx index c0bde3fe0..bd10a50ca 100644 --- a/surfsense_web/components/ModernHeroWithGradients.tsx +++ b/surfsense_web/components/ModernHeroWithGradients.tsx @@ -36,7 +36,7 @@ export function ModernHeroWithGradients() {

- A Customizable AI Research Agent just like NotebookLM or Perplexity, but connected to external sources such as search engines (Tavily), Slack, Notion, and more. + A Customizable AI Research Agent just like NotebookLM or Perplexity, but connected to external sources such as search engines (Tavily), Slack, Notion, YouTube, GitHub and more.

{ const iconProps = { className: "h-4 w-4" }; switch(connectorType) { + case 'GITHUB_CONNECTOR': + return ; case 'YOUTUBE_VIDEO': return ; case 'CRAWLED_URL': From 61a9ceb7a0a193e61874b135ee9b7b38fd1d076a Mon Sep 17 00:00:00 2001 From: Rohan Verma <122026167+MODSetter@users.noreply.github.com> Date: Mon, 14 Apr 2025 19:26:23 -0700 Subject: [PATCH 08/31] Update README.md --- README.md | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c890b2de3..028ab03bd 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ -![headnew](https://github.com/user-attachments/assets/a44fd1e7-1861-46d0-aff7-19cf33e86baa) + +![new_header](https://github.com/user-attachments/assets/e236b764-0ddc-42ff-a1f1-8fbb3d2e0e65) # SurfSense -While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as search engines (Tavily), Slack, Notion, and more to come. +While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic/query, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as search engines (Tavily), Slack, Notion, YouTube, GitHub and more to come. # Video @@ -45,6 +46,7 @@ Open source and easy to deploy locally. - Slack - Notion - Youtube Videos +- GitHub - and more to come..... #### 🔖 Cross Browser Extension @@ -150,6 +152,20 @@ For local frontend setup just fill out the `.env` file of frontend. You should see your Next.js frontend running at `localhost:3000` +#### Some FrontEnd Screens + +**Search Spaces** + +![search_spaces](https://github.com/user-attachments/assets/e254c38c-f937-44b6-9e9d-770db583d099) + +**Research Agent** + +![researcher](https://github.com/user-attachments/assets/fda3e61f-f936-4b66-b565-d84edde44a67) + + +**Agent Chat** + +![chat](https://github.com/user-attachments/assets/bb352d52-1c6d-4020-926b-722d0b98b491) --- @@ -193,12 +209,15 @@ After filling in your SurfSense API key you should be able to use extension now. ### **BackEnd** - **FastAPI**: Modern, fast web framework for building APIs with Python - + - **PostgreSQL with pgvector**: Database with vector search capabilities for similarity searches - **SQLAlchemy**: SQL toolkit and ORM (Object-Relational Mapping) for database interactions +- **Alembic**: A database migrations tool for SQLAlchemy. + - **FastAPI Users**: Authentication and user management with JWT and OAuth support + - **LangChain**: Framework for developing AI-powered applications - **GPT Integration**: Integration with LLM models through LiteLLM @@ -214,10 +233,8 @@ After filling in your SurfSense API key you should be able to use extension now. - **pgvector**: PostgreSQL extension for efficient vector similarity operations - **Chonkie**: Advanced document chunking and embedding library - -- Uses `AutoEmbeddings` for flexible embedding model selection - -- `LateChunker` for optimized document chunking based on embedding model's max sequence length + - Uses `AutoEmbeddings` for flexible embedding model selection + - `LateChunker` for optimized document chunking based on embedding model's max sequence length From dcf3bf02cdcdeed20103c9dac3bac1aa49717bb1 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 14 Apr 2025 19:32:00 -0700 Subject: [PATCH 09/31] refactor(connectors): streamline imports --- .../[search_space_id]/connectors/add/page.tsx | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx index 7329c44de..5eeaae5a8 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx @@ -1,32 +1,24 @@ "use client"; -import { cn } from "@/lib/utils"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardFooter, CardHeader } from "@/components/ui/card"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { - IconBrandGoogle, - IconBrandSlack, - IconBrandWindows, IconBrandDiscord, - IconSearch, - IconMessages, - IconDatabase, - IconCloud, IconBrandGithub, IconBrandNotion, - IconMail, + IconBrandSlack, + IconBrandWindows, IconBrandZoom, - IconChevronRight, - IconWorldWww, IconChevronDown, + IconChevronRight, + IconMail, + IconWorldWww, } from "@tabler/icons-react"; -import { motion, AnimatePresence } from "framer-motion"; -import { useState } from "react"; -import { useParams } from "next/navigation"; +import { AnimatePresence, motion } from "framer-motion"; import Link from "next/link"; -import { Button } from "@/components/ui/button"; -import { Separator } from "@/components/ui/separator"; -import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; -import { Badge } from "@/components/ui/badge"; -import { Card, CardContent, CardFooter, CardHeader } from "@/components/ui/card"; -import { useForm } from "react-hook-form"; +import { useParams } from "next/navigation"; +import { useState } from "react"; // Define the Connector type interface Connector { From e34b444bfe08c888d5a19a59e0a2b4b9d3cb0bb6 Mon Sep 17 00:00:00 2001 From: Rohan Verma <122026167+MODSetter@users.noreply.github.com> Date: Mon, 14 Apr 2025 19:40:04 -0700 Subject: [PATCH 10/31] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 028ab03bd..4ebe65569 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,9 @@ You should see your Next.js frontend running at `localhost:3000` ![search_spaces](https://github.com/user-attachments/assets/e254c38c-f937-44b6-9e9d-770db583d099) +**Manage Documents** +![documents](https://github.com/user-attachments/assets/7001e306-eb06-4009-89c6-8fadfdc3fc4d) + **Research Agent** ![researcher](https://github.com/user-attachments/assets/fda3e61f-f936-4b66-b565-d84edde44a67) From e0eb9d4b8b337d78b859132a16ca700229c0dc11 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 15 Apr 2025 23:10:35 -0700 Subject: [PATCH 11/31] feat: Added Linear Connector --- README.md | 3 +- .../versions/2_add_linear_connector_enum.py | 45 ++ ...3_add_linear_connector_to_documenttype_.py | 71 +++ .../app/connectors/linear_connector.py | 454 ++++++++++++++++++ surfsense_backend/app/db.py | 2 + .../routes/search_source_connectors_routes.py | 77 ++- .../app/schemas/search_source_connector.py | 10 + .../app/tasks/connectors_indexing_tasks.py | 294 +++++++++++- .../tasks/stream_connector_search_results.py | 26 + .../app/utils/connector_service.py | 84 ++++ .../connectors/(manage)/page.tsx | 1 + .../connectors/add/linear-connector/page.tsx | 321 +++++++++++++ .../[search_space_id]/connectors/add/page.tsx | 24 +- .../documents/(manage)/page.tsx | 39 +- .../components/ModernHeroWithGradients.tsx | 2 +- .../components/chat/ConnectorComponents.tsx | 4 +- 16 files changed, 1419 insertions(+), 38 deletions(-) create mode 100644 surfsense_backend/alembic/versions/2_add_linear_connector_enum.py create mode 100644 surfsense_backend/alembic/versions/3_add_linear_connector_to_documenttype_.py create mode 100644 surfsense_backend/app/connectors/linear_connector.py create mode 100644 surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx diff --git a/README.md b/README.md index 4ebe65569..4b9ea5a64 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ # SurfSense -While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic/query, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as search engines (Tavily), Slack, Notion, YouTube, GitHub and more to come. +While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic/query, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as search engines (Tavily), Slack, Linear, Notion, YouTube, GitHub and more to come. # Video @@ -44,6 +44,7 @@ Open source and easy to deploy locally. #### ℹ️ **External Sources** - Search Engines (Tavily) - Slack +- Linear - Notion - Youtube Videos - GitHub diff --git a/surfsense_backend/alembic/versions/2_add_linear_connector_enum.py b/surfsense_backend/alembic/versions/2_add_linear_connector_enum.py new file mode 100644 index 000000000..d3527d34a --- /dev/null +++ b/surfsense_backend/alembic/versions/2_add_linear_connector_enum.py @@ -0,0 +1,45 @@ +"""Add LINEAR_CONNECTOR to SearchSourceConnectorType enum + +Revision ID: 2 +Revises: e55302644c51 +Create Date: 2025-04-16 10:00:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '2' +down_revision: Union[str, None] = 'e55302644c51' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + # Manually add the command to add the enum value + op.execute("ALTER TYPE searchsourceconnectortype ADD VALUE 'LINEAR_CONNECTOR'") + + # Pass for the rest, as autogenerate didn't run to add other schema details + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + # Downgrading removal of an enum value requires recreating the type + op.execute("ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old") + op.execute("CREATE TYPE searchsourceconnectortype AS ENUM('SERPER_API', 'TAVILY_API', 'SLACK_CONNECTOR', 'NOTION_CONNECTOR', 'GITHUB_CONNECTOR')") + op.execute(( + "ALTER TABLE search_source_connectors ALTER COLUMN connector_type TYPE searchsourceconnectortype USING " + "connector_type::text::searchsourceconnectortype" + )) + op.execute("DROP TYPE searchsourceconnectortype_old") + + pass + # ### end Alembic commands ### \ No newline at end of file diff --git a/surfsense_backend/alembic/versions/3_add_linear_connector_to_documenttype_.py b/surfsense_backend/alembic/versions/3_add_linear_connector_to_documenttype_.py new file mode 100644 index 000000000..ab50d8550 --- /dev/null +++ b/surfsense_backend/alembic/versions/3_add_linear_connector_to_documenttype_.py @@ -0,0 +1,71 @@ +"""Add LINEAR_CONNECTOR to DocumentType enum + +Revision ID: 3 +Revises: 2 +Create Date: 2025-04-16 10:05:00.059921 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '3' +down_revision: Union[str, None] = '2' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# Define the ENUM type name and the new value +ENUM_NAME = 'documenttype' # Make sure this matches the name in your DB (usually lowercase class name) +NEW_VALUE = 'LINEAR_CONNECTOR' + +def upgrade() -> None: + """Upgrade schema.""" + op.execute(f"ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}'") + + +# Warning: This will delete all rows with the new value +def downgrade() -> None: + """Downgrade schema - remove LINEAR_CONNECTOR from enum.""" + + # The old type name + old_enum_name = f"{ENUM_NAME}_old" + + # Enum values *before* LINEAR_CONNECTOR was added + old_values = ( + 'EXTENSION', + 'CRAWLED_URL', + 'FILE', + 'SLACK_CONNECTOR', + 'NOTION_CONNECTOR', + 'YOUTUBE_VIDEO', + 'GITHUB_CONNECTOR' + ) + old_values_sql = ", ".join([f"'{v}'" for v in old_values]) + + # Table and column names (adjust if different) + table_name = 'documents' + column_name = 'document_type' + + # 1. Rename the current enum type + op.execute(f"ALTER TYPE {ENUM_NAME} RENAME TO {old_enum_name}") + + # 2. Create the new enum type with the old values + op.execute(f"CREATE TYPE {ENUM_NAME} AS ENUM({old_values_sql})") + + # 3. Update the table: + op.execute( + f"DELETE FROM {table_name} WHERE {column_name}::text = '{NEW_VALUE}'" + ) + + # 4. Alter the column to use the new enum type (casting old values) + op.execute( + f"ALTER TABLE {table_name} ALTER COLUMN {column_name} " + f"TYPE {ENUM_NAME} USING {column_name}::text::{ENUM_NAME}" + ) + + # 5. Drop the old enum type + op.execute(f"DROP TYPE {old_enum_name}") + # ### end Alembic commands ### \ No newline at end of file diff --git a/surfsense_backend/app/connectors/linear_connector.py b/surfsense_backend/app/connectors/linear_connector.py new file mode 100644 index 000000000..be9a1a49d --- /dev/null +++ b/surfsense_backend/app/connectors/linear_connector.py @@ -0,0 +1,454 @@ +""" +Linear Connector Module + +A module for retrieving issues and comments from Linear. +Allows fetching issue lists and their comments with date range filtering. +""" + +import requests +from datetime import datetime, timedelta +from typing import Dict, List, Optional, Tuple, Any, Union + + +class LinearConnector: + """Class for retrieving issues and comments from Linear.""" + + def __init__(self, token: str = None): + """ + Initialize the LinearConnector class. + + Args: + token: Linear API token (optional, can be set later with set_token) + """ + self.token = token + self.api_url = "https://api.linear.app/graphql" + + def set_token(self, token: str) -> None: + """ + Set the Linear API token. + + Args: + token: Linear API token + """ + self.token = token + + def get_headers(self) -> Dict[str, str]: + """ + Get headers for Linear API requests. + + Returns: + Dictionary of headers + + Raises: + ValueError: If no Linear token has been set + """ + if not self.token: + raise ValueError("Linear token not initialized. Call set_token() first.") + + return { + 'Content-Type': 'application/json', + 'Authorization': self.token + } + + def execute_graphql_query(self, query: str, variables: Dict[str, Any] = None) -> Dict[str, Any]: + """ + Execute a GraphQL query against the Linear API. + + Args: + query: GraphQL query string + variables: Variables for the GraphQL query (optional) + + Returns: + Response data from the API + + Raises: + ValueError: If no Linear token has been set + Exception: If the API request fails + """ + if not self.token: + raise ValueError("Linear token not initialized. Call set_token() first.") + + headers = self.get_headers() + payload = {'query': query} + + if variables: + payload['variables'] = variables + + response = requests.post( + self.api_url, + headers=headers, + json=payload + ) + + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Query failed with status code {response.status_code}: {response.text}") + + def get_all_issues(self, include_comments: bool = True) -> List[Dict[str, Any]]: + """ + Fetch all issues from Linear. + + Args: + include_comments: Whether to include comments in the response + + Returns: + List of issue objects + + Raises: + ValueError: If no Linear token has been set + Exception: If the API request fails + """ + comments_query = "" + if include_comments: + comments_query = """ + comments { + nodes { + id + body + user { + id + name + email + } + createdAt + updatedAt + } + } + """ + + query = f""" + query {{ + issues {{ + nodes {{ + id + identifier + title + description + state {{ + id + name + type + }} + assignee {{ + id + name + email + }} + creator {{ + id + name + email + }} + createdAt + updatedAt + {comments_query} + }} + }} + }} + """ + + result = self.execute_graphql_query(query) + + # Extract issues from the response + if "data" in result and "issues" in result["data"] and "nodes" in result["data"]["issues"]: + return result["data"]["issues"]["nodes"] + + return [] + + def get_issues_by_date_range( + self, + start_date: str, + end_date: str, + include_comments: bool = True + ) -> Tuple[List[Dict[str, Any]], Optional[str]]: + """ + Fetch issues within a date range. + + Args: + start_date: Start date in YYYY-MM-DD format + end_date: End date in YYYY-MM-DD format (inclusive) + include_comments: Whether to include comments in the response + + Returns: + Tuple containing (issues list, error message or None) + """ + # Convert date strings to ISO format + try: + # For Linear API: we need to use a more specific format for the filter + # Instead of DateTime, use a string in the filter for DateTimeOrDuration + comments_query = "" + if include_comments: + comments_query = """ + comments { + nodes { + id + body + user { + id + name + email + } + createdAt + updatedAt + } + } + """ + + # Query issues that were either created OR updated within the date range + # This ensures we catch both new issues and updated existing issues + query = f""" + query IssuesByDateRange($after: String) {{ + issues( + first: 100, + after: $after, + filter: {{ + or: [ + {{ + createdAt: {{ + gte: "{start_date}T00:00:00Z" + lte: "{end_date}T23:59:59Z" + }} + }}, + {{ + updatedAt: {{ + gte: "{start_date}T00:00:00Z" + lte: "{end_date}T23:59:59Z" + }} + }} + ] + }} + ) {{ + nodes {{ + id + identifier + title + description + state {{ + id + name + type + }} + assignee {{ + id + name + email + }} + creator {{ + id + name + email + }} + createdAt + updatedAt + {comments_query} + }} + pageInfo {{ + hasNextPage + endCursor + }} + }} + }} + """ + + try: + all_issues = [] + has_next_page = True + cursor = None + + # Handle pagination to get all issues + while has_next_page: + variables = {"after": cursor} if cursor else {} + result = self.execute_graphql_query(query, variables) + + # Check for errors + if "errors" in result: + error_message = "; ".join([error.get("message", "Unknown error") for error in result["errors"]]) + return [], f"GraphQL errors: {error_message}" + + # Extract issues from the response + if "data" in result and "issues" in result["data"]: + issues_page = result["data"]["issues"] + + # Add issues from this page + if "nodes" in issues_page: + all_issues.extend(issues_page["nodes"]) + + # Check if there are more pages + if "pageInfo" in issues_page: + page_info = issues_page["pageInfo"] + has_next_page = page_info.get("hasNextPage", False) + cursor = page_info.get("endCursor") if has_next_page else None + else: + has_next_page = False + else: + has_next_page = False + + if not all_issues: + return [], "No issues found in the specified date range." + + return all_issues, None + + except Exception as e: + return [], f"Error fetching issues: {str(e)}" + + except ValueError as e: + return [], f"Invalid date format: {str(e)}. Please use YYYY-MM-DD." + + def format_issue(self, issue: Dict[str, Any]) -> Dict[str, Any]: + """ + Format an issue for easier consumption. + + Args: + issue: The issue object from Linear API + + Returns: + Formatted issue dictionary + """ + # Extract basic issue details + formatted = { + "id": issue.get("id", ""), + "identifier": issue.get("identifier", ""), + "title": issue.get("title", ""), + "description": issue.get("description", ""), + "state": issue.get("state", {}).get("name", "Unknown") if issue.get("state") else "Unknown", + "state_type": issue.get("state", {}).get("type", "Unknown") if issue.get("state") else "Unknown", + "created_at": issue.get("createdAt", ""), + "updated_at": issue.get("updatedAt", ""), + "creator": { + "id": issue.get("creator", {}).get("id", "") if issue.get("creator") else "", + "name": issue.get("creator", {}).get("name", "Unknown") if issue.get("creator") else "Unknown", + "email": issue.get("creator", {}).get("email", "") if issue.get("creator") else "" + } if issue.get("creator") else {"id": "", "name": "Unknown", "email": ""}, + "assignee": { + "id": issue.get("assignee", {}).get("id", ""), + "name": issue.get("assignee", {}).get("name", "Unknown"), + "email": issue.get("assignee", {}).get("email", "") + } if issue.get("assignee") else None, + "comments": [] + } + + # Extract comments if available + if "comments" in issue and "nodes" in issue["comments"]: + for comment in issue["comments"]["nodes"]: + formatted_comment = { + "id": comment.get("id", ""), + "body": comment.get("body", ""), + "created_at": comment.get("createdAt", ""), + "updated_at": comment.get("updatedAt", ""), + "user": { + "id": comment.get("user", {}).get("id", "") if comment.get("user") else "", + "name": comment.get("user", {}).get("name", "Unknown") if comment.get("user") else "Unknown", + "email": comment.get("user", {}).get("email", "") if comment.get("user") else "" + } if comment.get("user") else {"id": "", "name": "Unknown", "email": ""} + } + formatted["comments"].append(formatted_comment) + + return formatted + + def format_issue_to_markdown(self, issue: Dict[str, Any]) -> str: + """ + Convert an issue to markdown format. + + Args: + issue: The issue object (either raw or formatted) + + Returns: + Markdown string representation of the issue + """ + # Format the issue if it's not already formatted + if "identifier" not in issue: + issue = self.format_issue(issue) + + # Build the markdown content + markdown = f"# {issue.get('identifier', 'No ID')}: {issue.get('title', 'No Title')}\n\n" + + if issue.get('state'): + markdown += f"**Status:** {issue['state']}\n\n" + + if issue.get('assignee') and issue['assignee'].get('name'): + markdown += f"**Assignee:** {issue['assignee']['name']}\n" + + if issue.get('creator') and issue['creator'].get('name'): + markdown += f"**Created by:** {issue['creator']['name']}\n" + + if issue.get('created_at'): + created_date = self.format_date(issue['created_at']) + markdown += f"**Created:** {created_date}\n" + + if issue.get('updated_at'): + updated_date = self.format_date(issue['updated_at']) + markdown += f"**Updated:** {updated_date}\n\n" + + if issue.get('description'): + markdown += f"## Description\n\n{issue['description']}\n\n" + + if issue.get('comments'): + markdown += f"## Comments ({len(issue['comments'])})\n\n" + + for comment in issue['comments']: + user_name = "Unknown" + if comment.get('user') and comment['user'].get('name'): + user_name = comment['user']['name'] + + comment_date = "Unknown date" + if comment.get('created_at'): + comment_date = self.format_date(comment['created_at']) + + markdown += f"### {user_name} ({comment_date})\n\n{comment.get('body', '')}\n\n---\n\n" + + return markdown + + @staticmethod + def format_date(iso_date: str) -> str: + """ + Format an ISO date string to a more readable format. + + Args: + iso_date: ISO format date string + + Returns: + Formatted date string + """ + if not iso_date or not isinstance(iso_date, str): + return "Unknown date" + + try: + dt = datetime.fromisoformat(iso_date.replace('Z', '+00:00')) + return dt.strftime('%Y-%m-%d %H:%M:%S') + except ValueError: + return iso_date + + +# Example usage (uncomment to use): +""" +if __name__ == "__main__": + # Set your token here + token = "YOUR_LINEAR_API_KEY" + + linear = LinearConnector(token) + + try: + # Get all issues with comments + issues = linear.get_all_issues() + print(f"Retrieved {len(issues)} issues") + + # Format and print the first issue as markdown + if issues: + issue_md = linear.format_issue_to_markdown(issues[0]) + print("\nSample Issue in Markdown:\n") + print(issue_md) + + # Get issues by date range + start_date = "2023-01-01" + end_date = "2023-01-31" + date_issues, error = linear.get_issues_by_date_range(start_date, end_date) + + if error: + print(f"Error: {error}") + else: + print(f"\nRetrieved {len(date_issues)} issues from {start_date} to {end_date}") + + except Exception as e: + print(f"Error: {e}") +""" diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 25b7bfbb4..4426f4ffa 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -41,6 +41,7 @@ class DocumentType(str, Enum): NOTION_CONNECTOR = "NOTION_CONNECTOR" YOUTUBE_VIDEO = "YOUTUBE_VIDEO" GITHUB_CONNECTOR = "GITHUB_CONNECTOR" + LINEAR_CONNECTOR = "LINEAR_CONNECTOR" class SearchSourceConnectorType(str, Enum): SERPER_API = "SERPER_API" @@ -48,6 +49,7 @@ class SearchSourceConnectorType(str, Enum): SLACK_CONNECTOR = "SLACK_CONNECTOR" NOTION_CONNECTOR = "NOTION_CONNECTOR" GITHUB_CONNECTOR = "GITHUB_CONNECTOR" + LINEAR_CONNECTOR = "LINEAR_CONNECTOR" class ChatType(str, Enum): GENERAL = "GENERAL" diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 482a8259d..8a003ff55 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -7,7 +7,7 @@ PUT /search-source-connectors/{connector_id} - Update a specific connector DELETE /search-source-connectors/{connector_id} - Delete a specific connector POST /search-source-connectors/{connector_id}/index - Index content from a connector to a search space -Note: Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, NOTION_CONNECTOR). +Note: Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, NOTION_CONNECTOR, GITHUB_CONNECTOR, LINEAR_CONNECTOR). """ from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks from sqlalchemy.ext.asyncio import AsyncSession @@ -19,8 +19,8 @@ from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate from app.users import current_active_user from app.utils.check_ownership import check_ownership from pydantic import ValidationError -from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos -from datetime import datetime, timezone +from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos, index_linear_issues +from datetime import datetime, timezone, timedelta import logging # Set up logging @@ -37,7 +37,7 @@ async def create_search_source_connector( """ Create a new search source connector. - Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR). + Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, etc.). The config must contain the appropriate keys for the connector type. """ try: @@ -131,7 +131,7 @@ async def update_search_source_connector( """ Update a search source connector. - Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR). + Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, etc.). The config must contain the appropriate keys for the connector type. """ try: @@ -216,10 +216,10 @@ async def index_connector_content( Index content from a connector to a search space. Currently supports: - - SLACK_CONNECTOR: Indexes messages from all accessible Slack channels since the last indexing - (or the last 365 days if never indexed before) - - NOTION_CONNECTOR: Indexes pages from all accessible Notion pages since the last indexing - (or the last 365 days if never indexed before) + - SLACK_CONNECTOR: Indexes messages from all accessible Slack channels + - NOTION_CONNECTOR: Indexes pages from all accessible Notion pages + - GITHUB_CONNECTOR: Indexes code and documentation from GitHub repositories + - LINEAR_CONNECTOR: Indexes issues and comments from Linear Args: connector_id: ID of the connector to use @@ -251,7 +251,7 @@ async def index_connector_content( today = datetime.now().date() if connector.last_indexed_at.date() == today: # If last indexed today, go back 1 day to ensure we don't miss anything - start_date = (today - datetime.timedelta(days=1)).strftime("%Y-%m-%d") + start_date = (today - timedelta(days=1)).strftime("%Y-%m-%d") else: start_date = connector.last_indexed_at.strftime("%Y-%m-%d") @@ -272,7 +272,7 @@ async def index_connector_content( today = datetime.now().date() if connector.last_indexed_at.date() == today: # If last indexed today, go back 1 day to ensure we don't miss anything - start_date = (today - datetime.timedelta(days=1)).strftime("%Y-%m-%d") + start_date = (today - timedelta(days=1)).strftime("%Y-%m-%d") else: start_date = connector.last_indexed_at.strftime("%Y-%m-%d") @@ -294,6 +294,27 @@ async def index_connector_content( logger.info(f"Triggering GitHub indexing for connector {connector_id} into search space {search_space_id}") background_tasks.add_task(run_github_indexing_with_new_session, connector_id, search_space_id) response_message = "GitHub indexing started in the background." + + elif connector.connector_type == SearchSourceConnectorType.LINEAR_CONNECTOR: + # Determine the time range that will be indexed + if not connector.last_indexed_at: + start_date = "365 days ago" + else: + # Check if last_indexed_at is today + today = datetime.now().date() + if connector.last_indexed_at.date() == today: + # If last indexed today, go back 1 day to ensure we don't miss anything + start_date = (today - timedelta(days=1)).strftime("%Y-%m-%d") + else: + start_date = connector.last_indexed_at.strftime("%Y-%m-%d") + + indexing_from = start_date + indexing_to = today_str + + # Run indexing in background + logger.info(f"Triggering Linear indexing for connector {connector_id} into search space {search_space_id}") + background_tasks.add_task(run_linear_indexing_with_new_session, connector_id, search_space_id) + response_message = "Linear indexing started in the background." else: raise HTTPException( @@ -460,3 +481,37 @@ async def run_github_indexing( await session.rollback() logger.error(f"Critical error in run_github_indexing for connector {connector_id}: {e}", exc_info=True) # Optionally update status in DB to indicate failure + +# Add new helper functions for Linear indexing +async def run_linear_indexing_with_new_session( + connector_id: int, + search_space_id: int +): + """Wrapper to run Linear indexing with its own database session.""" + logger.info(f"Background task started: Indexing Linear connector {connector_id} into space {search_space_id}") + async with async_session_maker() as session: + await run_linear_indexing(session, connector_id, search_space_id) + logger.info(f"Background task finished: Indexing Linear connector {connector_id}") + +async def run_linear_indexing( + session: AsyncSession, + connector_id: int, + search_space_id: int +): + """Runs the Linear indexing task and updates the timestamp.""" + try: + indexed_count, error_message = await index_linear_issues( + session, connector_id, search_space_id, update_last_indexed=False + ) + if error_message: + logger.error(f"Linear indexing failed for connector {connector_id}: {error_message}") + # Optionally update status in DB to indicate failure + else: + logger.info(f"Linear indexing successful for connector {connector_id}. Indexed {indexed_count} documents.") + # Update the last indexed timestamp only on success + await update_connector_last_indexed(session, connector_id) + await session.commit() # Commit timestamp update + except Exception as e: + await session.rollback() + logger.error(f"Critical error in run_linear_indexing for connector {connector_id}: {e}", exc_info=True) + # Optionally update status in DB to indicate failure diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index 41e1086e7..032e48698 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -67,6 +67,16 @@ class SearchSourceConnectorBase(BaseModel): # Ensure the token is not empty if not config.get("GITHUB_PAT"): raise ValueError("GITHUB_PAT cannot be empty") + + elif connector_type == SearchSourceConnectorType.LINEAR_CONNECTOR: + # For LINEAR_CONNECTOR, only allow LINEAR_API_KEY + allowed_keys = ["LINEAR_API_KEY"] + if set(config.keys()) != set(allowed_keys): + raise ValueError(f"For LINEAR_CONNECTOR connector type, config must only contain these keys: {allowed_keys}") + + # Ensure the token is not empty + if not config.get("LINEAR_API_KEY"): + raise ValueError("LINEAR_API_KEY cannot be empty") return config diff --git a/surfsense_backend/app/tasks/connectors_indexing_tasks.py b/surfsense_backend/app/tasks/connectors_indexing_tasks.py index 670fa26ad..dbdb24c80 100644 --- a/surfsense_backend/app/tasks/connectors_indexing_tasks.py +++ b/surfsense_backend/app/tasks/connectors_indexing_tasks.py @@ -10,6 +10,7 @@ from app.prompts import SUMMARY_PROMPT_TEMPLATE from app.connectors.slack_history import SlackHistory from app.connectors.notion_history import NotionHistoryConnector from app.connectors.github_connector import GitHubConnector +from app.connectors.linear_connector import LinearConnector from slack_sdk.errors import SlackApiError import logging @@ -60,8 +61,20 @@ async def index_slack_messages( end_date = datetime.now() # Use last_indexed_at as start date if available, otherwise use 365 days ago - - start_date = end_date - timedelta(days=365) + if connector.last_indexed_at: + # Convert dates to be comparable (both timezone-naive) + last_indexed_naive = connector.last_indexed_at.replace(tzinfo=None) if connector.last_indexed_at.tzinfo else connector.last_indexed_at + + # Check if last_indexed_at is in the future or after end_date + if last_indexed_naive > end_date: + logger.warning(f"Last indexed date ({last_indexed_naive.strftime('%Y-%m-%d')}) is in the future. Using 30 days ago instead.") + start_date = end_date - timedelta(days=30) + else: + start_date = last_indexed_naive + logger.info(f"Using last_indexed_at ({start_date.strftime('%Y-%m-%d')}) as start date") + else: + start_date = end_date - timedelta(days=30) # Use 30 days instead of 365 to catch recent issues + logger.info(f"No last_indexed_at found, using {start_date.strftime('%Y-%m-%d')} (30 days ago) as start date") # Format dates for Slack API start_date_str = start_date.strftime("%Y-%m-%d") @@ -782,3 +795,280 @@ async def index_github_repos( error_message = "; ".join(errors) if errors else None return documents_processed, error_message + +async def index_linear_issues( + session: AsyncSession, + connector_id: int, + search_space_id: int, + update_last_indexed: bool = True +) -> Tuple[int, Optional[str]]: + """ + Index Linear issues and comments. + + Args: + session: Database session + connector_id: ID of the Linear connector + search_space_id: ID of the search space to store documents in + update_last_indexed: Whether to update the last_indexed_at timestamp (default: True) + + Returns: + Tuple containing (number of documents indexed, error message or None) + """ + try: + # Get the connector + result = await session.execute( + select(SearchSourceConnector) + .filter( + SearchSourceConnector.id == connector_id, + SearchSourceConnector.connector_type == SearchSourceConnectorType.LINEAR_CONNECTOR + ) + ) + connector = result.scalars().first() + + if not connector: + return 0, f"Connector with ID {connector_id} not found or is not a Linear connector" + + # Get the Linear token from the connector config + linear_token = connector.config.get("LINEAR_API_KEY") + if not linear_token: + return 0, "Linear API token not found in connector config" + + # Initialize Linear client + linear_client = LinearConnector(token=linear_token) + + # Calculate date range + end_date = datetime.now() + + # Use last_indexed_at as start date if available, otherwise use 365 days ago + if connector.last_indexed_at: + # Convert dates to be comparable (both timezone-naive) + last_indexed_naive = connector.last_indexed_at.replace(tzinfo=None) if connector.last_indexed_at.tzinfo else connector.last_indexed_at + + # Check if last_indexed_at is in the future or after end_date + if last_indexed_naive > end_date: + logger.warning(f"Last indexed date ({last_indexed_naive.strftime('%Y-%m-%d')}) is in the future. Using 30 days ago instead.") + start_date = end_date - timedelta(days=30) + else: + start_date = last_indexed_naive + logger.info(f"Using last_indexed_at ({start_date.strftime('%Y-%m-%d')}) as start date") + else: + start_date = end_date - timedelta(days=30) # Use 30 days instead of 365 to catch recent issues + logger.info(f"No last_indexed_at found, using {start_date.strftime('%Y-%m-%d')} (30 days ago) as start date") + + # Format dates for Linear API + start_date_str = start_date.strftime("%Y-%m-%d") + end_date_str = end_date.strftime("%Y-%m-%d") + + logger.info(f"Fetching Linear issues from {start_date_str} to {end_date_str}") + + # Get issues within date range + try: + issues, error = linear_client.get_issues_by_date_range( + start_date=start_date_str, + end_date=end_date_str, + include_comments=True + ) + + if error: + logger.error(f"Failed to get Linear issues: {error}") + + # Don't treat "No issues found" as an error that should stop indexing + if "No issues found" in error: + logger.info("No issues found is not a critical error, continuing with update") + if update_last_indexed: + connector.last_indexed_at = datetime.now() + await session.commit() + logger.info(f"Updated last_indexed_at to {connector.last_indexed_at} despite no issues found") + return 0, None + else: + return 0, f"Failed to get Linear issues: {error}" + + logger.info(f"Retrieved {len(issues)} issues from Linear API") + + except Exception as e: + logger.error(f"Exception when calling Linear API: {str(e)}", exc_info=True) + return 0, f"Failed to get Linear issues: {str(e)}" + + if not issues: + logger.info("No Linear issues found for the specified date range") + if update_last_indexed: + connector.last_indexed_at = datetime.now() + await session.commit() + logger.info(f"Updated last_indexed_at to {connector.last_indexed_at} despite no issues found") + return 0, None # Return None instead of error message when no issues found + + # Log issue IDs and titles for debugging + logger.info("Issues retrieved from Linear API:") + for idx, issue in enumerate(issues[:10]): # Log first 10 issues + logger.info(f" {idx+1}. {issue.get('identifier', 'Unknown')} - {issue.get('title', 'Unknown')} - Created: {issue.get('createdAt', 'Unknown')} - Updated: {issue.get('updatedAt', 'Unknown')}") + if len(issues) > 10: + logger.info(f" ...and {len(issues) - 10} more issues") + + # Get existing documents for this search space and connector type to prevent duplicates + existing_docs_result = await session.execute( + select(Document) + .filter( + Document.search_space_id == search_space_id, + Document.document_type == DocumentType.LINEAR_CONNECTOR + ) + ) + existing_docs = existing_docs_result.scalars().all() + + # Create a lookup dictionary of existing documents by issue_id + existing_docs_by_issue_id = {} + for doc in existing_docs: + if "issue_id" in doc.document_metadata: + existing_docs_by_issue_id[doc.document_metadata["issue_id"]] = doc + + logger.info(f"Found {len(existing_docs_by_issue_id)} existing Linear documents in database") + + # Log existing document IDs for debugging + if existing_docs_by_issue_id: + logger.info("Existing Linear document issue IDs in database:") + for idx, (issue_id, doc) in enumerate(list(existing_docs_by_issue_id.items())[:10]): # Log first 10 + logger.info(f" {idx+1}. {issue_id} - {doc.document_metadata.get('issue_identifier', 'Unknown')} - {doc.document_metadata.get('issue_title', 'Unknown')}") + if len(existing_docs_by_issue_id) > 10: + logger.info(f" ...and {len(existing_docs_by_issue_id) - 10} more existing documents") + + # Track the number of documents indexed + documents_indexed = 0 + documents_updated = 0 + documents_skipped = 0 + skipped_issues = [] + + # Process each issue + for issue in issues: + try: + issue_id = issue.get("id") + issue_identifier = issue.get("identifier", "") + issue_title = issue.get("title", "") + + if not issue_id or not issue_title: + logger.warning(f"Skipping issue with missing ID or title: {issue_id or 'Unknown'}") + skipped_issues.append(f"{issue_identifier or 'Unknown'} (missing data)") + documents_skipped += 1 + continue + + # Format the issue first to get well-structured data + formatted_issue = linear_client.format_issue(issue) + + # Convert issue to markdown format + issue_content = linear_client.format_issue_to_markdown(formatted_issue) + + if not issue_content: + logger.warning(f"Skipping issue with no content: {issue_identifier} - {issue_title}") + skipped_issues.append(f"{issue_identifier} (no content)") + documents_skipped += 1 + continue + + # Create a short summary for the embedding + # This avoids using the LLM and just uses the issue data directly + state = formatted_issue.get("state", "Unknown") + description = formatted_issue.get("description", "") + # Truncate description if it's too long for the summary + if description and len(description) > 500: + description = description[:497] + "..." + + # Create a simple summary from the issue data + summary_content = f"Linear Issue {issue_identifier}: {issue_title}\n\nStatus: {state}\n\n" + if description: + summary_content += f"Description: {description}\n\n" + + # Add comment count + comment_count = len(formatted_issue.get("comments", [])) + summary_content += f"Comments: {comment_count}" + + # Generate embedding for the summary + summary_embedding = config.embedding_model_instance.embed(summary_content) + + # Process chunks - using the full issue content with comments + chunks = [ + Chunk(content=chunk.text, embedding=chunk.embedding) + for chunk in config.chunker_instance.chunk(issue_content) + ] + + # Check if this issue already exists in our database + existing_document = existing_docs_by_issue_id.get(issue_id) + + if existing_document: + # Update existing document instead of creating a new one + logger.info(f"Updating existing document for issue {issue_identifier} - {issue_title}") + + # Update document fields + existing_document.title = f"Linear - {issue_identifier}: {issue_title}" + existing_document.document_metadata = { + "issue_id": issue_id, + "issue_identifier": issue_identifier, + "issue_title": issue_title, + "state": state, + "comment_count": comment_count, + "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "last_updated": datetime.now().strftime("%Y-%m-%d %H:%M:%S") + } + existing_document.content = summary_content + existing_document.embedding = summary_embedding + + # Delete existing chunks and add new ones + await session.execute( + delete(Chunk) + .where(Chunk.document_id == existing_document.id) + ) + + # Assign new chunks to existing document + for chunk in chunks: + chunk.document_id = existing_document.id + session.add(chunk) + + documents_updated += 1 + else: + # Create and store new document + logger.info(f"Creating new document for issue {issue_identifier} - {issue_title}") + document = Document( + search_space_id=search_space_id, + title=f"Linear - {issue_identifier}: {issue_title}", + document_type=DocumentType.LINEAR_CONNECTOR, + document_metadata={ + "issue_id": issue_id, + "issue_identifier": issue_identifier, + "issue_title": issue_title, + "state": state, + "comment_count": comment_count, + "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S") + }, + content=summary_content, + embedding=summary_embedding, + chunks=chunks + ) + + session.add(document) + documents_indexed += 1 + logger.info(f"Successfully indexed new issue {issue_identifier} - {issue_title}") + + except Exception as e: + logger.error(f"Error processing issue {issue.get('identifier', 'Unknown')}: {str(e)}", exc_info=True) + skipped_issues.append(f"{issue.get('identifier', 'Unknown')} (processing error)") + documents_skipped += 1 + continue # Skip this issue and continue with others + + # Update the last_indexed_at timestamp for the connector only if requested + total_processed = documents_indexed + documents_updated + if update_last_indexed: + connector.last_indexed_at = datetime.now() + logger.info(f"Updated last_indexed_at to {connector.last_indexed_at}") + + # Commit all changes + await session.commit() + logger.info(f"Successfully committed all Linear document changes to database") + + + logger.info(f"Linear indexing completed: {documents_indexed} new issues, {documents_updated} updated, {documents_skipped} skipped") + return total_processed, None # Return None as the error message to indicate success + + except SQLAlchemyError as db_error: + await session.rollback() + logger.error(f"Database error: {str(db_error)}", exc_info=True) + return 0, f"Database error: {str(db_error)}" + except Exception as e: + await session.rollback() + logger.error(f"Failed to index Linear issues: {str(e)}", exc_info=True) + return 0, f"Failed to index Linear issues: {str(e)}" diff --git a/surfsense_backend/app/tasks/stream_connector_search_results.py b/surfsense_backend/app/tasks/stream_connector_search_results.py index fbefaedb7..a1dc0a3e2 100644 --- a/surfsense_backend/app/tasks/stream_connector_search_results.py +++ b/surfsense_backend/app/tasks/stream_connector_search_results.py @@ -270,6 +270,32 @@ async def stream_connector_search_results( # Add documents to collection all_raw_documents.extend(github_chunks) + # Linear Connector + if connector == "LINEAR_CONNECTOR": + # Send terminal message about starting search + yield streaming_service.add_terminal_message("Starting to search for Linear issues...") + + # Search using Linear API with reformulated query + result_object, linear_chunks = await connector_service.search_linear( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=TOP_K + ) + + # Send terminal message about search results + yield streaming_service.add_terminal_message( + f"Found {len(result_object['sources'])} relevant results from Linear", + "success" + ) + + # Update sources + all_sources.append(result_object) + yield streaming_service.update_sources(all_sources) + + # Add documents to collection + all_raw_documents.extend(linear_chunks) + diff --git a/surfsense_backend/app/utils/connector_service.py b/surfsense_backend/app/utils/connector_service.py index fe08572a2..9a6e13c43 100644 --- a/surfsense_backend/app/utils/connector_service.py +++ b/surfsense_backend/app/utils/connector_service.py @@ -559,3 +559,87 @@ class ConnectorService: } return result_object, github_chunks + + async def search_linear(self, user_query: str, user_id: str, search_space_id: int, top_k: int = 20) -> tuple: + """ + Search for Linear issues and comments and return both the source information and langchain documents + + Args: + user_query: The user's query + user_id: The user's ID + search_space_id: The search space ID to search in + top_k: Maximum number of results to return + + Returns: + tuple: (sources_info, langchain_documents) + """ + linear_chunks = await self.retriever.hybrid_search( + query_text=user_query, + top_k=top_k, + user_id=user_id, + search_space_id=search_space_id, + document_type="LINEAR_CONNECTOR" + ) + + # Process each chunk and create sources directly without deduplication + sources_list = [] + for i, chunk in enumerate(linear_chunks): + # Fix for UI + linear_chunks[i]['document']['id'] = self.source_id_counter + + # Extract document metadata + document = chunk.get('document', {}) + metadata = document.get('metadata', {}) + + # Extract Linear-specific metadata + issue_identifier = metadata.get('issue_identifier', '') + issue_title = metadata.get('issue_title', 'Untitled Issue') + issue_state = metadata.get('state', '') + comment_count = metadata.get('comment_count', 0) + + # Create a more descriptive title for Linear issues + title = f"Linear: {issue_identifier} - {issue_title}" + if issue_state: + title += f" ({issue_state})" + + # Create a more descriptive description for Linear issues + description = chunk.get('content', '')[:100] + if len(description) == 100: + description += "..." + + # Add comment count info to description + if comment_count: + if description: + description += f" | Comments: {comment_count}" + else: + description = f"Comments: {comment_count}" + + # For URL, we could construct a URL to the Linear issue if we have the workspace info + # For now, use a generic placeholder + url = "" + if issue_identifier: + # This is a generic format, may need to be adjusted based on actual Linear workspace + url = f"https://linear.app/issue/{issue_identifier}" + + source = { + "id": self.source_id_counter, + "title": title, + "description": description, + "url": url, + "issue_identifier": issue_identifier, + "state": issue_state, + "comment_count": comment_count + } + + self.source_id_counter += 1 + sources_list.append(source) + + # Create result object + result_object = { + "id": 9, # Assign a unique ID for the Linear connector + "name": "Linear Issues", + "type": "LINEAR_CONNECTOR", + "sources": sources_list, + } + + return result_object, linear_chunks diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index 817ca584d..8bfec6326 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -45,6 +45,7 @@ const getConnectorTypeDisplay = (type: string): string => { "SLACK_CONNECTOR": "Slack", "NOTION_CONNECTOR": "Notion", "GITHUB_CONNECTOR": "GitHub", + "LINEAR_CONNECTOR": "Linear", // Add other connector types here as needed }; return typeMap[type] || type; diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx new file mode 100644 index 000000000..a594ed31e --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx @@ -0,0 +1,321 @@ +"use client"; + +import { useState } from "react"; +import { useRouter, useParams } from "next/navigation"; +import { motion } from "framer-motion"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useForm } from "react-hook-form"; +import * as z from "zod"; +import { toast } from "sonner"; +import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; + +import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + Alert, + AlertDescription, + AlertTitle, +} from "@/components/ui/alert"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@/components/ui/accordion"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; + +// Define the form schema with Zod +const linearConnectorFormSchema = z.object({ + name: z.string().min(3, { + message: "Connector name must be at least 3 characters.", + }), + api_key: z.string().min(10, { + message: "Linear API Key is required and must be valid.", + }).regex(/^lin_api_/, { + message: "Linear API Key should start with 'lin_api_'", + }), +}); + +// Define the type for the form values +type LinearConnectorFormValues = z.infer; + +export default function LinearConnectorPage() { + const router = useRouter(); + const params = useParams(); + const searchSpaceId = params.search_space_id as string; + const [isSubmitting, setIsSubmitting] = useState(false); + const { createConnector } = useSearchSourceConnectors(); + + // Initialize the form + const form = useForm({ + resolver: zodResolver(linearConnectorFormSchema), + defaultValues: { + name: "Linear Connector", + api_key: "", + }, + }); + + // Handle form submission + const onSubmit = async (values: LinearConnectorFormValues) => { + setIsSubmitting(true); + try { + await createConnector({ + name: values.name, + connector_type: "LINEAR_CONNECTOR", + config: { + LINEAR_API_KEY: values.api_key, + }, + is_indexable: true, + last_indexed_at: null, + }); + + toast.success("Linear connector created successfully!"); + + // Navigate back to connectors page + router.push(`/dashboard/${searchSpaceId}/connectors`); + } catch (error) { + console.error("Error creating connector:", error); + toast.error(error instanceof Error ? error.message : "Failed to create connector"); + } finally { + setIsSubmitting(false); + } + }; + + return ( +
+ + + + + + Connect + Documentation + + + + + + Connect Linear Workspace + + Integrate with Linear to search and retrieve information from your issues and comments. This connector can index your Linear content for search. + + + + + + Linear API Key Required + + You'll need a Linear API Key to use this connector. You can create a Linear API key from{" "} + + Linear API Settings + + + + +
+ + ( + + Connector Name + + + + + A friendly name to identify this connector. + + + + )} + /> + + ( + + Linear API Key + + + + + Your Linear API Key will be encrypted and stored securely. It typically starts with "lin_api_". + + + + )} + /> + +
+ +
+ + +
+ +

What you get with Linear integration:

+
    +
  • Search through all your Linear issues and comments
    • +
  • Access issue titles, descriptions, and full discussion threads
    • +
  • Connect your team's project management directly to your search space
    • +
  • Keep your search results up-to-date with latest Linear content
    • +
  • Index your Linear issues for enhanced search capabilities
    • +
+ + + + + + + + Linear Connector Documentation + + Learn how to set up and use the Linear connector to index your project management data. + + + +
+

How it works

+

+ The Linear connector uses the Linear GraphQL API to fetch all issues and comments that the API key has access to within a workspace. +

+
    +
  • For follow up indexing runs, the connector retrieves issues and comments that have been updated since the last indexing attempt.
    • +
  • Indexing is configured to run periodically, so updates should appear in your search results within minutes.
    • +
+
+ + + + Authorization + + + + Read-Only Access is Sufficient + + You only need a read-only API key for this connector to work. This limits the permissions to just reading your Linear data. + + + +
+
+

Step 1: Create an API key

+
    +
  1. Log in to your Linear account
    2. +
  2. Navigate to https://linear.app/settings/api in your browser.
    3. +
  3. Alternatively, click on your profile picture → Settings → API
    4. +
  4. Click the + New API key button.
    5. +
  5. Enter a description for your key (like "Search Connector").
    6. +
  6. Select "Read-only" as the permission.
    7. +
  7. Click Create to generate the API key.
    8. +
  8. Copy the generated API key that starts with 'lin_api_' as it will only be shown once.
    9. +
+
+ +
+

Step 2: Grant necessary access

+

+ The API key will have access to all issues and comments that your user account can see. If you're creating the key as an admin, it will have access to all issues in the workspace. +

+ + + Data Privacy + + Only issues and comments will be indexed. Linear attachments and linked files are not indexed by this connector. + + +
+
+ + + + + Indexing + +
    +
  1. Navigate to the Connector Dashboard and select the Linear Connector.
    2. +
  2. Place the API Key in the form field.
    3. +
  3. Click Connect to establish the connection.
    4. +
  4. Once connected, your Linear issues will be indexed automatically.
    5. +
+ + + + What Gets Indexed + +

The Linear connector indexes the following data:

+
    +
  • Issue titles and identifiers (e.g., PROJ-123)
    • +
  • Issue descriptions
    • +
  • Issue comments
    • +
  • Issue status and metadata
    • +
+ + + + + + + + + + +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx index 5eeaae5a8..1f7490270 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx @@ -14,6 +14,8 @@ import { IconChevronRight, IconMail, IconWorldWww, + IconTicket, + IconLayoutKanban, } from "@tabler/icons-react"; import { AnimatePresence, motion } from "framer-motion"; import Link from "next/link"; @@ -78,6 +80,26 @@ const connectorCategories: ConnectorCategory[] = [ }, ], }, + { + id: "project-management", + title: "Project Management", + connectors: [ + { + id: "linear-connector", + title: "Linear", + description: "Connect to Linear to search issues, comments and project data.", + icon: , + status: "available", + }, + { + id: "jira-connector", + title: "Jira", + description: "Connect to Jira to search issues, tickets and project data.", + icon: , + status: "coming-soon", + }, + ], + }, { id: "knowledge-bases", title: "Knowledge Bases", @@ -161,7 +183,7 @@ const cardVariants = { export default function ConnectorsPage() { const params = useParams(); const searchSpaceId = params.search_space_id as string; - const [expandedCategories, setExpandedCategories] = useState(["search-engines", "knowledge-bases"]); + const [expandedCategories, setExpandedCategories] = useState(["search-engines", "knowledge-bases", "project-management"]); const toggleCategory = (categoryId: string) => { setExpandedCategories(prev => diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 18b43579b..b7b4bf3ff 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -1,6 +1,7 @@ "use client"; -import { cn } from "@/lib/utils"; +import { DocumentViewer } from "@/components/document-viewer"; +import { JsonMetadataViewer } from "@/components/json-metadata-viewer"; import { AlertDialog, AlertDialogAction, @@ -12,7 +13,6 @@ import { AlertDialogTitle, AlertDialogTrigger, } from "@/components/ui/alert-dialog"; -import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { @@ -43,6 +43,9 @@ import { TableHeader, TableRow, } from "@/components/ui/table"; +import { useDocuments } from "@/hooks/use-documents"; +import { cn } from "@/lib/utils"; +import { IconBrandGithub, IconBrandNotion, IconBrandSlack, IconBrandYoutube, IconLayoutKanban } from "@tabler/icons-react"; import { ColumnDef, ColumnFiltersState, @@ -59,6 +62,7 @@ import { getSortedRowModel, useReactTable, } from "@tanstack/react-table"; +import { AnimatePresence, motion } from "framer-motion"; import { AlertCircle, ChevronDown, @@ -70,31 +74,22 @@ import { CircleAlert, CircleX, Columns3, - Filter, - ListFilter, - Plus, - FileText, - Globe, - MessageSquare, - FileX, File, - Trash, + FileX, + Filter, + Globe, + ListFilter, MoreHorizontal, - Webhook, + Trash, + Webhook } from "lucide-react"; -import { useEffect, useId, useMemo, useRef, useState, useContext } from "react"; -import { motion, AnimatePresence } from "framer-motion"; import { useParams } from "next/navigation"; -import { useDocuments } from "@/hooks/use-documents"; -import React from "react"; -import { toast } from "sonner"; +import React, { useContext, useEffect, useId, useMemo, useRef, useState } from "react"; import ReactMarkdown from "react-markdown"; import rehypeRaw from "rehype-raw"; import rehypeSanitize from "rehype-sanitize"; import remarkGfm from "remark-gfm"; -import { DocumentViewer } from "@/components/document-viewer"; -import { JsonMetadataViewer } from "@/components/json-metadata-viewer"; -import { IconBrandGithub, IconBrandNotion, IconBrandSlack, IconBrandYoutube } from "@tabler/icons-react"; +import { toast } from "sonner"; // Define animation variants for reuse const fadeInScale = { @@ -114,7 +109,7 @@ const fadeInScale = { type Document = { id: number; title: string; - document_type: "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE" | "YOUTUBE_VIDEO"; + document_type: "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE" | "YOUTUBE_VIDEO" | "LINEAR_CONNECTOR"; document_metadata: any; content: string; created_at: string; @@ -143,6 +138,7 @@ const documentTypeIcons = { FILE: File, YOUTUBE_VIDEO: IconBrandYoutube, GITHUB_CONNECTOR: IconBrandGithub, + LINEAR_CONNECTOR: IconLayoutKanban, } as const; const columns: ColumnDef[] = [ @@ -1029,4 +1025,5 @@ function RowActions({ row }: { row: Row }) { ); } -export { DocumentsTable } +export { DocumentsTable }; + diff --git a/surfsense_web/components/ModernHeroWithGradients.tsx b/surfsense_web/components/ModernHeroWithGradients.tsx index bd10a50ca..5bdf4e677 100644 --- a/surfsense_web/components/ModernHeroWithGradients.tsx +++ b/surfsense_web/components/ModernHeroWithGradients.tsx @@ -36,7 +36,7 @@ export function ModernHeroWithGradients() {

- A Customizable AI Research Agent just like NotebookLM or Perplexity, but connected to external sources such as search engines (Tavily), Slack, Notion, YouTube, GitHub and more. + A Customizable AI Research Agent just like NotebookLM or Perplexity, but connected to external sources such as search engines (Tavily), Slack, Linear, Notion, YouTube, GitHub and more.

{ const iconProps = { className: "h-4 w-4" }; switch(connectorType) { + case 'LINEAR_CONNECTOR': + return ; case 'GITHUB_CONNECTOR': return ; case 'YOUTUBE_VIDEO': From ae8c74a5aa1637243cad798d5f2db78af6f2d626 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Wed, 16 Apr 2025 19:59:38 -0700 Subject: [PATCH 12/31] select repos when adding gh connector --- .../routes/search_source_connectors_routes.py | 33 +- .../app/schemas/search_source_connector.py | 10 +- .../app/tasks/connectors_indexing_tasks.py | 29 +- .../connectors/add/github-connector/page.tsx | 396 ++++++++++++------ 4 files changed, 330 insertions(+), 138 deletions(-) diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 482a8259d..5bfe0a955 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -9,7 +9,7 @@ POST /search-source-connectors/{connector_id}/index - Index content from a conne Note: Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, NOTION_CONNECTOR). """ -from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks +from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks, Body from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.exc import IntegrityError @@ -18,8 +18,9 @@ from app.db import get_async_session, User, SearchSourceConnector, SearchSourceC from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead from app.users import current_active_user from app.utils.check_ownership import check_ownership -from pydantic import ValidationError +from pydantic import ValidationError, BaseModel, Field from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos +from app.connectors.github_connector import GitHubConnector from datetime import datetime, timezone import logging @@ -28,6 +29,34 @@ logger = logging.getLogger(__name__) router = APIRouter() +# --- New Schema for GitHub PAT --- +class GitHubPATRequest(BaseModel): + github_pat: str = Field(..., description="GitHub Personal Access Token") + +# --- New Endpoint to list GitHub Repositories --- +@router.post("/github/repositories/", response_model=List[Dict[str, Any]]) +async def list_github_repositories( + pat_request: GitHubPATRequest, + user: User = Depends(current_active_user) # Ensure the user is logged in +): + """ + Fetches a list of repositories accessible by the provided GitHub PAT. + The PAT is used for this request only and is not stored. + """ + try: + # Initialize GitHubConnector with the provided PAT + github_client = GitHubConnector(token=pat_request.github_pat) + # Fetch repositories + repositories = github_client.get_user_repositories() + return repositories + except ValueError as e: + # Handle invalid token error specifically + logger.error(f"GitHub PAT validation failed for user {user.id}: {str(e)}") + raise HTTPException(status_code=400, detail=f"Invalid GitHub PAT: {str(e)}") + except Exception as e: + logger.error(f"Failed to fetch GitHub repositories for user {user.id}: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to fetch GitHub repositories.") + @router.post("/search-source-connectors/", response_model=SearchSourceConnectorRead) async def create_search_source_connector( connector: SearchSourceConnectorCreate, diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index 41e1086e7..1005a63d8 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -4,7 +4,6 @@ from typing import Dict, Any from pydantic import BaseModel, field_validator from .base import IDModel, TimestampModel from app.db import SearchSourceConnectorType -from fastapi import HTTPException class SearchSourceConnectorBase(BaseModel): name: str @@ -59,14 +58,19 @@ class SearchSourceConnectorBase(BaseModel): raise ValueError("NOTION_INTEGRATION_TOKEN cannot be empty") elif connector_type == SearchSourceConnectorType.GITHUB_CONNECTOR: - # For GITHUB_CONNECTOR, only allow GITHUB_PAT - allowed_keys = ["GITHUB_PAT"] + # For GITHUB_CONNECTOR, only allow GITHUB_PAT and repo_full_names + allowed_keys = ["GITHUB_PAT", "repo_full_names"] if set(config.keys()) != set(allowed_keys): raise ValueError(f"For GITHUB_CONNECTOR connector type, config must only contain these keys: {allowed_keys}") # Ensure the token is not empty if not config.get("GITHUB_PAT"): raise ValueError("GITHUB_PAT cannot be empty") + + # Ensure the repo_full_names is present and is a non-empty list + repo_full_names = config.get("repo_full_names") + if not isinstance(repo_full_names, list) or not repo_full_names: + raise ValueError("repo_full_names must be a non-empty list of strings") return config diff --git a/surfsense_backend/app/tasks/connectors_indexing_tasks.py b/surfsense_backend/app/tasks/connectors_indexing_tasks.py index 670fa26ad..31a7d4d65 100644 --- a/surfsense_backend/app/tasks/connectors_indexing_tasks.py +++ b/surfsense_backend/app/tasks/connectors_indexing_tasks.py @@ -1,4 +1,4 @@ -from typing import Optional, List, Dict, Any, Tuple +from typing import Optional, Tuple from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.future import select @@ -626,10 +626,15 @@ async def index_github_repos( if not connector: return 0, f"Connector with ID {connector_id} not found or is not a GitHub connector" - # 2. Get the GitHub PAT from the connector config + # 2. Get the GitHub PAT and selected repositories from the connector config github_pat = connector.config.get("GITHUB_PAT") + repo_full_names_to_index = connector.config.get("repo_full_names") + if not github_pat: return 0, "GitHub Personal Access Token (PAT) not found in connector config" + + if not repo_full_names_to_index or not isinstance(repo_full_names_to_index, list): + return 0, "'repo_full_names' not found or is not a list in connector config" # 3. Initialize GitHub connector client try: @@ -637,13 +642,10 @@ async def index_github_repos( except ValueError as e: return 0, f"Failed to initialize GitHub client: {str(e)}" - # 4. Get list of accessible repositories - repositories = github_client.get_user_repositories() - if not repositories: - logger.info("No accessible GitHub repositories found for the provided token.") - return 0, "No accessible GitHub repositories found." - - logger.info(f"Found {len(repositories)} repositories to potentially index.") + # 4. Validate selected repositories + # For simplicity, we'll proceed with the list provided. + # If a repo is inaccessible, get_repository_files will likely fail gracefully later. + logger.info(f"Starting indexing for {len(repo_full_names_to_index)} selected repositories.") # 5. Get existing documents for this search space and connector type to prevent duplicates existing_docs_result = await session.execute( @@ -658,11 +660,10 @@ async def index_github_repos( existing_docs_lookup = {doc.document_metadata.get("full_path"): doc for doc in existing_docs if doc.document_metadata.get("full_path")} logger.info(f"Found {len(existing_docs_lookup)} existing GitHub documents in database for search space {search_space_id}") - # 6. Iterate through repositories and index files - for repo_info in repositories: - repo_full_name = repo_info.get("full_name") - if not repo_full_name: - logger.warning(f"Skipping repository with missing full_name: {repo_info.get('name')}") + # 6. Iterate through selected repositories and index files + for repo_full_name in repo_full_names_to_index: + if not repo_full_name or not isinstance(repo_full_name, str): + logger.warning(f"Skipping invalid repository entry: {repo_full_name}") continue logger.info(f"Processing repository: {repo_full_name}") diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx index 45534d6a1..fc7a60276 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx @@ -7,7 +7,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { useForm } from "react-hook-form"; import * as z from "zod"; import { toast } from "sonner"; -import { ArrowLeft, Check, Info, Loader2, Github } from "lucide-react"; +import { ArrowLeft, Check, Info, Loader2, Github, CircleAlert, ListChecks } from "lucide-react"; // Assuming useSearchSourceConnectors hook exists and works similarly import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; @@ -42,9 +42,10 @@ import { AccordionTrigger, } from "@/components/ui/accordion"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Checkbox } from "@/components/ui/checkbox"; -// Define the form schema with Zod for GitHub -const githubConnectorFormSchema = z.object({ +// Define the form schema with Zod for GitHub PAT entry step +const githubPatFormSchema = z.object({ name: z.string().min(3, { message: "Connector name must be at least 3 characters.", }), @@ -58,61 +59,144 @@ const githubConnectorFormSchema = z.object({ }); // Define the type for the form values -type GithubConnectorFormValues = z.infer; +type GithubPatFormValues = z.infer; + +// Type for fetched GitHub repositories +interface GithubRepo { + id: number; + name: string; + full_name: string; + private: boolean; + url: string; + description: string | null; + last_updated: string | null; +} export default function GithubConnectorPage() { const router = useRouter(); const params = useParams(); const searchSpaceId = params.search_space_id as string; - const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); // Assuming this hook exists + const [step, setStep] = useState<'enter_pat' | 'select_repos'>('enter_pat'); + const [isFetchingRepos, setIsFetchingRepos] = useState(false); + const [isCreatingConnector, setIsCreatingConnector] = useState(false); + const [repositories, setRepositories] = useState([]); + const [selectedRepos, setSelectedRepos] = useState([]); + const [connectorName, setConnectorName] = useState("GitHub Connector"); + const [validatedPat, setValidatedPat] = useState(""); // Store the validated PAT - // Initialize the form - const form = useForm({ - resolver: zodResolver(githubConnectorFormSchema), + const { createConnector } = useSearchSourceConnectors(); + + // Initialize the form for PAT entry + const form = useForm({ + resolver: zodResolver(githubPatFormSchema), defaultValues: { - name: "GitHub Connector", + name: connectorName, github_pat: "", }, }); - // Handle form submission - const onSubmit = async (values: GithubConnectorFormValues) => { - setIsSubmitting(true); + // Function to fetch repositories using the new backend endpoint + const fetchRepositories = async (values: GithubPatFormValues) => { + setIsFetchingRepos(true); + setConnectorName(values.name); // Store the name + setValidatedPat(values.github_pat); // Store the PAT temporarily + try { + const token = localStorage.getItem('surfsense_bearer_token'); + if (!token) { + throw new Error('No authentication token found'); + } + + const response = await fetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/github/repositories/`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}` + }, + body: JSON.stringify({ github_pat: values.github_pat }) + } + ); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.detail || `Failed to fetch repositories: ${response.statusText}`); + } + + const data: GithubRepo[] = await response.json(); + setRepositories(data); + setStep('select_repos'); // Move to the next step + toast.success(`Found ${data.length} repositories.`); + } catch (error) { + console.error("Error fetching GitHub repositories:", error); + const errorMessage = error instanceof Error ? error.message : "Failed to fetch repositories. Please check the PAT and try again."; + toast.error(errorMessage); + } finally { + setIsFetchingRepos(false); + } + }; + + // Handle final connector creation + const handleCreateConnector = async () => { + if (selectedRepos.length === 0) { + toast.warning("Please select at least one repository to index."); + return; + } + + setIsCreatingConnector(true); try { await createConnector({ - name: values.name, + name: connectorName, // Use the stored name connector_type: "GITHUB_CONNECTOR", config: { - GITHUB_PAT: values.github_pat, + GITHUB_PAT: validatedPat, // Use the stored validated PAT + repo_full_names: selectedRepos, // Add the selected repo names }, - is_indexable: true, // GitHub connector is indexable - last_indexed_at: null, // New connector hasn't been indexed + is_indexable: true, + last_indexed_at: null, }); toast.success("GitHub connector created successfully!"); - - // Navigate back to connectors management page (or the add page) router.push(`/dashboard/${searchSpaceId}/connectors`); - } catch (error) { // Added type check for error + } catch (error) { console.error("Error creating GitHub connector:", error); - // Display specific backend error message if available - const errorMessage = error instanceof Error ? error.message : "Failed to create GitHub connector. Please check the PAT and permissions."; + const errorMessage = error instanceof Error ? error.message : "Failed to create GitHub connector."; toast.error(errorMessage); } finally { - setIsSubmitting(false); + setIsCreatingConnector(false); } }; + // Handle checkbox changes + const handleRepoSelection = (repoFullName: string, checked: boolean) => { + setSelectedRepos(prev => + checked + ? [...prev, repoFullName] + : prev.filter(name => name !== repoFullName) + ); + }; + return (
- Connect GitHub Account + + {step === 'enter_pat' ? : } + {step === 'enter_pat' ? "Connect GitHub Account" : "Select Repositories to Index"} + - Integrate with GitHub using a Personal Access Token (PAT) to search and retrieve information from accessible repositories. This connector can index your code and documentation. + {step === 'enter_pat' + ? "Provide a name and GitHub Personal Access Token (PAT) to fetch accessible repositories." + : `Select which repositories you want SurfSense to index for search. Found ${repositories.length} repositories accessible via your PAT.` + } - - - - GitHub Personal Access Token (PAT) Required - - You'll need a GitHub PAT with the appropriate scopes (e.g., 'repo') to use this connector. You can create one from your - - GitHub Developer Settings - . - - -
- - ( - - Connector Name - - - - - A friendly name to identify this GitHub connection. - - - - )} - /> + + {step === 'enter_pat' && ( + + + + GitHub Personal Access Token (PAT) Required + + You'll need a GitHub PAT with the appropriate scopes (e.g., 'repo') to fetch repositories. You can create one from your{' '} + + GitHub Developer Settings + . The PAT will be used to fetch repositories and then stored securely to enable indexing. + + - ( - - GitHub Personal Access Token (PAT) - - - - - Your GitHub PAT will be encrypted and stored securely. Ensure it has the necessary 'repo' scopes. - - - - )} - /> - -
- -
- - - + /> + + ( + + GitHub Personal Access Token (PAT) + + + + + Enter your GitHub PAT here to fetch your repositories. It will be stored encrypted later. + + + + )} + /> + +
+ +
+ +
+ )} + + {step === 'select_repos' && ( + + {repositories.length === 0 ? ( + + + No Repositories Found + + No repositories were found or accessible with the provided PAT. Please check the token and its permissions, then go back and try again. + + + ) : ( +
+ Repositories ({selectedRepos.length} selected) +
+ {repositories.map((repo) => ( +
+ handleRepoSelection(repo.full_name, !!checked)} + /> + +
+ ))} +
+ + Select the repositories you wish to index. Only checked repositories will be processed. + + +
+ + +
+
+ )} + + )} + +

What you get with GitHub integration:

    -
  • Search through code and documentation in your repositories
    • +
  • Search through code and documentation in your selected repositories
  • Access READMEs, Markdown files, and common code files
  • Connect your project knowledge directly to your search space
    • -
  • Index your repositories for enhanced search capabilities
    • +
  • Index your selected repositories for enhanced search capabilities
 @@ -237,27 +398,20 @@ export default function GithubConnectorPage() {

How it works

- The GitHub connector uses a Personal Access Token (PAT) to authenticate with the GitHub API. It fetches information about repositories accessible to the token and indexes relevant files (code, markdown, text). + The GitHub connector uses a Personal Access Token (PAT) to authenticate with the GitHub API. First, it fetches a list of repositories accessible to the token. You then select which repositories you want to index. The connector indexes relevant files (code, markdown, text) from only the selected repositories.

  • The connector indexes files based on common code and documentation extensions.
  • Large files (over 1MB) are skipped during indexing.
    • +
  • Only selected repositories are indexed.
  • Indexing runs periodically (check connector settings for frequency) to keep content up-to-date.
- Step 1: Create a GitHub PAT - - - - Token Security - - Treat your PAT like a password. Store it securely and consider using fine-grained tokens if possible. - - - + Step 1: Generate GitHub PAT +

Generating a Token:

@@ -280,9 +434,13 @@ export default function GithubConnectorPage() { Step 2: Connect in SurfSense
    -
  1. Paste the copied GitHub PAT into the "GitHub Personal Access Token (PAT)" field on the "Connect GitHub" tab.
    2. -
  2. Optionally, give the connector a custom name.
    3. -
  3. Click the Connect GitHub button.
    4. +
  4. Navigate to the "Connect GitHub" tab.
    5. +
  5. Enter a name for your connector.
    6. +
  6. Paste the copied GitHub PAT into the "GitHub Personal Access Token (PAT)" field.
    7. +
  7. Click Fetch Repositories.
    8. +
  8. If the PAT is valid, you'll see a list of your accessible repositories.
    9. +
  9. Select the repositories you want SurfSense to index using the checkboxes.
    10. +
  10. Click the Create Connector button.
  11. If the connection is successful, you will be redirected and can start indexing from the Connectors page.
 From 5176569e30e2e717e4305e3fd198510116985c43 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Wed, 16 Apr 2025 20:29:50 -0700 Subject: [PATCH 13/31] edit repos for gh connector --- .../routes/search_source_connectors_routes.py | 78 +++- .../app/schemas/search_source_connector.py | 12 +- .../connectors/(manage)/page.tsx | 2 +- .../connectors/[connector_id]/edit/page.tsx | 442 ++++++++++++++++++ .../connectors/[connector_id]/page.tsx | 2 +- 5 files changed, 506 insertions(+), 30 deletions(-) create mode 100644 surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 5bfe0a955..f843029e2 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -15,7 +15,7 @@ from sqlalchemy.future import select from sqlalchemy.exc import IntegrityError from typing import List, Dict, Any from app.db import get_async_session, User, SearchSourceConnector, SearchSourceConnectorType, SearchSpace, async_session_maker -from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead +from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead, SearchSourceConnectorBase from app.users import current_active_user from app.utils.check_ownership import check_ownership from pydantic import ValidationError, BaseModel, Field @@ -159,54 +159,84 @@ async def update_search_source_connector( ): """ Update a search source connector. - - Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR). - The config must contain the appropriate keys for the connector type. + Handles partial updates, including merging changes into the 'config' field. """ - try: - db_connector = await check_ownership(session, SearchSourceConnector, connector_id, user) + db_connector = await check_ownership(session, SearchSourceConnector, connector_id, user) + + # Convert the sparse update data (only fields present in request) to a dict + update_data = connector_update.model_dump(exclude_unset=True) + + # Special handling for 'config' field + if "config" in update_data: + incoming_config = update_data["config"] # Config data from the request + existing_config = db_connector.config if db_connector.config else {} # Current config from DB - # If connector type is being changed, check if one of that type already exists - if connector_update.connector_type != db_connector.connector_type: + # Merge incoming config into existing config + # This preserves existing keys (like GITHUB_PAT) if they are not in the incoming data + merged_config = existing_config.copy() + merged_config.update(incoming_config) + + # -- Validation after merging -- + # Validate the *merged* config based on the connector type + # We need the connector type - use the one from the update if provided, else the existing one + current_connector_type = connector_update.connector_type if connector_update.connector_type is not None else db_connector.connector_type + + try: + # We can reuse the base validator by creating a temporary base model instance + # Note: This assumes 'name' and 'is_indexable' are not crucial for config validation itself + temp_data_for_validation = { + "name": db_connector.name, # Use existing name + "connector_type": current_connector_type, + "is_indexable": db_connector.is_indexable, # Use existing value + "last_indexed_at": db_connector.last_indexed_at, # Not used by validator + "config": merged_config + } + SearchSourceConnectorBase.model_validate(temp_data_for_validation) + except ValidationError as e: + # Raise specific validation error for the merged config + raise HTTPException( + status_code=422, + detail=f"Validation error for merged config: {str(e)}" + ) + + # If validation passes, update the main update_data dict with the merged config + update_data["config"] = merged_config + + # Apply all updates (including the potentially merged config) + for key, value in update_data.items(): + # Prevent changing connector_type if it causes a duplicate (check moved here) + if key == "connector_type" and value != db_connector.connector_type: result = await session.execute( select(SearchSourceConnector) .filter( SearchSourceConnector.user_id == user.id, - SearchSourceConnector.connector_type == connector_update.connector_type, + SearchSourceConnector.connector_type == value, SearchSourceConnector.id != connector_id ) ) existing_connector = result.scalars().first() - if existing_connector: raise HTTPException( status_code=409, - detail=f"A connector with type {connector_update.connector_type} already exists. Each user can have only one connector of each type." + detail=f"A connector with type {value} already exists. Each user can have only one connector of each type." ) - update_data = connector_update.model_dump(exclude_unset=True) - for key, value in update_data.items(): - setattr(db_connector, key, value) + setattr(db_connector, key, value) + + try: await session.commit() await session.refresh(db_connector) return db_connector - except ValidationError as e: - await session.rollback() - raise HTTPException( - status_code=422, - detail=f"Validation error: {str(e)}" - ) except IntegrityError as e: await session.rollback() + # This might occur if connector_type constraint is violated somehow after the check raise HTTPException( status_code=409, - detail=f"Integrity error: A connector with this type already exists. {str(e)}" + detail=f"Database integrity error during update: {str(e)}" ) - except HTTPException: - await session.rollback() - raise except Exception as e: await session.rollback() + logger.error(f"Failed to update search source connector {connector_id}: {e}", exc_info=True) raise HTTPException( status_code=500, detail=f"Failed to update search source connector: {str(e)}" diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index 1005a63d8..d0c46313b 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -1,6 +1,6 @@ from datetime import datetime import uuid -from typing import Dict, Any +from typing import Dict, Any, Optional from pydantic import BaseModel, field_validator from .base import IDModel, TimestampModel from app.db import SearchSourceConnectorType @@ -9,7 +9,7 @@ class SearchSourceConnectorBase(BaseModel): name: str connector_type: SearchSourceConnectorType is_indexable: bool - last_indexed_at: datetime | None + last_indexed_at: Optional[datetime] = None config: Dict[str, Any] @field_validator('config') @@ -77,8 +77,12 @@ class SearchSourceConnectorBase(BaseModel): class SearchSourceConnectorCreate(SearchSourceConnectorBase): pass -class SearchSourceConnectorUpdate(SearchSourceConnectorBase): - pass +class SearchSourceConnectorUpdate(BaseModel): + name: Optional[str] = None + connector_type: Optional[SearchSourceConnectorType] = None + is_indexable: Optional[bool] = None + last_indexed_at: Optional[datetime] = None + config: Optional[Dict[str, Any]] = None class SearchSourceConnectorRead(SearchSourceConnectorBase, IDModel, TimestampModel): user_id: uuid.UUID diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index 817ca584d..bb890ebd5 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -205,7 +205,7 @@ export default function ConnectorsPage() { + + + + + Edit GitHub Connector + + Modify the connector name and repository selections. To change repository selections, you need to re-enter your PAT. + + + + {/* Use editForm for the main form structure */} +
+ + + {/* Connector Name Field */} + ( + + Connector Name + + + + + + )} + /> + +
+ + {/* Repository Selection Section */} +
+

Repository Selection

+ + {editMode === 'viewing' && ( +
+ Currently Indexed Repositories: + {currentSelectedRepos.length > 0 ? ( +
    + {currentSelectedRepos.map(repo =>
  • {repo}
    • )} +
+ ) : ( +

(No repositories currently selected for indexing)

+ )} + + Click "Change Selection" to re-enter your PAT and update the list. +
+ )} + + {editMode === 'editing_repos' && ( +
+ {/* PAT Input Section (No nested Form provider) */} + {/* We still use patForm fields but trigger validation manually */} +
+ ( + + Re-enter PAT to Fetch Repos + + + + + + )} + /> + +
+ + {/* Fetched Repository List (shown after fetch) */} + {isFetchingRepos && } + {!isFetchingRepos && fetchedRepos !== null && ( + fetchedRepos.length === 0 ? ( + + + No Repositories Found + Check the PAT and permissions. + + ) : ( +
+ Select Repositories to Index ({newSelectedRepos.length} selected): +
+ {fetchedRepos.map((repo) => ( +
+ handleRepoSelectionChange(repo.full_name, !!checked)} + /> + +
+ ))} +
+
+ ) + )} + +
+ )} +
+ + + + + + +
+ + + +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx index e841639cd..ad6ceb7bf 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx @@ -70,7 +70,7 @@ export default function EditConnectorPage() { const [connector, setConnector] = useState(null); const [isLoading, setIsLoading] = useState(true); const [isSubmitting, setIsSubmitting] = useState(false); - + console.log("connector", connector); // Initialize the form const form = useForm({ resolver: zodResolver(apiConnectorFormSchema), From 69eea7485b00e673bcc6fb2783a1f2280d7882b1 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Wed, 16 Apr 2025 20:38:44 -0700 Subject: [PATCH 14/31] reuse edit page for other connectors --- .../connectors/[connector_id]/edit/page.tsx | 482 +++++++++++------- 1 file changed, 295 insertions(+), 187 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx index e526976db..cc348bab7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx @@ -37,7 +37,19 @@ import { import { Checkbox } from "@/components/ui/checkbox"; import { Skeleton } from "@/components/ui/skeleton"; -// Schema for PAT input when editing repos +// Helper function to get connector type display name (copied from manage page) +const getConnectorTypeDisplay = (type: string): string => { + const typeMap: Record = { + "SERPER_API": "Serper API", + "TAVILY_API": "Tavily API", + "SLACK_CONNECTOR": "Slack", + "NOTION_CONNECTOR": "Notion", + "GITHUB_CONNECTOR": "GitHub", + }; + return typeMap[type] || type; +}; + +// Schema for PAT input when editing GitHub repos (remains separate) const githubPatSchema = z.object({ github_pat: z.string() .min(20, { message: "GitHub Personal Access Token seems too short." }) @@ -47,9 +59,15 @@ const githubPatSchema = z.object({ }); type GithubPatFormValues = z.infer; -// Schema for main edit form (just the name for now) +// Updated schema for main edit form - includes optional fields for other connector configs const editConnectorSchema = z.object({ name: z.string().min(3, { message: "Connector name must be at least 3 characters." }), + // Add optional fields for other connector types' configs + SLACK_BOT_TOKEN: z.string().optional(), + NOTION_INTEGRATION_TOKEN: z.string().optional(), + SERPER_API_KEY: z.string().optional(), + TAVILY_API_KEY: z.string().optional(), + // GITHUB_PAT is handled separately via patForm for repo editing flow }); type EditConnectorFormValues = z.infer; @@ -63,9 +81,9 @@ interface GithubRepo { last_updated: string | null; } -type EditMode = 'viewing' | 'editing_repos'; +type EditMode = 'viewing' | 'editing_repos'; // Only relevant for GitHub -export default function EditGithubConnectorPage() { +export default function EditConnectorPage() { // Renamed for clarity const router = useRouter(); const params = useParams(); const searchSpaceId = params.search_space_id as string; @@ -74,54 +92,74 @@ export default function EditGithubConnectorPage() { const { connectors, updateConnector, isLoading: connectorsLoading } = useSearchSourceConnectors(); const [connector, setConnector] = useState(null); + const [originalConfig, setOriginalConfig] = useState | null>(null); // Store original config object + + // GitHub specific state (only used if connector type is GitHub) const [currentSelectedRepos, setCurrentSelectedRepos] = useState([]); - const [originalPat, setOriginalPat] = useState(""); // State to hold the initial PAT - const [editMode, setEditMode] = useState('viewing'); - const [fetchedRepos, setFetchedRepos] = useState(null); // Null indicates not fetched yet for edit - const [newSelectedRepos, setNewSelectedRepos] = useState([]); // Tracks selections *during* edit + const [originalPat, setOriginalPat] = useState(""); + const [editMode, setEditMode] = useState('viewing'); + const [fetchedRepos, setFetchedRepos] = useState(null); + const [newSelectedRepos, setNewSelectedRepos] = useState([]); const [isFetchingRepos, setIsFetchingRepos] = useState(false); + const [isSaving, setIsSaving] = useState(false); - // Form for just the PAT input + // Form for GitHub PAT input (only used for GitHub repo editing) const patForm = useForm({ resolver: zodResolver(githubPatSchema), - defaultValues: { github_pat: "" }, // Default empty, will be reset + defaultValues: { github_pat: "" }, }); - // Form for the main connector details (e.g., name) + // Main form for connector details (name + simple config fields) const editForm = useForm({ resolver: zodResolver(editConnectorSchema), - defaultValues: { name: "" }, + defaultValues: { + name: "", + SLACK_BOT_TOKEN: "", + NOTION_INTEGRATION_TOKEN: "", + SERPER_API_KEY: "", + TAVILY_API_KEY: "", + }, }); - // Effect to find and set the current connector details on load + // Effect to load connector data useEffect(() => { - if (!connectorsLoading && connectors.length > 0 && !connector) { // Added !connector check to prevent loop + if (!connectorsLoading && connectors.length > 0 && !connector) { const currentConnector = connectors.find(c => c.id === connectorId); - if (currentConnector && currentConnector.connector_type === 'GITHUB_CONNECTOR') { + if (currentConnector) { setConnector(currentConnector); - const savedRepos = currentConnector.config?.repo_full_names || []; - const savedPat = currentConnector.config?.GITHUB_PAT || ""; - setCurrentSelectedRepos(savedRepos); - setNewSelectedRepos(savedRepos); - setOriginalPat(savedPat); // Store the original PAT - editForm.reset({ name: currentConnector.name }); - patForm.reset({ github_pat: savedPat }); // Also reset PAT form initially - } else if (currentConnector) { - toast.error("This connector is not a GitHub connector."); - router.push(`/dashboard/${searchSpaceId}/connectors`); + setOriginalConfig(currentConnector.config || {}); // Store original config + + // Reset main form with common and type-specific fields + editForm.reset({ + name: currentConnector.name, + SLACK_BOT_TOKEN: currentConnector.config?.SLACK_BOT_TOKEN || "", + NOTION_INTEGRATION_TOKEN: currentConnector.config?.NOTION_INTEGRATION_TOKEN || "", + SERPER_API_KEY: currentConnector.config?.SERPER_API_KEY || "", + TAVILY_API_KEY: currentConnector.config?.TAVILY_API_KEY || "", + }); + + // If GitHub, set up GitHub-specific state + if (currentConnector.connector_type === 'GITHUB_CONNECTOR') { + const savedRepos = currentConnector.config?.repo_full_names || []; + const savedPat = currentConnector.config?.GITHUB_PAT || ""; + setCurrentSelectedRepos(savedRepos); + setNewSelectedRepos(savedRepos); + setOriginalPat(savedPat); + patForm.reset({ github_pat: savedPat }); + setEditMode('viewing'); // Start in viewing mode for repos + } } else { toast.error("Connector not found."); router.push(`/dashboard/${searchSpaceId}/connectors`); } } - }, [connectorId, connectors, connectorsLoading, router, searchSpaceId, connector]); // Removed editForm, patForm from dependencies + }, [connectorId, connectors, connectorsLoading, router, searchSpaceId, connector, editForm, patForm]); // Fetch repositories using the entered PAT const handleFetchRepositories = async (values: GithubPatFormValues) => { setIsFetchingRepos(true); setFetchedRepos(null); - // No need for patInputValue state, values.github_pat has the submitted value try { const token = localStorage.getItem('surfsense_bearer_token'); if (!token) throw new Error('No authentication token found'); @@ -137,21 +175,17 @@ export default function EditGithubConnectorPage() { body: JSON.stringify({ github_pat: values.github_pat }) } ); - if (!response.ok) { const errorData = await response.json(); throw new Error(errorData.detail || `Failed to fetch repositories: ${response.statusText}`); } - const data: GithubRepo[] = await response.json(); setFetchedRepos(data); - // Reset selection based on currently SAVED repos when fetching - setNewSelectedRepos(currentSelectedRepos); + setNewSelectedRepos(currentSelectedRepos); toast.success(`Found ${data.length} repositories. Select which ones to index.`); } catch (error) { console.error("Error fetching GitHub repositories:", error); toast.error(error instanceof Error ? error.message : "Failed to fetch repositories."); - // Don't clear PAT form on error, let user fix it } finally { setIsFetchingRepos(false); } @@ -166,62 +200,97 @@ export default function EditGithubConnectorPage() { ); }; - // Save all changes (name and potentially repo selection + PAT) + // Save changes - updated to handle different connector types const handleSaveChanges = async (formData: EditConnectorFormValues) => { - if (!connector) return; + if (!connector || !originalConfig) return; setIsSaving(true); const updatePayload: Partial = {}; let configChanged = false; + let newConfig: Record | null = null; // 1. Check if name changed if (formData.name !== connector.name) { updatePayload.name = formData.name; } - // 2. Check PAT and Repo changes - const currentPatInForm = patForm.getValues('github_pat'); - let patChanged = false; + // 2. Check for config changes based on connector type + switch (connector.connector_type) { + case 'GITHUB_CONNECTOR': + const currentPatInForm = patForm.getValues('github_pat'); + const patChanged = currentPatInForm !== originalPat; + const initialRepoSet = new Set(currentSelectedRepos); + const newRepoSet = new Set(newSelectedRepos); + const reposChanged = initialRepoSet.size !== newRepoSet.size || ![...initialRepoSet].every(repo => newRepoSet.has(repo)); - // Check if PAT input field was actually edited - if (editMode === 'editing_repos' && currentPatInForm !== originalPat) { - patChanged = true; + if (patChanged || (editMode === 'editing_repos' && reposChanged && fetchedRepos !== null)) { + if (!currentPatInForm || !(currentPatInForm.startsWith('ghp_') || currentPatInForm.startsWith('github_pat_'))) { + toast.error("Invalid GitHub PAT format. Cannot save."); + setIsSaving(false); return; + } + newConfig = { + GITHUB_PAT: currentPatInForm, + repo_full_names: newSelectedRepos, + }; + if (reposChanged && newSelectedRepos.length === 0) { + toast.warning("Warning: No repositories selected."); + } + } + break; + + case 'SLACK_CONNECTOR': + if (formData.SLACK_BOT_TOKEN !== originalConfig.SLACK_BOT_TOKEN) { + if (!formData.SLACK_BOT_TOKEN) { + toast.error("Slack Bot Token cannot be empty."); setIsSaving(false); return; + } + newConfig = { SLACK_BOT_TOKEN: formData.SLACK_BOT_TOKEN }; + } + break; + + case 'NOTION_CONNECTOR': + if (formData.NOTION_INTEGRATION_TOKEN !== originalConfig.NOTION_INTEGRATION_TOKEN) { + if (!formData.NOTION_INTEGRATION_TOKEN) { + toast.error("Notion Integration Token cannot be empty."); setIsSaving(false); return; + } + newConfig = { NOTION_INTEGRATION_TOKEN: formData.NOTION_INTEGRATION_TOKEN }; + } + break; + + case 'SERPER_API': + if (formData.SERPER_API_KEY !== originalConfig.SERPER_API_KEY) { + if (!formData.SERPER_API_KEY) { + toast.error("Serper API Key cannot be empty."); setIsSaving(false); return; + } + newConfig = { SERPER_API_KEY: formData.SERPER_API_KEY }; + } + break; + + case 'TAVILY_API': + if (formData.TAVILY_API_KEY !== originalConfig.TAVILY_API_KEY) { + if (!formData.TAVILY_API_KEY) { + toast.error("Tavily API Key cannot be empty."); setIsSaving(false); return; + } + newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY }; + } + break; + + // Add cases for other connector types if necessary } - // Check if repo selection was modified - const initialRepoSet = new Set(currentSelectedRepos); - const newRepoSet = new Set(newSelectedRepos); - const reposChanged = initialRepoSet.size !== newRepoSet.size || ![...initialRepoSet].every(repo => newRepoSet.has(repo)); - - // If PAT was changed OR repos were changed (implying PAT was involved) - if (patChanged || (editMode === 'editing_repos' && reposChanged && fetchedRepos !== null)) { - // Validate the PAT from the form before including it - if (!currentPatInForm || !(currentPatInForm.startsWith('ghp_') || currentPatInForm.startsWith('github_pat_'))) { - toast.error("Invalid GitHub PAT format in the input field. Cannot save config changes."); - setIsSaving(false); - return; - } - - updatePayload.config = { - // Use the PAT value currently in the form field - GITHUB_PAT: currentPatInForm, - // Use the latest repo selection state - repo_full_names: newSelectedRepos, - }; - configChanged = true; // Mark config as changed - - if (reposChanged && newSelectedRepos.length === 0) { - toast.warning("Warning: You haven't selected any repositories. The connector won't index anything."); - } + // If config was determined to have changed, add it to the payload + if (newConfig !== null) { + updatePayload.config = newConfig; + configChanged = true; } // 3. Check if there are actual changes to save if (Object.keys(updatePayload).length === 0) { toast.info("No changes detected."); setIsSaving(false); - setEditMode('viewing'); - // Reset PAT form to original value if returning to view mode without saving PAT change - patForm.reset({ github_pat: originalPat }); + if (connector.connector_type === 'GITHUB_CONNECTOR') { + setEditMode('viewing'); + patForm.reset({ github_pat: originalPat }); + } return; } @@ -230,29 +299,38 @@ export default function EditGithubConnectorPage() { await updateConnector(connectorId, updatePayload); toast.success("Connector updated successfully!"); - // Update local state based on what was *actually* saved - if (updatePayload.config) { - setCurrentSelectedRepos(updatePayload.config.repo_full_names || []); - setOriginalPat(updatePayload.config.GITHUB_PAT || ""); - // Reset PAT form with the newly saved PAT - patForm.reset({ github_pat: updatePayload.config.GITHUB_PAT || "" }); - } else { - // If config wasn't in payload, ensure PAT form is reset to original value - patForm.reset({ github_pat: originalPat }); - } - // Update connector name state if it changed (or rely on hook refresh) + // Update local state after successful save + const newlySavedConfig = updatePayload.config || originalConfig; + setOriginalConfig(newlySavedConfig); if (updatePayload.name) { - setConnector(prev => prev ? { ...prev, name: updatePayload.name! } : null); + setConnector(prev => prev ? { ...prev, name: updatePayload.name!, config: newlySavedConfig } : null); + editForm.setValue('name', updatePayload.name); + } else { + setConnector(prev => prev ? { ...prev, config: newlySavedConfig } : null); } - // Reset edit state - setEditMode('viewing'); - setFetchedRepos(null); - // Reset working selection to match saved state (use the updated currentSelectedRepos) - setNewSelectedRepos(updatePayload.config?.repo_full_names || currentSelectedRepos); + if (connector.connector_type === 'GITHUB_CONNECTOR' && configChanged) { + const savedGitHubConfig = newlySavedConfig as { GITHUB_PAT?: string; repo_full_names?: string[] }; + setCurrentSelectedRepos(savedGitHubConfig.repo_full_names || []); + setOriginalPat(savedGitHubConfig.GITHUB_PAT || ""); + setNewSelectedRepos(savedGitHubConfig.repo_full_names || []); + patForm.reset({ github_pat: savedGitHubConfig.GITHUB_PAT || "" }); + } else if (connector.connector_type === 'SLACK_CONNECTOR' && configChanged) { + editForm.setValue('SLACK_BOT_TOKEN', newlySavedConfig.SLACK_BOT_TOKEN || ""); + } // Add similar blocks for Notion, Serper, Tavily + else if (connector.connector_type === 'NOTION_CONNECTOR' && configChanged) { + editForm.setValue('NOTION_INTEGRATION_TOKEN', newlySavedConfig.NOTION_INTEGRATION_TOKEN || ""); + } else if (connector.connector_type === 'SERPER_API' && configChanged) { + editForm.setValue('SERPER_API_KEY', newlySavedConfig.SERPER_API_KEY || ""); + } else if (connector.connector_type === 'TAVILY_API' && configChanged) { + editForm.setValue('TAVILY_API_KEY', newlySavedConfig.TAVILY_API_KEY || ""); + } - // Optionally redirect or rely on hook refresh - // router.push(`/dashboard/${searchSpaceId}/connectors`); + // Reset GitHub specific edit state + if (connector.connector_type === 'GITHUB_CONNECTOR') { + setEditMode('viewing'); + setFetchedRepos(null); + } } catch (error) { console.error("Error updating connector:", error); @@ -298,26 +376,27 @@ export default function EditGithubConnectorPage() { > - Edit GitHub Connector + {/* Title can be dynamic based on type */} + + {/* TODO: Make icon dynamic */} + Edit {getConnectorTypeDisplay(connector.connector_type)} Connector + - Modify the connector name and repository selections. To change repository selections, you need to re-enter your PAT. + Modify the connector name and configuration. - {/* Use editForm for the main form structure */}
- {/* Connector Name Field */} + {/* Name Field (Common) */} ( Connector Name - - - + )} @@ -325,108 +404,137 @@ export default function EditGithubConnectorPage() {
- {/* Repository Selection Section */} -
-

Repository Selection

+ {/* --- Conditional Configuration Section --- */} +

Configuration

- {editMode === 'viewing' && ( -
- Currently Indexed Repositories: - {currentSelectedRepos.length > 0 ? ( -
    - {currentSelectedRepos.map(repo =>
  • {repo}
    • )} -
- ) : ( -

(No repositories currently selected for indexing)

- )} - - Click "Change Selection" to re-enter your PAT and update the list. -
- )} - - {editMode === 'editing_repos' && ( -
- {/* PAT Input Section (No nested Form provider) */} - {/* We still use patForm fields but trigger validation manually */} -
- ( + {/* == GitHub == */} + {connector.connector_type === 'GITHUB_CONNECTOR' && ( +
+

Repository Selection & Access

+ {editMode === 'viewing' && ( +
+ Currently Indexed Repositories: + {currentSelectedRepos.length > 0 ? ( +
    + {currentSelectedRepos.map(repo =>
  • {repo}
    • )} +
+ ) : ( +

(No repositories currently selected)

+ )} + + To change repo selections or update the PAT, click above. +
+ )} + {editMode === 'editing_repos' && ( +
+ {/* PAT Input */} +
+ ( - Re-enter PAT to Fetch Repos - - - + GitHub PAT + + Enter PAT to fetch/update repos or if you need to update the stored token. - )} - /> - +
+ {/* Repo List */} + {isFetchingRepos && } + {!isFetchingRepos && fetchedRepos !== null && ( + fetchedRepos.length === 0 ? ( + No Repositories FoundCheck PAT & permissions. + ) : ( +
+ Select Repositories to Index ({newSelectedRepos.length} selected): +
+ {fetchedRepos.map((repo) => ( +
+ handleRepoSelectionChange(repo.full_name, !!checked)} /> + +
+ ))} +
+
+ ) + )} +
+ )} +
+ )} - {/* Fetched Repository List (shown after fetch) */} - {isFetchingRepos && } - {!isFetchingRepos && fetchedRepos !== null && ( - fetchedRepos.length === 0 ? ( - - - No Repositories Found - Check the PAT and permissions. - - ) : ( -
- Select Repositories to Index ({newSelectedRepos.length} selected): -
- {fetchedRepos.map((repo) => ( -
- handleRepoSelectionChange(repo.full_name, !!checked)} - /> - -
- ))} -
-
- ) - )} - -
- )} -
+ {/* == Slack == */} + {connector.connector_type === 'SLACK_CONNECTOR' && ( + ( + + Slack Bot Token + + Update the Slack Bot Token if needed. + + + )} + /> + )} + + {/* == Notion == */} + {connector.connector_type === 'NOTION_CONNECTOR' && ( + ( + + Notion Integration Token + + Update the Notion Integration Token if needed. + + + )} + /> + )} + + {/* == Serper API == */} + {connector.connector_type === 'SERPER_API' && ( + ( + + Serper API Key + + Update the Serper API Key if needed. + + + )} + /> + )} + + {/* == Tavily API == */} + {connector.connector_type === 'TAVILY_API' && ( + ( + + Tavily API Key + + Update the Tavily API Key if needed. + + + )} + /> + )} - - To change repo selections or update the PAT, click above. -
- )} - {editMode === 'editing_repos' && ( -
- {/* PAT Input */} -
- ( - - GitHub PAT - - Enter PAT to fetch/update repos or if you need to update the stored token. - - - )} /> - -
- {/* Repo List */} - {isFetchingRepos && } - {!isFetchingRepos && fetchedRepos !== null && ( - fetchedRepos.length === 0 ? ( - No Repositories FoundCheck PAT & permissions. - ) : ( -
- Select Repositories to Index ({newSelectedRepos.length} selected): -
- {fetchedRepos.map((repo) => ( -
- handleRepoSelectionChange(repo.full_name, !!checked)} /> - -
- ))} -
-
- ) - )} - -
- )} -
+ )} {/* == Slack == */} {connector.connector_type === 'SLACK_CONNECTOR' && ( - ( - - Slack Bot Token - - Update the Slack Bot Token if needed. - - - )} + )} {/* == Notion == */} {connector.connector_type === 'NOTION_CONNECTOR' && ( - ( - - Notion Integration Token - - Update the Notion Integration Token if needed. - - - )} + )} {/* == Serper API == */} {connector.connector_type === 'SERPER_API' && ( - ( - - Serper API Key - - Update the Serper API Key if needed. - - - )} + )} {/* == Tavily API == */} {connector.connector_type === 'TAVILY_API' && ( - ( - - Tavily API Key - - Update the Tavily API Key if needed. - - - )} + )} diff --git a/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx b/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx new file mode 100644 index 000000000..dc1320fa3 --- /dev/null +++ b/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx @@ -0,0 +1,21 @@ +import React from 'react'; +import { Skeleton } from "@/components/ui/skeleton"; +import { Card, CardContent, CardHeader } from "@/components/ui/card"; + +export function EditConnectorLoadingSkeleton() { + return ( +
+ + + + + + + + + + + +
+ ); +} diff --git a/surfsense_web/components/editConnector/EditConnectorNameForm.tsx b/surfsense_web/components/editConnector/EditConnectorNameForm.tsx new file mode 100644 index 000000000..3f1882004 --- /dev/null +++ b/surfsense_web/components/editConnector/EditConnectorNameForm.tsx @@ -0,0 +1,25 @@ +import React from 'react'; +import { Control } from 'react-hook-form'; +import { FormField, FormItem, FormLabel, FormControl, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; + +// Assuming EditConnectorFormValues is defined elsewhere or passed as generic +interface EditConnectorNameFormProps { + control: Control; // Use Control if type is available +} + +export function EditConnectorNameForm({ control }: EditConnectorNameFormProps) { + return ( + ( + + Connector Name + + + + )} + /> + ); +} diff --git a/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx b/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx new file mode 100644 index 000000000..17f83f7ac --- /dev/null +++ b/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx @@ -0,0 +1,160 @@ +import React from 'react'; +import { UseFormReturn } from 'react-hook-form'; +import { FormField, FormItem, FormLabel, FormControl, FormDescription, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Edit, KeyRound, Loader2, CircleAlert } from 'lucide-react'; + +// Types needed from parent +interface GithubRepo { + id: number; + name: string; + full_name: string; + private: boolean; + url: string; + description: string | null; + last_updated: string | null; +} +type GithubPatFormValues = { github_pat: string; }; +type EditMode = 'viewing' | 'editing_repos'; + +interface EditGitHubConnectorConfigProps { + // State from parent + editMode: EditMode; + originalPat: string; + currentSelectedRepos: string[]; + fetchedRepos: GithubRepo[] | null; + newSelectedRepos: string[]; + isFetchingRepos: boolean; + // Forms from parent + patForm: UseFormReturn; + // Handlers from parent + setEditMode: (mode: EditMode) => void; + handleFetchRepositories: (values: GithubPatFormValues) => Promise; + handleRepoSelectionChange: (repoFullName: string, checked: boolean) => void; + setNewSelectedRepos: React.Dispatch>; + setFetchedRepos: React.Dispatch>; +} + +export function EditGitHubConnectorConfig({ + editMode, + originalPat, + currentSelectedRepos, + fetchedRepos, + newSelectedRepos, + isFetchingRepos, + patForm, + setEditMode, + handleFetchRepositories, + handleRepoSelectionChange, + setNewSelectedRepos, + setFetchedRepos +}: EditGitHubConnectorConfigProps) { + + return ( +
+

Repository Selection & Access

+ + {/* Viewing Mode */} + {editMode === 'viewing' && ( +
+ Currently Indexed Repositories: + {currentSelectedRepos.length > 0 ? ( +
    + {currentSelectedRepos.map(repo =>
  • {repo}
    • )} +
+ ) : ( +

(No repositories currently selected)

+ )} + + To change repo selections or update the PAT, click above. +
+ )} + + {/* Editing Mode */} + {editMode === 'editing_repos' && ( +
+ {/* PAT Input */} +
+ ( + + GitHub PAT + + Enter PAT to fetch/update repos or if you need to update the stored token. + + + )} + /> + +
+ + {/* Repo List */} + {isFetchingRepos && } + {!isFetchingRepos && fetchedRepos !== null && ( + fetchedRepos.length === 0 ? ( + + + No Repositories Found + Check PAT & permissions. + + ) : ( +
+ Select Repositories to Index ({newSelectedRepos.length} selected): +
+ {fetchedRepos.map((repo) => ( +
+ handleRepoSelectionChange(repo.full_name, !!checked)} + /> + +
+ ))} +
+
+ ) + )} + +
+ )} +
+ ); +} diff --git a/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx b/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx new file mode 100644 index 000000000..c0c803209 --- /dev/null +++ b/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { Control } from 'react-hook-form'; +import { FormField, FormItem, FormLabel, FormControl, FormDescription, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { KeyRound } from 'lucide-react'; + +// Assuming EditConnectorFormValues is defined elsewhere or passed as generic +interface EditSimpleTokenFormProps { + control: Control; + fieldName: string; // e.g., "SLACK_BOT_TOKEN" + fieldLabel: string; // e.g., "Slack Bot Token" + fieldDescription: string; + placeholder?: string; +} + +export function EditSimpleTokenForm({ + control, + fieldName, + fieldLabel, + fieldDescription, + placeholder +}: EditSimpleTokenFormProps) { + return ( + ( + + {fieldLabel} + + {fieldDescription} + + + )} + /> + ); +} From 53436370d6fdbb707e37ab86614a535a7a3ea2f8 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Wed, 16 Apr 2025 21:25:29 -0700 Subject: [PATCH 16/31] refactor edit connector page for smaller file size --- .../connectors/[connector_id]/edit/page.tsx | 400 +++--------------- .../components/editConnector/types.ts | 33 ++ surfsense_web/hooks/useConnectorEditPage.ts | 210 +++++++++ surfsense_web/lib/connectors/utils.ts | 11 + 4 files changed, 312 insertions(+), 342 deletions(-) create mode 100644 surfsense_web/components/editConnector/types.ts create mode 100644 surfsense_web/hooks/useConnectorEditPage.ts create mode 100644 surfsense_web/lib/connectors/utils.ts diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx index 2032aab05..00824bc56 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx @@ -1,372 +1,90 @@ "use client"; -import { useState, useEffect } from "react"; +import React, { useEffect } from 'react'; import { useRouter, useParams } from "next/navigation"; import { motion } from "framer-motion"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { useForm } from "react-hook-form"; -import * as z from "zod"; import { toast } from "sonner"; -import { ArrowLeft, Check, Loader2, Github, } from "lucide-react"; +import { ArrowLeft, Check, Loader2, Github } from "lucide-react"; -import { useSearchSourceConnectors, SearchSourceConnector } from "@/hooks/useSearchSourceConnectors"; -import { - Form, -} from "@/components/ui/form"; +import { Form } from "@/components/ui/form"; import { Button } from "@/components/ui/button"; -import { - Card, - CardContent, - CardDescription, - CardFooter, - CardHeader, - CardTitle, -} from "@/components/ui/card"; +import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } from "@/components/ui/card"; + +// Import Utils, Types, Hook, and Components +import { getConnectorTypeDisplay } from '@/lib/connectors/utils'; +import { useConnectorEditPage } from '@/hooks/useConnectorEditPage'; import { EditConnectorLoadingSkeleton } from "@/components/editConnector/EditConnectorLoadingSkeleton"; import { EditConnectorNameForm } from "@/components/editConnector/EditConnectorNameForm"; import { EditGitHubConnectorConfig } from "@/components/editConnector/EditGitHubConnectorConfig"; import { EditSimpleTokenForm } from "@/components/editConnector/EditSimpleTokenForm"; - -// Helper function to get connector type display name (copied from manage page) -const getConnectorTypeDisplay = (type: string): string => { - const typeMap: Record = { - "SERPER_API": "Serper API", - "TAVILY_API": "Tavily API", - "SLACK_CONNECTOR": "Slack", - "NOTION_CONNECTOR": "Notion", - "GITHUB_CONNECTOR": "GitHub", - }; - return typeMap[type] || type; -}; - -// Schema for PAT input when editing GitHub repos (remains separate) -const githubPatSchema = z.object({ - github_pat: z.string() - .min(20, { message: "GitHub Personal Access Token seems too short." }) - .refine(pat => pat.startsWith('ghp_') || pat.startsWith('github_pat_'), { - message: "GitHub PAT should start with 'ghp_' or 'github_pat_'", - }), -}); -type GithubPatFormValues = z.infer; - -// Updated schema for main edit form - includes optional fields for other connector configs -const editConnectorSchema = z.object({ - name: z.string().min(3, { message: "Connector name must be at least 3 characters." }), - // Add optional fields for other connector types' configs - SLACK_BOT_TOKEN: z.string().optional(), - NOTION_INTEGRATION_TOKEN: z.string().optional(), - SERPER_API_KEY: z.string().optional(), - TAVILY_API_KEY: z.string().optional(), - // GITHUB_PAT is handled separately via patForm for repo editing flow -}); -type EditConnectorFormValues = z.infer; - -interface GithubRepo { - id: number; - name: string; - full_name: string; - private: boolean; - url: string; - description: string | null; - last_updated: string | null; -} - -type EditMode = 'viewing' | 'editing_repos'; // Only relevant for GitHub - -export default function EditConnectorPage() { // Renamed for clarity +export default function EditConnectorPage() { const router = useRouter(); const params = useParams(); const searchSpaceId = params.search_space_id as string; - const connectorId = parseInt(params.connector_id as string, 10); + // Ensure connectorId is parsed safely + const connectorIdParam = params.connector_id as string; + const connectorId = connectorIdParam ? parseInt(connectorIdParam, 10) : NaN; - const { connectors, updateConnector, isLoading: connectorsLoading } = useSearchSourceConnectors(); + // Use the custom hook to manage state and logic + const { + connectorsLoading, + connector, + isSaving, + editForm, + patForm, // Needed for GitHub child component + handleSaveChanges, + // GitHub specific props for the child component + editMode, + setEditMode, // Pass down if needed by GitHub component + originalPat, + currentSelectedRepos, + fetchedRepos, + setFetchedRepos, + newSelectedRepos, + setNewSelectedRepos, + isFetchingRepos, + handleFetchRepositories, + handleRepoSelectionChange, + } = useConnectorEditPage(connectorId, searchSpaceId); - const [connector, setConnector] = useState(null); - const [originalConfig, setOriginalConfig] = useState | null>(null); // Store original config object - - // GitHub specific state (only used if connector type is GitHub) - const [currentSelectedRepos, setCurrentSelectedRepos] = useState([]); - const [originalPat, setOriginalPat] = useState(""); - const [editMode, setEditMode] = useState('viewing'); - const [fetchedRepos, setFetchedRepos] = useState(null); - const [newSelectedRepos, setNewSelectedRepos] = useState([]); - const [isFetchingRepos, setIsFetchingRepos] = useState(false); - - const [isSaving, setIsSaving] = useState(false); - - // Form for GitHub PAT input (only used for GitHub repo editing) - const patForm = useForm({ - resolver: zodResolver(githubPatSchema), - defaultValues: { github_pat: "" }, - }); - - // Main form for connector details (name + simple config fields) - const editForm = useForm({ - resolver: zodResolver(editConnectorSchema), - defaultValues: { - name: "", - SLACK_BOT_TOKEN: "", - NOTION_INTEGRATION_TOKEN: "", - SERPER_API_KEY: "", - TAVILY_API_KEY: "", - }, - }); - - // Effect to load connector data + // Redirect if connectorId is not a valid number after parsing useEffect(() => { - if (!connectorsLoading && connectors.length > 0 && !connector) { - const currentConnector = connectors.find(c => c.id === connectorId); - if (currentConnector) { - setConnector(currentConnector); - setOriginalConfig(currentConnector.config || {}); // Store original config - - // Reset main form with common and type-specific fields - editForm.reset({ - name: currentConnector.name, - SLACK_BOT_TOKEN: currentConnector.config?.SLACK_BOT_TOKEN || "", - NOTION_INTEGRATION_TOKEN: currentConnector.config?.NOTION_INTEGRATION_TOKEN || "", - SERPER_API_KEY: currentConnector.config?.SERPER_API_KEY || "", - TAVILY_API_KEY: currentConnector.config?.TAVILY_API_KEY || "", - }); - - // If GitHub, set up GitHub-specific state - if (currentConnector.connector_type === 'GITHUB_CONNECTOR') { - const savedRepos = currentConnector.config?.repo_full_names || []; - const savedPat = currentConnector.config?.GITHUB_PAT || ""; - setCurrentSelectedRepos(savedRepos); - setNewSelectedRepos(savedRepos); - setOriginalPat(savedPat); - patForm.reset({ github_pat: savedPat }); - setEditMode('viewing'); // Start in viewing mode for repos - } - } else { - toast.error("Connector not found."); - router.push(`/dashboard/${searchSpaceId}/connectors`); - } + if (isNaN(connectorId)) { + toast.error("Invalid Connector ID."); + router.push(`/dashboard/${searchSpaceId}/connectors`); } - }, [connectorId, connectors, connectorsLoading, router, searchSpaceId, connector, editForm, patForm]); - - // Fetch repositories using the entered PAT - const handleFetchRepositories = async (values: GithubPatFormValues) => { - setIsFetchingRepos(true); - setFetchedRepos(null); - try { - const token = localStorage.getItem('surfsense_bearer_token'); - if (!token) throw new Error('No authentication token found'); - - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/github/repositories/`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${token}` - }, - body: JSON.stringify({ github_pat: values.github_pat }) - } - ); - if (!response.ok) { - const errorData = await response.json(); - throw new Error(errorData.detail || `Failed to fetch repositories: ${response.statusText}`); - } - const data: GithubRepo[] = await response.json(); - setFetchedRepos(data); - setNewSelectedRepos(currentSelectedRepos); - toast.success(`Found ${data.length} repositories. Select which ones to index.`); - } catch (error) { - console.error("Error fetching GitHub repositories:", error); - toast.error(error instanceof Error ? error.message : "Failed to fetch repositories."); - } finally { - setIsFetchingRepos(false); - } - }; - - // Handle checkbox changes during editing - const handleRepoSelectionChange = (repoFullName: string, checked: boolean) => { - setNewSelectedRepos(prev => - checked - ? [...prev, repoFullName] - : prev.filter(name => name !== repoFullName) - ); - }; - - // Save changes - updated to handle different connector types - const handleSaveChanges = async (formData: EditConnectorFormValues) => { - if (!connector || !originalConfig) return; - - setIsSaving(true); - const updatePayload: Partial = {}; - let configChanged = false; - let newConfig: Record | null = null; - - // 1. Check if name changed - if (formData.name !== connector.name) { - updatePayload.name = formData.name; - } - - // 2. Check for config changes based on connector type - switch (connector.connector_type) { - case 'GITHUB_CONNECTOR': - const currentPatInForm = patForm.getValues('github_pat'); - const patChanged = currentPatInForm !== originalPat; - const initialRepoSet = new Set(currentSelectedRepos); - const newRepoSet = new Set(newSelectedRepos); - const reposChanged = initialRepoSet.size !== newRepoSet.size || ![...initialRepoSet].every(repo => newRepoSet.has(repo)); - - if (patChanged || (editMode === 'editing_repos' && reposChanged && fetchedRepos !== null)) { - if (!currentPatInForm || !(currentPatInForm.startsWith('ghp_') || currentPatInForm.startsWith('github_pat_'))) { - toast.error("Invalid GitHub PAT format. Cannot save."); - setIsSaving(false); return; - } - newConfig = { - GITHUB_PAT: currentPatInForm, - repo_full_names: newSelectedRepos, - }; - if (reposChanged && newSelectedRepos.length === 0) { - toast.warning("Warning: No repositories selected."); - } - } - break; - - case 'SLACK_CONNECTOR': - if (formData.SLACK_BOT_TOKEN !== originalConfig.SLACK_BOT_TOKEN) { - if (!formData.SLACK_BOT_TOKEN) { - toast.error("Slack Bot Token cannot be empty."); setIsSaving(false); return; - } - newConfig = { SLACK_BOT_TOKEN: formData.SLACK_BOT_TOKEN }; - } - break; - - case 'NOTION_CONNECTOR': - if (formData.NOTION_INTEGRATION_TOKEN !== originalConfig.NOTION_INTEGRATION_TOKEN) { - if (!formData.NOTION_INTEGRATION_TOKEN) { - toast.error("Notion Integration Token cannot be empty."); setIsSaving(false); return; - } - newConfig = { NOTION_INTEGRATION_TOKEN: formData.NOTION_INTEGRATION_TOKEN }; - } - break; - - case 'SERPER_API': - if (formData.SERPER_API_KEY !== originalConfig.SERPER_API_KEY) { - if (!formData.SERPER_API_KEY) { - toast.error("Serper API Key cannot be empty."); setIsSaving(false); return; - } - newConfig = { SERPER_API_KEY: formData.SERPER_API_KEY }; - } - break; - - case 'TAVILY_API': - if (formData.TAVILY_API_KEY !== originalConfig.TAVILY_API_KEY) { - if (!formData.TAVILY_API_KEY) { - toast.error("Tavily API Key cannot be empty."); setIsSaving(false); return; - } - newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY }; - } - break; - - // Add cases for other connector types if necessary - } - - // If config was determined to have changed, add it to the payload - if (newConfig !== null) { - updatePayload.config = newConfig; - configChanged = true; - } - - // 3. Check if there are actual changes to save - if (Object.keys(updatePayload).length === 0) { - toast.info("No changes detected."); - setIsSaving(false); - if (connector.connector_type === 'GITHUB_CONNECTOR') { - setEditMode('viewing'); - patForm.reset({ github_pat: originalPat }); - } - return; - } - - // 4. Proceed with update API call - try { - await updateConnector(connectorId, updatePayload); - toast.success("Connector updated successfully!"); - - // Update local state after successful save - const newlySavedConfig = updatePayload.config || originalConfig; - setOriginalConfig(newlySavedConfig); - if (updatePayload.name) { - setConnector(prev => prev ? { ...prev, name: updatePayload.name!, config: newlySavedConfig } : null); - editForm.setValue('name', updatePayload.name); - } else { - setConnector(prev => prev ? { ...prev, config: newlySavedConfig } : null); - } - - if (connector.connector_type === 'GITHUB_CONNECTOR' && configChanged) { - const savedGitHubConfig = newlySavedConfig as { GITHUB_PAT?: string; repo_full_names?: string[] }; - setCurrentSelectedRepos(savedGitHubConfig.repo_full_names || []); - setOriginalPat(savedGitHubConfig.GITHUB_PAT || ""); - setNewSelectedRepos(savedGitHubConfig.repo_full_names || []); - patForm.reset({ github_pat: savedGitHubConfig.GITHUB_PAT || "" }); - } else if (connector.connector_type === 'SLACK_CONNECTOR' && configChanged) { - editForm.setValue('SLACK_BOT_TOKEN', newlySavedConfig.SLACK_BOT_TOKEN || ""); - } // Add similar blocks for Notion, Serper, Tavily - else if (connector.connector_type === 'NOTION_CONNECTOR' && configChanged) { - editForm.setValue('NOTION_INTEGRATION_TOKEN', newlySavedConfig.NOTION_INTEGRATION_TOKEN || ""); - } else if (connector.connector_type === 'SERPER_API' && configChanged) { - editForm.setValue('SERPER_API_KEY', newlySavedConfig.SERPER_API_KEY || ""); - } else if (connector.connector_type === 'TAVILY_API' && configChanged) { - editForm.setValue('TAVILY_API_KEY', newlySavedConfig.TAVILY_API_KEY || ""); - } - - // Reset GitHub specific edit state - if (connector.connector_type === 'GITHUB_CONNECTOR') { - setEditMode('viewing'); - setFetchedRepos(null); - } - - } catch (error) { - console.error("Error updating connector:", error); - toast.error(error instanceof Error ? error.message : "Failed to update connector."); - } finally { - setIsSaving(false); - } - }; + }, [connectorId, router, searchSpaceId]); + // Loading State if (connectorsLoading || !connector) { + // Handle NaN case before showing skeleton + if (isNaN(connectorId)) return null; return ; } + // Main Render using data/handlers from the hook return (
- - + - {/* Title can be dynamic based on type */} - {/* TODO: Make icon dynamic */} + {/* TODO: Dynamic icon */} Edit {getConnectorTypeDisplay(connector.connector_type)} Connector - - Modify the connector name and configuration. - + Modify connector name and configuration. - + + {/* Pass hook's handleSaveChanges */} - {/* Name Component */} + {/* Pass form control from hook */}
@@ -376,14 +94,15 @@ export default function EditConnectorPage() { // Renamed for clarity {/* == GitHub == */} {connector.connector_type === 'GITHUB_CONNECTOR' && ( )} - {/* == Notion == */} {connector.connector_type === 'NOTION_CONNECTOR' && ( )} - - {/* == Serper API == */} + {/* == Serper == */} {connector.connector_type === 'SERPER_API' && ( )} - - {/* == Tavily API == */} + {/* == Tavily == */} {connector.connector_type === 'TAVILY_API' && ( pat.startsWith('ghp_') || pat.startsWith('github_pat_'), { + message: "GitHub PAT should start with 'ghp_' or 'github_pat_'", + }), +}); +export type GithubPatFormValues = z.infer; + +export const editConnectorSchema = z.object({ + name: z.string().min(3, { message: "Connector name must be at least 3 characters." }), + SLACK_BOT_TOKEN: z.string().optional(), + NOTION_INTEGRATION_TOKEN: z.string().optional(), + SERPER_API_KEY: z.string().optional(), + TAVILY_API_KEY: z.string().optional(), +}); +export type EditConnectorFormValues = z.infer; diff --git a/surfsense_web/hooks/useConnectorEditPage.ts b/surfsense_web/hooks/useConnectorEditPage.ts new file mode 100644 index 000000000..e75998ef9 --- /dev/null +++ b/surfsense_web/hooks/useConnectorEditPage.ts @@ -0,0 +1,210 @@ +import { useState, useEffect, useCallback } from 'react'; +import { useRouter } from 'next/navigation'; +import { useForm } from 'react-hook-form'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { toast } from 'sonner'; +import { useSearchSourceConnectors, SearchSourceConnector } from '@/hooks/useSearchSourceConnectors'; +import { + GithubRepo, + EditMode, + githubPatSchema, + editConnectorSchema, + GithubPatFormValues, + EditConnectorFormValues +} from '@/components/editConnector/types'; + +export function useConnectorEditPage(connectorId: number, searchSpaceId: string) { + const router = useRouter(); + const { connectors, updateConnector, isLoading: connectorsLoading } = useSearchSourceConnectors(); + + // State managed by the hook + const [connector, setConnector] = useState(null); + const [originalConfig, setOriginalConfig] = useState | null>(null); + const [isSaving, setIsSaving] = useState(false); + const [currentSelectedRepos, setCurrentSelectedRepos] = useState([]); + const [originalPat, setOriginalPat] = useState(""); + const [editMode, setEditMode] = useState('viewing'); + const [fetchedRepos, setFetchedRepos] = useState(null); + const [newSelectedRepos, setNewSelectedRepos] = useState([]); + const [isFetchingRepos, setIsFetchingRepos] = useState(false); + + // Forms managed by the hook + const patForm = useForm({ + resolver: zodResolver(githubPatSchema), + defaultValues: { github_pat: "" }, + }); + const editForm = useForm({ + resolver: zodResolver(editConnectorSchema), + defaultValues: { name: "", SLACK_BOT_TOKEN: "", NOTION_INTEGRATION_TOKEN: "", SERPER_API_KEY: "", TAVILY_API_KEY: "" }, + }); + + // Effect to load initial data + useEffect(() => { + if (!connectorsLoading && connectors.length > 0 && !connector) { + const currentConnector = connectors.find(c => c.id === connectorId); + if (currentConnector) { + setConnector(currentConnector); + const config = currentConnector.config || {}; + setOriginalConfig(config); + editForm.reset({ + name: currentConnector.name, + SLACK_BOT_TOKEN: config.SLACK_BOT_TOKEN || "", + NOTION_INTEGRATION_TOKEN: config.NOTION_INTEGRATION_TOKEN || "", + SERPER_API_KEY: config.SERPER_API_KEY || "", + TAVILY_API_KEY: config.TAVILY_API_KEY || "", + }); + if (currentConnector.connector_type === 'GITHUB_CONNECTOR') { + const savedRepos = config.repo_full_names || []; + const savedPat = config.GITHUB_PAT || ""; + setCurrentSelectedRepos(savedRepos); + setNewSelectedRepos(savedRepos); + setOriginalPat(savedPat); + patForm.reset({ github_pat: savedPat }); + setEditMode('viewing'); + } + } else { + toast.error("Connector not found."); + router.push(`/dashboard/${searchSpaceId}/connectors`); + } + } + }, [connectorId, connectors, connectorsLoading, router, searchSpaceId, connector, editForm, patForm]); + + // Handlers managed by the hook + const handleFetchRepositories = useCallback(async (values: GithubPatFormValues) => { + setIsFetchingRepos(true); + setFetchedRepos(null); + try { + const token = localStorage.getItem('surfsense_bearer_token'); + if (!token) throw new Error('No auth token'); + const response = await fetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/github/repositories/`, + { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${token}` }, body: JSON.stringify({ github_pat: values.github_pat }) } + ); + if (!response.ok) { const err = await response.json(); throw new Error(err.detail || 'Fetch failed'); } + const data: GithubRepo[] = await response.json(); + setFetchedRepos(data); + setNewSelectedRepos(currentSelectedRepos); + toast.success(`Found ${data.length} repos.`); + } catch (error) { + console.error("Error fetching GitHub repositories:", error); + toast.error(error instanceof Error ? error.message : "Failed to fetch repositories."); + } finally { setIsFetchingRepos(false); } + }, [currentSelectedRepos]); // Added dependency + + const handleRepoSelectionChange = useCallback((repoFullName: string, checked: boolean) => { + setNewSelectedRepos(prev => checked ? [...prev, repoFullName] : prev.filter(name => name !== repoFullName)); + }, []); + + const handleSaveChanges = useCallback(async (formData: EditConnectorFormValues) => { + if (!connector || !originalConfig) return; + setIsSaving(true); + const updatePayload: Partial = {}; + let configChanged = false; + let newConfig: Record | null = null; + + if (formData.name !== connector.name) { + updatePayload.name = formData.name; + } + + switch (connector.connector_type) { + case 'GITHUB_CONNECTOR': + const currentPatInForm = patForm.getValues('github_pat'); + const patChanged = currentPatInForm !== originalPat; + const initialRepoSet = new Set(currentSelectedRepos); + const newRepoSet = new Set(newSelectedRepos); + const reposChanged = initialRepoSet.size !== newRepoSet.size || ![...initialRepoSet].every(repo => newRepoSet.has(repo)); + if (patChanged || (editMode === 'editing_repos' && reposChanged && fetchedRepos !== null)) { + if (!currentPatInForm || !(currentPatInForm.startsWith('ghp_') || currentPatInForm.startsWith('github_pat_'))) { + toast.error("Invalid GitHub PAT format. Cannot save."); setIsSaving(false); return; + } + newConfig = { GITHUB_PAT: currentPatInForm, repo_full_names: newSelectedRepos }; + if (reposChanged && newSelectedRepos.length === 0) { toast.warning("Warning: No repositories selected."); } + } + break; + case 'SLACK_CONNECTOR': + if (formData.SLACK_BOT_TOKEN !== originalConfig.SLACK_BOT_TOKEN) { + if (!formData.SLACK_BOT_TOKEN) { toast.error("Slack Token empty."); setIsSaving(false); return; } + newConfig = { SLACK_BOT_TOKEN: formData.SLACK_BOT_TOKEN }; + } + break; + // ... other cases ... + case 'NOTION_CONNECTOR': + if (formData.NOTION_INTEGRATION_TOKEN !== originalConfig.NOTION_INTEGRATION_TOKEN) { + if (!formData.NOTION_INTEGRATION_TOKEN) { toast.error("Notion Token empty."); setIsSaving(false); return; } + newConfig = { NOTION_INTEGRATION_TOKEN: formData.NOTION_INTEGRATION_TOKEN }; + } + break; + case 'SERPER_API': + if (formData.SERPER_API_KEY !== originalConfig.SERPER_API_KEY) { + if (!formData.SERPER_API_KEY) { toast.error("Serper Key empty."); setIsSaving(false); return; } + newConfig = { SERPER_API_KEY: formData.SERPER_API_KEY }; + } + break; + case 'TAVILY_API': + if (formData.TAVILY_API_KEY !== originalConfig.TAVILY_API_KEY) { + if (!formData.TAVILY_API_KEY) { toast.error("Tavily Key empty."); setIsSaving(false); return; } + newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY }; + } + break; + } + + if (newConfig !== null) { + updatePayload.config = newConfig; + configChanged = true; + } + + if (Object.keys(updatePayload).length === 0) { + toast.info("No changes detected."); + setIsSaving(false); + if (connector.connector_type === 'GITHUB_CONNECTOR') { setEditMode('viewing'); patForm.reset({ github_pat: originalPat }); } + return; + } + + try { + await updateConnector(connectorId, updatePayload); + toast.success("Connector updated!"); + const newlySavedConfig = updatePayload.config || originalConfig; + setOriginalConfig(newlySavedConfig); + if (updatePayload.name) { + setConnector(prev => prev ? { ...prev, name: updatePayload.name!, config: newlySavedConfig } : null); + } + if (connector.connector_type === 'GITHUB_CONNECTOR' && configChanged) { + const savedGitHubConfig = newlySavedConfig as { GITHUB_PAT?: string; repo_full_names?: string[] }; + setCurrentSelectedRepos(savedGitHubConfig.repo_full_names || []); + setOriginalPat(savedGitHubConfig.GITHUB_PAT || ""); + setNewSelectedRepos(savedGitHubConfig.repo_full_names || []); + patForm.reset({ github_pat: savedGitHubConfig.GITHUB_PAT || "" }); + } + if (connector.connector_type === 'GITHUB_CONNECTOR') { + setEditMode('viewing'); + setFetchedRepos(null); + } + // Resetting simple form values is handled by useEffect if connector state updates + } catch (error) { + console.error("Error updating connector:", error); + toast.error(error instanceof Error ? error.message : "Failed to update connector."); + } finally { setIsSaving(false); } + }, [connector, originalConfig, updateConnector, connectorId, patForm, originalPat, currentSelectedRepos, newSelectedRepos, editMode, fetchedRepos]); // Added dependencies + + // Return values needed by the component + return { + connectorsLoading, + connector, + isSaving, + editForm, + patForm, + handleSaveChanges, + // GitHub specific props + editMode, + setEditMode, + originalPat, + currentSelectedRepos, + fetchedRepos, + setFetchedRepos, + newSelectedRepos, + setNewSelectedRepos, + isFetchingRepos, + handleFetchRepositories, + handleRepoSelectionChange, + }; +} diff --git a/surfsense_web/lib/connectors/utils.ts b/surfsense_web/lib/connectors/utils.ts new file mode 100644 index 000000000..f0a6dab45 --- /dev/null +++ b/surfsense_web/lib/connectors/utils.ts @@ -0,0 +1,11 @@ +// Helper function to get connector type display name +export const getConnectorTypeDisplay = (type: string): string => { + const typeMap: Record = { + "SERPER_API": "Serper API", + "TAVILY_API": "Tavily API", + "SLACK_CONNECTOR": "Slack", + "NOTION_CONNECTOR": "Notion", + "GITHUB_CONNECTOR": "GitHub", + }; + return typeMap[type] || type; +}; From 32c721113cda58763ec3e28f6b4809014952f416 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Wed, 16 Apr 2025 22:06:50 -0700 Subject: [PATCH 17/31] update edit connectors page to support linear connector --- .../routes/search_source_connectors_routes.py | 6 +- .../connectors/[connector_id]/edit/page.tsx | 11 +++ .../components/editConnector/types.ts | 1 + surfsense_web/hooks/useConnectorEditPage.ts | 72 +++++++++++++------ surfsense_web/lib/connectors/utils.ts | 1 + 5 files changed, 68 insertions(+), 23 deletions(-) diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 6c0349687..786c19dfb 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -18,8 +18,9 @@ from app.db import get_async_session, User, SearchSourceConnector, SearchSourceC from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead, SearchSourceConnectorBase from app.users import current_active_user from app.utils.check_ownership import check_ownership -from pydantic import ValidationError +from pydantic import BaseModel, Field, ValidationError from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos, index_linear_issues +from app.connectors.github_connector import GitHubConnector from datetime import datetime, timezone, timedelta import logging @@ -28,7 +29,7 @@ logger = logging.getLogger(__name__) router = APIRouter() -# --- New Schema for GitHub PAT --- +# Use Pydantic's BaseModel here class GitHubPATRequest(BaseModel): github_pat: str = Field(..., description="GitHub Personal Access Token") @@ -104,6 +105,7 @@ async def create_search_source_connector( await session.rollback() raise except Exception as e: + logger.error(f"Failed to create search source connector: {str(e)}") await session.rollback() raise HTTPException( status_code=500, diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx index 00824bc56..d41295faa 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx @@ -149,6 +149,17 @@ export default function EditConnectorPage() { /> )} + {/* == Linear == */} + {connector.connector_type === 'LINEAR_CONNECTOR' && ( + + )} +
}> - + ); } \ No newline at end of file diff --git a/surfsense_web/components/sidebar/AppSidebarProvider.tsx b/surfsense_web/components/sidebar/AppSidebarProvider.tsx index c4306e98f..679157c1f 100644 --- a/surfsense_web/components/sidebar/AppSidebarProvider.tsx +++ b/surfsense_web/components/sidebar/AppSidebarProvider.tsx @@ -118,8 +118,8 @@ export function AppSidebarProvider({ if (typeof window === 'undefined') return; try { - // Use the API client instead of direct fetch - const chats: Chat[] = await apiClient.get('api/v1/chats/?limit=5&skip=0'); + // Use the API client instead of direct fetch - filter by current search space ID + const chats: Chat[] = await apiClient.get(`api/v1/chats/?limit=5&skip=0&search_space_id=${searchSpaceId}`); // Transform API response to the format expected by AppSidebar const formattedChats = chats.map(chat => ({ @@ -170,7 +170,7 @@ export function AppSidebarProvider({ // Clean up interval on component unmount return () => clearInterval(intervalId); - }, []); + }, [searchSpaceId]); // Handle delete chat const handleDeleteChat = async () => { diff --git a/surfsense_web/hooks/use-documents.ts b/surfsense_web/hooks/use-documents.ts index 923d6e04e..cfe2b0546 100644 --- a/surfsense_web/hooks/use-documents.ts +++ b/surfsense_web/hooks/use-documents.ts @@ -22,7 +22,7 @@ export function useDocuments(searchSpaceId: number) { try { setLoading(true); const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}`, { headers: { Authorization: `Bearer ${localStorage.getItem('surfsense_bearer_token')}`, @@ -57,7 +57,7 @@ export function useDocuments(searchSpaceId: number) { setLoading(true); try { const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}`, { headers: { Authorization: `Bearer ${localStorage.getItem('surfsense_bearer_token')}`, From 34300ead029c6ccd445dc6d516d5ee8b24fd95f9 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sat, 19 Apr 2025 23:25:06 -0700 Subject: [PATCH 19/31] feat: Initial version of SurfSense own LangGraph Agent. --- surfsense_backend/app/agents/__init__.py | 1 + .../app/agents/researcher/configuration.py | 30 ++ .../app/agents/researcher/graph.py | 43 ++ .../app/agents/researcher/nodes.py | 476 ++++++++++++++++++ .../app/agents/researcher/prompts.py | 91 ++++ .../app/agents/researcher/state.py | 30 ++ .../sub_section_writer/configuration.py | 7 +- .../researcher/sub_section_writer/graph.py | 12 +- .../researcher/sub_section_writer/nodes.py | 221 +++----- .../researcher/sub_section_writer/prompts.py | 4 + .../researcher/sub_section_writer/state.py | 2 +- .../app/agents/researcher/test_researcher.py | 132 +++++ surfsense_backend/app/config/__init__.py | 2 - 13 files changed, 884 insertions(+), 167 deletions(-) create mode 100644 surfsense_backend/app/agents/researcher/configuration.py create mode 100644 surfsense_backend/app/agents/researcher/graph.py create mode 100644 surfsense_backend/app/agents/researcher/nodes.py create mode 100644 surfsense_backend/app/agents/researcher/prompts.py create mode 100644 surfsense_backend/app/agents/researcher/state.py create mode 100644 surfsense_backend/app/agents/researcher/test_researcher.py diff --git a/surfsense_backend/app/agents/__init__.py b/surfsense_backend/app/agents/__init__.py index e69de29bb..944afebc6 100644 --- a/surfsense_backend/app/agents/__init__.py +++ b/surfsense_backend/app/agents/__init__.py @@ -0,0 +1 @@ +"""This is upcoming research agent. Work in progress.""" \ No newline at end of file diff --git a/surfsense_backend/app/agents/researcher/configuration.py b/surfsense_backend/app/agents/researcher/configuration.py new file mode 100644 index 000000000..8ba3849a3 --- /dev/null +++ b/surfsense_backend/app/agents/researcher/configuration.py @@ -0,0 +1,30 @@ +"""Define the configurable parameters for the agent.""" + +from __future__ import annotations + +from dataclasses import dataclass, fields +from typing import Optional, List, Any + +from langchain_core.runnables import RunnableConfig + + +@dataclass(kw_only=True) +class Configuration: + """The configuration for the agent.""" + + # Input parameters provided at invocation + user_query: str + num_sections: int + connectors_to_search: List[str] + user_id: str + search_space_id: int + + + @classmethod + def from_runnable_config( + cls, config: Optional[RunnableConfig] = None + ) -> Configuration: + """Create a Configuration instance from a RunnableConfig object.""" + configurable = (config.get("configurable") or {}) if config else {} + _fields = {f.name for f in fields(cls) if f.init} + return cls(**{k: v for k, v in configurable.items() if k in _fields}) diff --git a/surfsense_backend/app/agents/researcher/graph.py b/surfsense_backend/app/agents/researcher/graph.py new file mode 100644 index 000000000..31835da4a --- /dev/null +++ b/surfsense_backend/app/agents/researcher/graph.py @@ -0,0 +1,43 @@ +from langgraph.graph import StateGraph +from .state import State +from .nodes import write_answer_outline, process_sections +from .configuration import Configuration +from typing import TypedDict, List, Dict, Any, Optional + +# Define what keys are in our state dict +class GraphState(TypedDict): + # Intermediate data produced during workflow + answer_outline: Optional[Any] + # Final output + final_written_report: Optional[str] + +def build_graph(): + """ + Build and return the LangGraph workflow. + + This function constructs the researcher agent graph with proper state management + and node connections following LangGraph best practices. + + Returns: + A compiled LangGraph workflow + """ + # Define a new graph with state class + workflow = StateGraph(State, config_schema=Configuration) + + # Add nodes to the graph + workflow.add_node("write_answer_outline", write_answer_outline) + workflow.add_node("process_sections", process_sections) + + # Define the edges - create a linear flow + workflow.add_edge("__start__", "write_answer_outline") + workflow.add_edge("write_answer_outline", "process_sections") + workflow.add_edge("process_sections", "__end__") + + # Compile the workflow into an executable graph + graph = workflow.compile() + graph.name = "Surfsense Researcher" # This defines the custom name in LangSmith + + return graph + +# Compile the graph once when the module is loaded +graph = build_graph() diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py new file mode 100644 index 000000000..9a0f62cf7 --- /dev/null +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -0,0 +1,476 @@ +from .configuration import Configuration +from langchain_core.runnables import RunnableConfig +from .state import State +from typing import Any, Dict, List +from app.config import config as app_config +from .prompts import answer_outline_system_prompt +from langchain_core.messages import HumanMessage, SystemMessage +from pydantic import BaseModel, Field +import json +import asyncio +from .sub_section_writer.graph import graph as sub_section_writer_graph +from app.utils.connector_service import ConnectorService +from app.utils.reranker_service import RerankerService +from sqlalchemy.ext.asyncio import AsyncSession +import copy + +class Section(BaseModel): + """A section in the answer outline.""" + section_id: int = Field(..., description="The zero-based index of the section") + section_title: str = Field(..., description="The title of the section") + questions: List[str] = Field(..., description="Questions to research for this section") + +class AnswerOutline(BaseModel): + """The complete answer outline with all sections.""" + answer_outline: List[Section] = Field(..., description="List of sections in the answer outline") + +async def write_answer_outline(state: State, config: RunnableConfig) -> Dict[str, Any]: + """ + Create a structured answer outline based on the user query. + + This node takes the user query and number of sections from the configuration and uses + an LLM to generate a comprehensive outline with logical sections and research questions + for each section. + + Returns: + Dict containing the answer outline in the "answer_outline" key for state update. + """ + + # Get configuration from runnable config + configuration = Configuration.from_runnable_config(config) + user_query = configuration.user_query + num_sections = configuration.num_sections + + # Initialize LLM + llm = app_config.strategic_llm_instance + + # Create the human message content + human_message_content = f""" + Now Please create an answer outline for the following query: + + User Query: {user_query} + Number of Sections: {num_sections} + + Remember to format your response as valid JSON exactly matching this structure: + {{ + "answer_outline": [ + {{ + "section_id": 0, + "section_title": "Section Title", + "questions": [ + "Question 1 to research for this section", + "Question 2 to research for this section" + ] + }} + ] + }} + + Your output MUST be valid JSON in exactly this format. Do not include any other text or explanation. + """ + + # Create messages for the LLM + messages = [ + SystemMessage(content=answer_outline_system_prompt), + HumanMessage(content=human_message_content) + ] + + # Call the LLM directly without using structured output + response = await llm.ainvoke(messages) + + # Parse the JSON response manually + try: + # Extract JSON content from the response + content = response.content + + # Find the JSON in the content (handle case where LLM might add additional text) + json_start = content.find('{') + json_end = content.rfind('}') + 1 + if json_start >= 0 and json_end > json_start: + json_str = content[json_start:json_end] + + # Parse the JSON string + parsed_data = json.loads(json_str) + + # Convert to Pydantic model + answer_outline = AnswerOutline(**parsed_data) + + print(f"Successfully generated answer outline with {len(answer_outline.answer_outline)} sections") + + # Return state update + return {"answer_outline": answer_outline} + else: + # If JSON structure not found, raise a clear error + raise ValueError(f"Could not find valid JSON in LLM response. Raw response: {content}") + + except (json.JSONDecodeError, ValueError) as e: + # Log the error and re-raise it + print(f"Error parsing LLM response: {str(e)}") + print(f"Raw response: {response.content}") + raise + +async def fetch_relevant_documents( + research_questions: List[str], + user_id: str, + search_space_id: int, + db_session: AsyncSession, + connectors_to_search: List[str], + top_k: int = 5 +) -> List[Dict[str, Any]]: + """ + Fetch relevant documents for research questions using the provided connectors. + + Args: + section_title: The title of the section being researched + research_questions: List of research questions to find documents for + user_id: The user ID + search_space_id: The search space ID + db_session: The database session + connectors_to_search: List of connectors to search + top_k: Number of top results to retrieve per connector per question + + Returns: + List of relevant documents + """ + # Initialize services + connector_service = ConnectorService(db_session) + reranker_service = RerankerService.get_reranker_instance(app_config) + + all_raw_documents = [] # Store all raw documents before reranking + + for user_query in research_questions: + # Use original research question as the query + reformulated_query = user_query + + # Process each selected connector + for connector in connectors_to_search: + try: + if connector == "YOUTUBE_VIDEO": + _, youtube_chunks = await connector_service.search_youtube( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k + ) + all_raw_documents.extend(youtube_chunks) + + elif connector == "EXTENSION": + _, extension_chunks = await connector_service.search_extension( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k + ) + all_raw_documents.extend(extension_chunks) + + elif connector == "CRAWLED_URL": + _, crawled_urls_chunks = await connector_service.search_crawled_urls( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k + ) + all_raw_documents.extend(crawled_urls_chunks) + + elif connector == "FILE": + _, files_chunks = await connector_service.search_files( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k + ) + all_raw_documents.extend(files_chunks) + + elif connector == "TAVILY_API": + _, tavily_chunks = await connector_service.search_tavily( + user_query=reformulated_query, + user_id=user_id, + top_k=top_k + ) + all_raw_documents.extend(tavily_chunks) + + elif connector == "SLACK_CONNECTOR": + _, slack_chunks = await connector_service.search_slack( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k + ) + all_raw_documents.extend(slack_chunks) + + elif connector == "NOTION_CONNECTOR": + _, notion_chunks = await connector_service.search_notion( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k + ) + all_raw_documents.extend(notion_chunks) + except Exception as e: + print(f"Error searching connector {connector}: {str(e)}") + # Continue with other connectors on error + continue + + # Deduplicate documents based on chunk_id or content + seen_chunk_ids = set() + seen_content_hashes = set() + deduplicated_docs = [] + + for doc in all_raw_documents: + chunk_id = doc.get("chunk_id") + content = doc.get("content", "") + content_hash = hash(content) + + # Skip if we've seen this chunk_id or content before + if (chunk_id and chunk_id in seen_chunk_ids) or content_hash in seen_content_hashes: + continue + + # Add to our tracking sets and keep this document + if chunk_id: + seen_chunk_ids.add(chunk_id) + seen_content_hashes.add(content_hash) + deduplicated_docs.append(doc) + + return deduplicated_docs + +async def process_section( + section_title: str, + user_id: str, + search_space_id: int, + session_maker, + research_questions: List[str], + connectors_to_search: List[str] +) -> str: + """ + Process a single section by sending it to the sub_section_writer graph. + + Args: + section_title: The title of the section + user_id: The user ID + search_space_id: The search space ID + session_maker: Factory for creating new database sessions + research_questions: List of research questions for this section + connectors_to_search: List of connectors to search + + Returns: + The written section content + """ + try: + # Create a new database session for this section + async with session_maker() as db_session: + # Fetch relevant documents using all research questions for this section + relevant_documents = await fetch_relevant_documents( + section_title=section_title, + research_questions=research_questions, + user_id=user_id, + search_space_id=search_space_id, + db_session=db_session, + connectors_to_search=connectors_to_search + ) + + # Fallback if no documents found + if not relevant_documents: + print(f"No relevant documents found for section: {section_title}") + relevant_documents = [ + { + "content": f"No specific information was found for: {question}" + for question in research_questions + } + ] + + # Call the sub_section_writer graph with the appropriate config + config = { + "configurable": { + "sub_section_title": section_title, + "relevant_documents": relevant_documents, + "user_id": user_id, + "search_space_id": search_space_id + } + } + + # Create the initial state with db_session + state = {"db_session": db_session} + + # Invoke the sub-section writer graph + print(f"Invoking sub_section_writer for: {section_title}") + result = await sub_section_writer_graph.ainvoke(state, config) + + # Return the final answer from the sub_section_writer + final_answer = result.get("final_answer", "No content was generated for this section.") + return final_answer + except Exception as e: + print(f"Error processing section '{section_title}': {str(e)}") + return f"Error processing section: {section_title}. Details: {str(e)}" + +async def process_sections(state: State, config: RunnableConfig) -> Dict[str, Any]: + """ + Process all sections in parallel and combine the results. + + This node takes the answer outline from the previous step, fetches relevant documents + for all questions across all sections once, and then processes each section in parallel + using the sub_section_writer graph with the shared document pool. + + Returns: + Dict containing the final written report in the "final_written_report" key. + """ + # Get configuration and answer outline from state + configuration = Configuration.from_runnable_config(config) + answer_outline = state.answer_outline + + print(f"Processing sections from outline: {answer_outline is not None}") + + if not answer_outline: + return { + "final_written_report": "No answer outline was provided. Cannot generate final report." + } + + # Create session maker from the engine or directly use the session + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.orm import sessionmaker + + # Use the engine if available, otherwise create a new session for each task + if state.engine: + session_maker = sessionmaker( + state.engine, class_=AsyncSession, expire_on_commit=False + ) + else: + # Fallback to using the same session (less optimal but will work) + print("Warning: No engine available. Using same session for all tasks.") + # Create a mock session maker that returns the same session + async def mock_session_maker(): + class ContextManager: + async def __aenter__(self): + return state.db_session + async def __aexit__(self, exc_type, exc_val, exc_tb): + pass + return ContextManager() + session_maker = mock_session_maker + + # Collect all questions from all sections + all_questions = [] + for section in answer_outline.answer_outline: + all_questions.extend(section.questions) + + print(f"Collected {len(all_questions)} questions from all sections") + + # Fetch relevant documents once for all questions + relevant_documents = [] + async with session_maker() as db_session: + + relevant_documents = await fetch_relevant_documents( + research_questions=all_questions, + user_id=configuration.user_id, + search_space_id=configuration.search_space_id, + db_session=db_session, + connectors_to_search=configuration.connectors_to_search + ) + + print(f"Fetched {len(relevant_documents)} relevant documents for all sections") + + # Create tasks to process each section in parallel with the same document set + section_tasks = [] + for section in answer_outline.answer_outline: + section_tasks.append( + process_section_with_documents( + section_title=section.section_title, + section_questions=section.questions, + user_id=configuration.user_id, + search_space_id=configuration.search_space_id, + session_maker=session_maker, + relevant_documents=relevant_documents + ) + ) + + # Run all section processing tasks in parallel + print(f"Running {len(section_tasks)} section processing tasks in parallel") + section_results = await asyncio.gather(*section_tasks, return_exceptions=True) + + # Handle any exceptions in the results + processed_results = [] + for i, result in enumerate(section_results): + if isinstance(result, Exception): + section_title = answer_outline.answer_outline[i].section_title + error_message = f"Error processing section '{section_title}': {str(result)}" + print(error_message) + processed_results.append(error_message) + else: + processed_results.append(result) + + # Combine the results into a final report with section titles + final_report = [] + for i, (section, content) in enumerate(zip(answer_outline.answer_outline, processed_results)): + section_header = f"## {section.section_title}" + final_report.append(section_header) + final_report.append(content) + final_report.append("\n") # Add spacing between sections + + # Join all sections with newlines + final_written_report = "\n".join(final_report) + print(f"Generated final report with {len(final_report)} parts") + + return { + "final_written_report": final_written_report + } + +async def process_section_with_documents( + section_title: str, + section_questions: List[str], + user_id: str, + search_space_id: int, + session_maker, + relevant_documents: List[Dict[str, Any]] +) -> str: + """ + Process a single section using pre-fetched documents. + + Args: + section_title: The title of the section + section_questions: List of research questions for this section + user_id: The user ID + search_space_id: The search space ID + session_maker: Factory for creating new database sessions + relevant_documents: Pre-fetched documents to use for this section + + Returns: + The written section content + """ + try: + # Create a new database session for this section + async with session_maker() as db_session: + # Use the provided documents + documents_to_use = relevant_documents + + # Fallback if no documents found + if not documents_to_use: + print(f"No relevant documents found for section: {section_title}") + documents_to_use = [ + { + "content": f"No specific information was found for: {question}" + for question in section_questions + } + ] + + # Call the sub_section_writer graph with the appropriate config + config = { + "configurable": { + "sub_section_title": section_title, + "sub_section_questions": section_questions, + "relevant_documents": documents_to_use, + "user_id": user_id, + "search_space_id": search_space_id + } + } + + # Create the initial state with db_session + state = {"db_session": db_session} + + # Invoke the sub-section writer graph + print(f"Invoking sub_section_writer for: {section_title}") + result = await sub_section_writer_graph.ainvoke(state, config) + + # Return the final answer from the sub_section_writer + final_answer = result.get("final_answer", "No content was generated for this section.") + return final_answer + except Exception as e: + print(f"Error processing section '{section_title}': {str(e)}") + return f"Error processing section: {section_title}. Details: {str(e)}" + diff --git a/surfsense_backend/app/agents/researcher/prompts.py b/surfsense_backend/app/agents/researcher/prompts.py new file mode 100644 index 000000000..0895590eb --- /dev/null +++ b/surfsense_backend/app/agents/researcher/prompts.py @@ -0,0 +1,91 @@ +import datetime + + +answer_outline_system_prompt = f""" +Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} + +You are an expert research assistant specializing in structuring information. Your task is to create a detailed and logical research outline based on the user's query. This outline will serve as the blueprint for generating a comprehensive research report. + + +- user_query (string): The main question or topic the user wants researched. This guides the entire outline creation process. +- num_sections (integer): The target number of distinct sections the final research report should have. This helps control the granularity and structure of the outline. + + + +A JSON object with the following structure: +{{ + "answer_outline": [ + {{ + "section_id": 0, + "section_title": "Section Title", + "questions": [ + "Question 1 to research for this section", + "Question 2 to research for this section" + ] + }} + ] +}} + + + +1. **Deconstruct the `user_query`:** Identify the key concepts, entities, and the core information requested by the user. +2. **Determine Section Themes:** Based on the analysis and the requested `num_sections`, divide the topic into distinct, logical themes or sub-topics. Each theme will become a section. Ensure these themes collectively address the `user_query` comprehensively. +3. **Develop Sections:** For *each* of the `num_sections`: + * **Assign `section_id`:** Start with 0 and increment sequentially for each section. + * **Craft `section_title`:** Write a concise, descriptive title that clearly defines the scope and focus of the section's theme. + * **Formulate Research `questions`:** Generate 2 to 5 specific, targeted research questions for this section. These questions must: + * Directly relate to the `section_title` and explore its key aspects. + * Be answerable through focused research (e.g., searching documents, databases, or knowledge bases). + * Be distinct from each other and from questions in other sections. Avoid redundancy. + * Collectively guide the gathering of information needed to fully address the section's theme. +4. **Ensure Logical Flow:** Arrange the sections in a coherent and intuitive sequence. Consider structures like: + * General background -> Specific details -> Analysis/Comparison -> Applications/Implications + * Problem definition -> Proposed solutions -> Evaluation -> Conclusion + * Chronological progression +5. **Verify Completeness and Cohesion:** Review the entire outline (`section_titles` and `questions`) to confirm that: + * All sections together provide a complete and well-structured answer to the original `user_query`. + * There are no significant overlaps or gaps in coverage between sections. +6. **Adhere Strictly to Output Format:** Ensure the final output is a valid JSON object matching the specified structure exactly, including correct field names (`answer_outline`, `section_id`, `section_title`, `questions`) and data types. + + + +User Query: "What are the health benefits of meditation?" +Number of Sections: 3 + +{{ + "answer_outline": [ + {{ + "section_id": 0, + "section_title": "Physical Health Benefits of Meditation", + "questions": [ + "What physiological changes occur in the body during meditation?", + "How does regular meditation affect blood pressure and heart health?", + "What impact does meditation have on inflammation and immune function?", + "Can meditation help with pain management, and if so, how?" + ] + }}, + {{ + "section_id": 1, + "section_title": "Mental Health Benefits of Meditation", + "questions": [ + "How does meditation affect stress and anxiety levels?", + "What changes in brain structure or function have been observed in meditation practitioners?", + "Can meditation help with depression and mood disorders?", + "What is the relationship between meditation and cognitive function?" + ] + }}, + {{ + "section_id": 2, + "section_title": "Best Meditation Practices for Maximum Benefits", + "questions": [ + "What are the most effective meditation techniques for beginners?", + "How long and how frequently should one meditate to see benefits?", + "Are there specific meditation approaches best suited for particular health goals?", + "What common obstacles prevent people from experiencing meditation benefits?" + ] + }} + ] +}} + + +""" \ No newline at end of file diff --git a/surfsense_backend/app/agents/researcher/state.py b/surfsense_backend/app/agents/researcher/state.py new file mode 100644 index 000000000..483e96ac9 --- /dev/null +++ b/surfsense_backend/app/agents/researcher/state.py @@ -0,0 +1,30 @@ +"""Define the state structures for the agent.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List, Optional, Any, Dict, Annotated +from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine +from langchain_core.messages import BaseMessage, HumanMessage +from pydantic import BaseModel + +@dataclass +class State: + """Defines the dynamic state for the agent during execution. + + This state tracks the database session and the outputs generated by the agent's nodes. + See: https://langchain-ai.github.io/langgraph/concepts/low_level/#state + for more information. + """ + # Runtime context (not part of actual graph state) + db_session: AsyncSession + engine: Optional[AsyncEngine] = None + + # Intermediate state - populated during workflow + # Using field to explicitly mark as part of state + answer_outline: Optional[Any] = field(default=None) + + # OUTPUT: Populated by agent nodes + # Using field to explicitly mark as part of state + final_written_report: Optional[str] = field(default=None) + diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py b/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py index b34090e7c..fbde94d57 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, fields -from typing import Optional, List +from typing import Optional, List, Any from langchain_core.runnables import RunnableConfig @@ -14,11 +14,10 @@ class Configuration: # Input parameters provided at invocation sub_section_title: str - sub_questions: List[str] - connectors_to_search: List[str] + sub_section_questions: List[str] + relevant_documents: List[Any] # Documents provided directly to the agent user_id: str search_space_id: int - top_k: int = 20 # Default top_k value @classmethod diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/graph.py b/surfsense_backend/app/agents/researcher/sub_section_writer/graph.py index e250cdee5..5a5a5bad2 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/graph.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/graph.py @@ -1,20 +1,18 @@ from langgraph.graph import StateGraph from .state import State -from .nodes import fetch_relevant_documents, write_sub_section +from .nodes import write_sub_section, rerank_documents from .configuration import Configuration # Define a new graph workflow = StateGraph(State, config_schema=Configuration) # Add the nodes to the graph -workflow.add_node("fetch_relevant_documents", fetch_relevant_documents) +workflow.add_node("rerank_documents", rerank_documents) workflow.add_node("write_sub_section", write_sub_section) -# Entry point -workflow.add_edge("__start__", "fetch_relevant_documents") -# Connect fetch_relevant_documents to write_sub_section -workflow.add_edge("fetch_relevant_documents", "write_sub_section") -# Exit point +# Connect the nodes +workflow.add_edge("__start__", "rerank_documents") +workflow.add_edge("rerank_documents", "write_sub_section") workflow.add_edge("write_sub_section", "__end__") # Compile the workflow into an executable graph diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py index 52fa877c9..a3384d95c 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -1,172 +1,80 @@ from .configuration import Configuration from langchain_core.runnables import RunnableConfig from .state import State -from typing import Any, Dict -from app.utils.connector_service import ConnectorService -from app.utils.reranker_service import RerankerService +from typing import Any, Dict, List from app.config import config as app_config from .prompts import citation_system_prompt from langchain_core.messages import HumanMessage, SystemMessage -async def fetch_relevant_documents(state: State, config: RunnableConfig) -> Dict[str, Any]: +async def rerank_documents(state: State, config: RunnableConfig) -> Dict[str, Any]: """ - Fetch relevant documents for the sub-section using specified connectors. + Rerank the documents based on relevance to the sub-section title. - This node retrieves documents from various data sources based on the sub-questions - derived from the sub-section title. It searches across all selected connectors - (YouTube, Extension, Crawled URLs, Files, Tavily API, Slack, Notion) and reranks - the results to provide the most relevant information for the agent workflow. + This node takes the relevant documents provided in the configuration, + reranks them using the reranker service based on the sub-section title, + and updates the state with the reranked documents. Returns: - Dict containing the reranked documents in the "relevant_documents_fetched" key. + Dict containing the reranked documents. """ - # Get configuration + # Get configuration and relevant documents configuration = Configuration.from_runnable_config(config) - - # Extract state parameters - db_session = state.db_session - - # Extract config parameters - user_id = configuration.user_id - search_space_id = configuration.search_space_id - TOP_K = configuration.top_k - - # Initialize services - connector_service = ConnectorService(db_session) - reranker_service = RerankerService.get_reranker_instance(app_config) + documents = configuration.relevant_documents + sub_section_questions = configuration.sub_section_questions - all_raw_documents = [] # Store all raw documents before reranking + # If no documents were provided, return empty list + if not documents or len(documents) == 0: + return { + "reranked_documents": [] + } - for user_query in configuration.sub_questions: - # Reformulate query (optional, consider if needed for each sub-question) - # reformulated_query = await QueryService.reformulate_query(user_query) - reformulated_query = user_query # Using original sub-question for now - - # Process each selected connector - for connector in configuration.connectors_to_search: - if connector == "YOUTUBE_VIDEO": - _, youtube_chunks = await connector_service.search_youtube( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - all_raw_documents.extend(youtube_chunks) - - elif connector == "EXTENSION": - _, extension_chunks = await connector_service.search_extension( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - all_raw_documents.extend(extension_chunks) - - elif connector == "CRAWLED_URL": - _, crawled_urls_chunks = await connector_service.search_crawled_urls( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - all_raw_documents.extend(crawled_urls_chunks) - - elif connector == "FILE": - _, files_chunks = await connector_service.search_files( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - all_raw_documents.extend(files_chunks) - - elif connector == "TAVILY_API": - _, tavily_chunks = await connector_service.search_tavily( - user_query=reformulated_query, - user_id=user_id, - top_k=TOP_K - ) - all_raw_documents.extend(tavily_chunks) - - elif connector == "SLACK_CONNECTOR": - _, slack_chunks = await connector_service.search_slack( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - all_raw_documents.extend(slack_chunks) - - elif connector == "NOTION_CONNECTOR": - _, notion_chunks = await connector_service.search_notion( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - all_raw_documents.extend(notion_chunks) + # Get reranker service from app config + reranker_service = getattr(app_config, "reranker_service", None) - # If we have documents and a reranker is available, rerank them - # Deduplicate documents based on chunk_id or content to avoid processing duplicates - seen_chunk_ids = set() - seen_content_hashes = set() - deduplicated_docs = [] + # Use documents as is if no reranker service is available + reranked_docs = documents - for doc in all_raw_documents: - chunk_id = doc.get("chunk_id") - content = doc.get("content", "") - content_hash = hash(content) - - # Skip if we've seen this chunk_id or content before - if (chunk_id and chunk_id in seen_chunk_ids) or content_hash in seen_content_hashes: - continue + if reranker_service: + try: + # Use the sub-section questions for reranking context + rerank_query = "\n".join(sub_section_questions) - # Add to our tracking sets and keep this document - if chunk_id: - seen_chunk_ids.add(chunk_id) - seen_content_hashes.add(content_hash) - deduplicated_docs.append(doc) + # Convert documents to format expected by reranker if needed + reranker_input_docs = [ + { + "chunk_id": doc.get("chunk_id", f"chunk_{i}"), + "content": doc.get("content", ""), + "score": doc.get("score", 0.0), + "document": { + "id": doc.get("document", {}).get("id", ""), + "title": doc.get("document", {}).get("title", ""), + "document_type": doc.get("document", {}).get("document_type", ""), + "metadata": doc.get("document", {}).get("metadata", {}) + } + } for i, doc in enumerate(documents) + ] + + # Rerank documents using the section title + reranked_docs = reranker_service.rerank_documents(rerank_query, reranker_input_docs) + + # Sort by score in descending order + reranked_docs.sort(key=lambda x: x.get("score", 0), reverse=True) + + print(f"Reranked {len(reranked_docs)} documents for section: {configuration.sub_section_title}") + except Exception as e: + print(f"Error during reranking: {str(e)}") + # Use original docs if reranking fails - # Use deduplicated documents for reranking - reranked_docs = deduplicated_docs - if deduplicated_docs and reranker_service: - # Use the main sub_section_title for reranking context - rerank_query = configuration.sub_section_title - - # Convert documents to format expected by reranker - reranker_input_docs = [ - { - "chunk_id": doc.get("chunk_id", f"chunk_{i}"), - "content": doc.get("content", ""), - "score": doc.get("score", 0.0), - "document": { - "id": doc.get("document", {}).get("id", ""), - "title": doc.get("document", {}).get("title", ""), - "document_type": doc.get("document", {}).get("document_type", ""), - "metadata": doc.get("document", {}).get("metadata", {}) - } - } for i, doc in enumerate(deduplicated_docs) - ] - - # Rerank documents using the main title query - reranked_docs = reranker_service.rerank_documents(rerank_query, reranker_input_docs) - - # Sort by score in descending order - reranked_docs.sort(key=lambda x: x.get("score", 0), reverse=True) - - # Update state with fetched documents return { - "relevant_documents_fetched": reranked_docs + "reranked_documents": reranked_docs } - - async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, Any]: """ - Write the sub-section using the fetched documents. + Write the sub-section using the provided documents. - This node takes the relevant documents fetched in the previous node and uses - an LLM to generate a comprehensive answer to the sub-section questions with + This node takes the relevant documents provided in the configuration and uses + an LLM to generate a comprehensive answer to the sub-section title with proper citations. The citations follow IEEE format using source IDs from the documents. @@ -174,17 +82,17 @@ async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, A Dict containing the final answer in the "final_answer" key. """ - # Get configuration and relevant documents + # Get configuration and relevant documents from configuration configuration = Configuration.from_runnable_config(config) - documents = state.relevant_documents_fetched + documents = configuration.relevant_documents # Initialize LLM llm = app_config.fast_llm_instance - # If no documents were found, return a message indicating this + # If no documents were provided, return a message indicating this if not documents or len(documents) == 0: return { - "final_answer": "No relevant documents were found to answer this question. Please try refining your search or providing more specific questions." + "final_answer": "No relevant documents were provided to answer this question. Please provide documents or try a different approach." } # Prepare documents for citation formatting @@ -208,18 +116,25 @@ async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, A """ formatted_documents.append(formatted_doc) - # Create the query that combines the section title and questions - # section_title = configuration.sub_section_title - questions = "\n".join([f"- {q}" for q in configuration.sub_questions]) + # Create the query that uses the section title and questions + section_title = configuration.sub_section_title + sub_section_questions = configuration.sub_section_questions documents_text = "\n".join(formatted_documents) + # Format the questions as bullet points for clarity + questions_text = "\n".join([f"- {question}" for question in sub_section_questions]) + # Construct a clear, structured query for the LLM human_message_content = f""" Please write a comprehensive answer for the title: - Address the following questions: + + {section_title} + + + Focus on answering these specific questions related to the title: - {questions} + {questions_text} Use the provided documents as your source material and cite them properly using the IEEE citation format [X] where X is the source_id. diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py index cc3ad6167..b8b3442a6 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py @@ -1,4 +1,8 @@ +import datetime + + citation_system_prompt = f""" +Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} You are a research assistant tasked with analyzing documents and providing comprehensive answers with proper citations in IEEE format. diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/state.py b/surfsense_backend/app/agents/researcher/sub_section_writer/state.py index fb5b08e87..b33abe6bd 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/state.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/state.py @@ -18,6 +18,6 @@ class State: db_session: AsyncSession # OUTPUT: Populated by agent nodes - relevant_documents_fetched: Optional[List[Any]] = None + reranked_documents: Optional[List[Any]] = None final_answer: Optional[str] = None diff --git a/surfsense_backend/app/agents/researcher/test_researcher.py b/surfsense_backend/app/agents/researcher/test_researcher.py new file mode 100644 index 000000000..72fbc2058 --- /dev/null +++ b/surfsense_backend/app/agents/researcher/test_researcher.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +""" +Test script for the Researcher LangGraph agent. + +This script demonstrates how to invoke the researcher agent with a sample query. +Run this script directly from VSCode using the "Run Python File" button or +right-click and select "Run Python File in Terminal". + +Before running: +1. Make sure your Python environment has all required dependencies +2. Create a .env file with any required API keys +3. Ensure database connection is properly configured +""" + +import asyncio +import os +import sys +from pathlib import Path + +# Add project root to Python path so that 'app' can be found as a module +# Get the absolute path to the surfsense_backend directory which contains the app module +project_root = str(Path(__file__).resolve().parents[3]) # Go up 3 levels from the script: app/agents/researcher -> app/agents -> app -> surfsense_backend +print(f"Adding to Python path: {project_root}") +sys.path.insert(0, project_root) + +# Now import the modules after fixing the path +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy.orm import sessionmaker +from dotenv import load_dotenv + +# These imports should now work with the correct path +from app.agents.researcher.graph import graph +from app.agents.researcher.state import State +from app.agents.researcher.nodes import write_answer_outline, process_sections + +# Load environment variables +load_dotenv() + +# Database connection string - use a test database or mock +DATABASE_URL = "postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense" + +# Create async engine and session +engine = create_async_engine(DATABASE_URL) +async_session_maker = sessionmaker( + engine, class_=AsyncSession, expire_on_commit=False +) + +async def run_test(): + """Run a test of the researcher agent.""" + print("Starting researcher agent test...") + + # Create a database session + async with async_session_maker() as db_session: + # Sample configuration + config = { + "configurable": { + "user_query": "What are the best clash royale decks recommended by Surgical Goblin?", + "num_sections": 1, + "connectors_to_search": [ + "YOUTUBE_VIDEO", + ], + "user_id": "d6ac2187-7407-4664-8734-af09926d161e", + "search_space_id": 2 + } + } + + try: + # Initialize state with database session and engine + initial_state = State(db_session=db_session, engine=engine) + + # Instead of using the graph directly, let's run the nodes manually + # to track the state transitions explicitly + print("\nSTEP 1: Running write_answer_outline node...") + outline_result = await write_answer_outline(initial_state, config) + + # Update the state with the outline + if "answer_outline" in outline_result: + initial_state.answer_outline = outline_result["answer_outline"] + print(f"Generated answer outline with {len(initial_state.answer_outline.answer_outline)} sections") + + # Print the outline + print("\nGenerated Answer Outline:") + for section in initial_state.answer_outline.answer_outline: + print(f"\nSection {section.section_id}: {section.section_title}") + print("Research Questions:") + for q in section.questions: + print(f" - {q}") + + # Run the second node with the updated state + print("\nSTEP 2: Running process_sections node...") + sections_result = await process_sections(initial_state, config) + + # Check if we got a final report + if "final_written_report" in sections_result: + final_report = sections_result["final_written_report"] + print("\nFinal Research Report generated successfully!") + print(f"Report length: {len(final_report)} characters") + + # Display the final report + print("\n==== FINAL RESEARCH REPORT ====\n") + print(final_report) + else: + print("\nNo final report was generated.") + print(f"Result keys: {list(sections_result.keys())}") + + return sections_result + + except Exception as e: + print(f"Error running researcher agent: {str(e)}") + import traceback + traceback.print_exc() + raise + +async def main(): + """Main entry point for the test script.""" + try: + result = await run_test() + print("\nTest completed successfully.") + return result + except Exception as e: + print(f"\nTest failed with error: {e}") + import traceback + traceback.print_exc() + return None + +if __name__ == "__main__": + # Run the async test + result = asyncio.run(main()) + + # Keep terminal open if run directly in VSCode + if 'VSCODE_PID' in os.environ: + input("\nPress Enter to close this window...") \ No newline at end of file diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index f4226ede8..82517a8df 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -42,10 +42,8 @@ class Config: # GPT Researcher FAST_LLM = os.getenv("FAST_LLM") - SMART_LLM = os.getenv("SMART_LLM") STRATEGIC_LLM = os.getenv("STRATEGIC_LLM") fast_llm_instance = ChatLiteLLM(model=extract_model_name(FAST_LLM)) - smart_llm_instance = ChatLiteLLM(model=extract_model_name(SMART_LLM)) strategic_llm_instance = ChatLiteLLM(model=extract_model_name(STRATEGIC_LLM)) From 154c5748fd21b937a46a654e9b693dc1ff047be7 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sat, 19 Apr 2025 23:44:42 -0700 Subject: [PATCH 20/31] refactor: coderabbit review --- .../app/agents/researcher/nodes.py | 17 ++++++----------- .../app/agents/researcher/prompts.py | 3 ++- .../researcher/sub_section_writer/nodes.py | 4 ++-- .../researcher/sub_section_writer/prompts.py | 3 ++- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index 9a0f62cf7..d13e77bd0 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -3,7 +3,7 @@ from langchain_core.runnables import RunnableConfig from .state import State from typing import Any, Dict, List from app.config import config as app_config -from .prompts import answer_outline_system_prompt +from .prompts import get_answer_outline_system_prompt from langchain_core.messages import HumanMessage, SystemMessage from pydantic import BaseModel, Field import json @@ -70,7 +70,7 @@ async def write_answer_outline(state: State, config: RunnableConfig) -> Dict[str # Create messages for the LLM messages = [ - SystemMessage(content=answer_outline_system_prompt), + SystemMessage(content=get_answer_outline_system_prompt()), HumanMessage(content=human_message_content) ] @@ -259,7 +259,6 @@ async def process_section( async with session_maker() as db_session: # Fetch relevant documents using all research questions for this section relevant_documents = await fetch_relevant_documents( - section_title=section_title, research_questions=research_questions, user_id=user_id, search_space_id=search_space_id, @@ -271,10 +270,8 @@ async def process_section( if not relevant_documents: print(f"No relevant documents found for section: {section_title}") relevant_documents = [ - { - "content": f"No specific information was found for: {question}" - for question in research_questions - } + {"content": f"No specific information was found for: {question}"} + for question in research_questions ] # Call the sub_section_writer graph with the appropriate config @@ -443,10 +440,8 @@ async def process_section_with_documents( if not documents_to_use: print(f"No relevant documents found for section: {section_title}") documents_to_use = [ - { - "content": f"No specific information was found for: {question}" - for question in section_questions - } + {"content": f"No specific information was found for: {question}"} + for question in section_questions ] # Call the sub_section_writer graph with the appropriate config diff --git a/surfsense_backend/app/agents/researcher/prompts.py b/surfsense_backend/app/agents/researcher/prompts.py index 0895590eb..3a2a3f755 100644 --- a/surfsense_backend/app/agents/researcher/prompts.py +++ b/surfsense_backend/app/agents/researcher/prompts.py @@ -1,7 +1,8 @@ import datetime -answer_outline_system_prompt = f""" +def get_answer_outline_system_prompt(): + return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} You are an expert research assistant specializing in structuring information. Your task is to create a detailed and logical research outline based on the user's query. This outline will serve as the blueprint for generating a comprehensive research report. diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py index a3384d95c..d76b1d8ad 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -3,7 +3,7 @@ from langchain_core.runnables import RunnableConfig from .state import State from typing import Any, Dict, List from app.config import config as app_config -from .prompts import citation_system_prompt +from .prompts import get_citation_system_prompt from langchain_core.messages import HumanMessage, SystemMessage async def rerank_documents(state: State, config: RunnableConfig) -> Dict[str, Any]: @@ -145,7 +145,7 @@ async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, A # Create messages for the LLM messages = [ - SystemMessage(content=citation_system_prompt), + SystemMessage(content=get_citation_system_prompt()), HumanMessage(content=human_message_content) ] diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py index b8b3442a6..18a91eb07 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py @@ -1,7 +1,8 @@ import datetime -citation_system_prompt = f""" +def get_citation_system_prompt(): + return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} You are a research assistant tasked with analyzing documents and providing comprehensive answers with proper citations in IEEE format. From 7be68ebf41e65af0949fa3dbd93e729aa108d574 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 20 Apr 2025 00:10:23 -0700 Subject: [PATCH 21/31] refactor: remove process_section function and streamline test workflow --- .../app/agents/researcher/nodes.py | 69 +------------------ .../app/agents/researcher/test_researcher.py | 30 ++++---- 2 files changed, 14 insertions(+), 85 deletions(-) diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index d13e77bd0..7099f02cb 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -232,71 +232,7 @@ async def fetch_relevant_documents( return deduplicated_docs -async def process_section( - section_title: str, - user_id: str, - search_space_id: int, - session_maker, - research_questions: List[str], - connectors_to_search: List[str] -) -> str: - """ - Process a single section by sending it to the sub_section_writer graph. - - Args: - section_title: The title of the section - user_id: The user ID - search_space_id: The search space ID - session_maker: Factory for creating new database sessions - research_questions: List of research questions for this section - connectors_to_search: List of connectors to search - - Returns: - The written section content - """ - try: - # Create a new database session for this section - async with session_maker() as db_session: - # Fetch relevant documents using all research questions for this section - relevant_documents = await fetch_relevant_documents( - research_questions=research_questions, - user_id=user_id, - search_space_id=search_space_id, - db_session=db_session, - connectors_to_search=connectors_to_search - ) - - # Fallback if no documents found - if not relevant_documents: - print(f"No relevant documents found for section: {section_title}") - relevant_documents = [ - {"content": f"No specific information was found for: {question}"} - for question in research_questions - ] - - # Call the sub_section_writer graph with the appropriate config - config = { - "configurable": { - "sub_section_title": section_title, - "relevant_documents": relevant_documents, - "user_id": user_id, - "search_space_id": search_space_id - } - } - - # Create the initial state with db_session - state = {"db_session": db_session} - - # Invoke the sub-section writer graph - print(f"Invoking sub_section_writer for: {section_title}") - result = await sub_section_writer_graph.ainvoke(state, config) - - # Return the final answer from the sub_section_writer - final_answer = result.get("final_answer", "No content was generated for this section.") - return final_answer - except Exception as e: - print(f"Error processing section '{section_title}': {str(e)}") - return f"Error processing section: {section_title}. Details: {str(e)}" + async def process_sections(state: State, config: RunnableConfig) -> Dict[str, Any]: """ @@ -395,8 +331,7 @@ async def process_sections(state: State, config: RunnableConfig) -> Dict[str, An # Combine the results into a final report with section titles final_report = [] for i, (section, content) in enumerate(zip(answer_outline.answer_outline, processed_results)): - section_header = f"## {section.section_title}" - final_report.append(section_header) + # Skip adding the section header since the content already contains the title final_report.append(content) final_report.append("\n") # Add spacing between sections diff --git a/surfsense_backend/app/agents/researcher/test_researcher.py b/surfsense_backend/app/agents/researcher/test_researcher.py index 72fbc2058..15c993e74 100644 --- a/surfsense_backend/app/agents/researcher/test_researcher.py +++ b/surfsense_backend/app/agents/researcher/test_researcher.py @@ -31,7 +31,7 @@ from dotenv import load_dotenv # These imports should now work with the correct path from app.agents.researcher.graph import graph from app.agents.researcher.state import State -from app.agents.researcher.nodes import write_answer_outline, process_sections + # Load environment variables load_dotenv() @@ -68,31 +68,25 @@ async def run_test(): # Initialize state with database session and engine initial_state = State(db_session=db_session, engine=engine) - # Instead of using the graph directly, let's run the nodes manually - # to track the state transitions explicitly - print("\nSTEP 1: Running write_answer_outline node...") - outline_result = await write_answer_outline(initial_state, config) + # Run the graph directly + print("\nRunning the complete researcher workflow...") + result = await graph.ainvoke(initial_state, config) - # Update the state with the outline - if "answer_outline" in outline_result: - initial_state.answer_outline = outline_result["answer_outline"] - print(f"Generated answer outline with {len(initial_state.answer_outline.answer_outline)} sections") + # Extract the answer outline for display + if "answer_outline" in result and result["answer_outline"]: + print(f"\nGenerated answer outline with {len(result['answer_outline'].answer_outline)} sections") # Print the outline print("\nGenerated Answer Outline:") - for section in initial_state.answer_outline.answer_outline: + for section in result["answer_outline"].answer_outline: print(f"\nSection {section.section_id}: {section.section_title}") print("Research Questions:") for q in section.questions: print(f" - {q}") - # Run the second node with the updated state - print("\nSTEP 2: Running process_sections node...") - sections_result = await process_sections(initial_state, config) - # Check if we got a final report - if "final_written_report" in sections_result: - final_report = sections_result["final_written_report"] + if "final_written_report" in result and result["final_written_report"]: + final_report = result["final_written_report"] print("\nFinal Research Report generated successfully!") print(f"Report length: {len(final_report)} characters") @@ -101,9 +95,9 @@ async def run_test(): print(final_report) else: print("\nNo final report was generated.") - print(f"Result keys: {list(sections_result.keys())}") + print(f"Available result keys: {list(result.keys())}") - return sections_result + return result except Exception as e: print(f"Error running researcher agent: {str(e)}") From 796e326cc8d5b7890d0fa8667feb9818b5cde951 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 20 Apr 2025 00:19:21 -0700 Subject: [PATCH 22/31] refactor: coderabbit --- .../app/agents/researcher/nodes.py | 23 +++++++++++-------- .../researcher/sub_section_writer/nodes.py | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index 7099f02cb..8fa72e111 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -12,7 +12,6 @@ from .sub_section_writer.graph import graph as sub_section_writer_graph from app.utils.connector_service import ConnectorService from app.utils.reranker_service import RerankerService from sqlalchemy.ext.asyncio import AsyncSession -import copy class Section(BaseModel): """A section in the answer outline.""" @@ -133,7 +132,6 @@ async def fetch_relevant_documents( """ # Initialize services connector_service = ConnectorService(db_session) - reranker_service = RerankerService.get_reranker_instance(app_config) all_raw_documents = [] # Store all raw documents before reranking @@ -289,13 +287,20 @@ async def process_sections(state: State, config: RunnableConfig) -> Dict[str, An relevant_documents = [] async with session_maker() as db_session: - relevant_documents = await fetch_relevant_documents( - research_questions=all_questions, - user_id=configuration.user_id, - search_space_id=configuration.search_space_id, - db_session=db_session, - connectors_to_search=configuration.connectors_to_search - ) + try: + relevant_documents = await fetch_relevant_documents( + research_questions=all_questions, + user_id=configuration.user_id, + search_space_id=configuration.search_space_id, + db_session=db_session, + connectors_to_search=configuration.connectors_to_search + ) + except Exception as e: + print(f"Error fetching relevant documents: {str(e)}") + # Log the error and continue with an empty list of documents + # This allows the process to continue, but the report might lack information + relevant_documents = [] + # Consider adding more robust error handling or reporting if needed print(f"Fetched {len(relevant_documents)} relevant documents for all sections") diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py index d76b1d8ad..af807e386 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -1,7 +1,7 @@ from .configuration import Configuration from langchain_core.runnables import RunnableConfig from .state import State -from typing import Any, Dict, List +from typing import Any, Dict from app.config import config as app_config from .prompts import get_citation_system_prompt from langchain_core.messages import HumanMessage, SystemMessage From ebc376473e2fed9d2f9cc662cc2ac656f43448be Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 20 Apr 2025 00:23:02 -0700 Subject: [PATCH 23/31] refactor: remove unused RerankerService import from nodes.py --- surfsense_backend/app/agents/researcher/nodes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index 8fa72e111..c88ad3308 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -10,7 +10,6 @@ import json import asyncio from .sub_section_writer.graph import graph as sub_section_writer_graph from app.utils.connector_service import ConnectorService -from app.utils.reranker_service import RerankerService from sqlalchemy.ext.asyncio import AsyncSession class Section(BaseModel): From 130f43a0fae4fd3aad6587a8f4bfb37af9fb16d1 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 20 Apr 2025 19:19:35 -0700 Subject: [PATCH 24/31] feat: Removed GPT-Researcher in favour of own SurfSense LangGraph Agent --- README.md | 11 +- surfsense_backend/.env.example | 5 +- .../app/agents/researcher/nodes.py | 398 ++++++++++++--- .../app/agents/researcher/state.py | 11 +- .../sub_section_writer/configuration.py | 1 + .../researcher/sub_section_writer/nodes.py | 16 +- .../app/agents/researcher/test_researcher.py | 126 ----- surfsense_backend/app/routes/chats_routes.py | 15 +- .../tasks/stream_connector_search_results.py | 466 ++---------------- surfsense_backend/app/utils/query_service.py | 6 +- .../app/utils/research_service.py | 211 -------- .../app/utils/streaming_service.py | 75 +-- surfsense_backend/pyproject.toml | 1 - surfsense_web/components/markdown-viewer.tsx | 15 +- 14 files changed, 439 insertions(+), 918 deletions(-) delete mode 100644 surfsense_backend/app/agents/researcher/test_researcher.py delete mode 100644 surfsense_backend/app/utils/research_service.py diff --git a/README.md b/README.md index 4b9ea5a64..aa337c7f2 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,6 @@ This is the core of SurfSense. Before we begin let's look at `.env` variables' t | RERANKERS_MODEL_NAME| Name of the reranker model for search result reranking. Eg. `ms-marco-MiniLM-L-12-v2`| | RERANKERS_MODEL_TYPE| Type of reranker model being used. Eg. `flashrank`| | FAST_LLM| LiteLLM routed Smaller, faster LLM for quick responses. Eg. `litellm:openai/gpt-4o`| -| SMART_LLM| LiteLLM routed Balanced LLM for general use. Eg. `litellm:openai/gpt-4o`| | STRATEGIC_LLM| LiteLLM routed Advanced LLM for complex reasoning tasks. Eg. `litellm:openai/gpt-4o`| | LONG_CONTEXT_LLM| LiteLLM routed LLM capable of handling longer context windows. Eg. `litellm:gemini/gemini-2.0-flash`| | UNSTRUCTURED_API_KEY| API key for Unstructured.io service for document parsing| @@ -221,15 +220,15 @@ After filling in your SurfSense API key you should be able to use extension now. - **Alembic**: A database migrations tool for SQLAlchemy. - **FastAPI Users**: Authentication and user management with JWT and OAuth support - -- **LangChain**: Framework for developing AI-powered applications -- **GPT Integration**: Integration with LLM models through LiteLLM +- **LangGraph**: Framework for developing AI-agents. + +- **LangChain**: Framework for developing AI-powered applications. + +- **LLM Integration**: Integration with LLM models through LiteLLM - **Rerankers**: Advanced result ranking for improved search relevance -- **GPT-Researcher**: Advanced research capabilities - - **Hybrid Search**: Combines vector similarity and full-text search for optimal results using Reciprocal Rank Fusion (RRF) - **Vector Embeddings**: Document and text embeddings for semantic search diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 380296e65..bc3f5cebb 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -10,9 +10,8 @@ RERANKERS_MODEL_NAME="ms-marco-MiniLM-L-12-v2" RERANKERS_MODEL_TYPE="flashrank" FAST_LLM="litellm:openai/gpt-4o-mini" -SMART_LLM="litellm:openai/gpt-4o-mini" -STRATEGIC_LLM="litellm:openai/gpt-4o-mini" -LONG_CONTEXT_LLM="litellm:gemini/gemini-2.0-flash-thinking-exp-01-21" +STRATEGIC_LLM="litellm:openai/gpt-4o" +LONG_CONTEXT_LLM="litellm:gemini/gemini-2.0-flash" OPENAI_API_KEY="sk-proj-iA" GEMINI_API_KEY="AIzaSyB6-1641124124124124124124124124124" diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index c88ad3308..15935f2ea 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -1,17 +1,23 @@ -from .configuration import Configuration -from langchain_core.runnables import RunnableConfig -from .state import State -from typing import Any, Dict, List -from app.config import config as app_config -from .prompts import get_answer_outline_system_prompt -from langchain_core.messages import HumanMessage, SystemMessage -from pydantic import BaseModel, Field -import json import asyncio -from .sub_section_writer.graph import graph as sub_section_writer_graph +import json +from typing import Any, Dict, List + +from app.config import config as app_config +from app.db import async_session_maker from app.utils.connector_service import ConnectorService +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_core.runnables import RunnableConfig +from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession +from .configuration import Configuration +from .prompts import get_answer_outline_system_prompt +from .state import State +from .sub_section_writer.graph import graph as sub_section_writer_graph + +from langgraph.types import StreamWriter + + class Section(BaseModel): """A section in the answer outline.""" section_id: int = Field(..., description="The zero-based index of the section") @@ -22,7 +28,7 @@ class AnswerOutline(BaseModel): """The complete answer outline with all sections.""" answer_outline: List[Section] = Field(..., description="List of sections in the answer outline") -async def write_answer_outline(state: State, config: RunnableConfig) -> Dict[str, Any]: +async def write_answer_outline(state: State, config: RunnableConfig, writer: StreamWriter) -> Dict[str, Any]: """ Create a structured answer outline based on the user query. @@ -33,12 +39,18 @@ async def write_answer_outline(state: State, config: RunnableConfig) -> Dict[str Returns: Dict containing the answer outline in the "answer_outline" key for state update. """ + streaming_service = state.streaming_service + streaming_service.only_update_terminal("Generating answer outline...") + writer({"yeild_value": streaming_service._format_annotations()}) # Get configuration from runnable config configuration = Configuration.from_runnable_config(config) user_query = configuration.user_query num_sections = configuration.num_sections + streaming_service.only_update_terminal(f"Planning research approach for query: {user_query[:100]}...") + writer({"yeild_value": streaming_service._format_annotations()}) + # Initialize LLM llm = app_config.strategic_llm_instance @@ -66,6 +78,9 @@ async def write_answer_outline(state: State, config: RunnableConfig) -> Dict[str Your output MUST be valid JSON in exactly this format. Do not include any other text or explanation. """ + streaming_service.only_update_terminal("Designing structured outline with AI...") + writer({"yeild_value": streaming_service._format_annotations()}) + # Create messages for the LLM messages = [ SystemMessage(content=get_answer_outline_system_prompt()), @@ -73,6 +88,9 @@ async def write_answer_outline(state: State, config: RunnableConfig) -> Dict[str ] # Call the LLM directly without using structured output + streaming_service.only_update_terminal("Processing answer structure...") + writer({"yeild_value": streaming_service._format_annotations()}) + response = await llm.ainvoke(messages) # Parse the JSON response manually @@ -92,16 +110,27 @@ async def write_answer_outline(state: State, config: RunnableConfig) -> Dict[str # Convert to Pydantic model answer_outline = AnswerOutline(**parsed_data) + total_questions = sum(len(section.questions) for section in answer_outline.answer_outline) + streaming_service.only_update_terminal(f"Successfully generated outline with {len(answer_outline.answer_outline)} sections and {total_questions} research questions") + writer({"yeild_value": streaming_service._format_annotations()}) + print(f"Successfully generated answer outline with {len(answer_outline.answer_outline)} sections") # Return state update return {"answer_outline": answer_outline} else: # If JSON structure not found, raise a clear error - raise ValueError(f"Could not find valid JSON in LLM response. Raw response: {content}") + error_message = f"Could not find valid JSON in LLM response. Raw response: {content}" + streaming_service.only_update_terminal(error_message, "error") + writer({"yeild_value": streaming_service._format_annotations()}) + raise ValueError(error_message) except (json.JSONDecodeError, ValueError) as e: # Log the error and re-raise it + error_message = f"Error parsing LLM response: {str(e)}" + streaming_service.only_update_terminal(error_message, "error") + writer({"yeild_value": streaming_service._format_annotations()}) + print(f"Error parsing LLM response: {str(e)}") print(f"Raw response: {response.content}") raise @@ -112,18 +141,21 @@ async def fetch_relevant_documents( search_space_id: int, db_session: AsyncSession, connectors_to_search: List[str], - top_k: int = 5 + writer: StreamWriter = None, + state: State = None, + top_k: int = 20 ) -> List[Dict[str, Any]]: """ Fetch relevant documents for research questions using the provided connectors. Args: - section_title: The title of the section being researched research_questions: List of research questions to find documents for user_id: The user ID search_space_id: The search space ID db_session: The database session connectors_to_search: List of connectors to search + writer: StreamWriter for sending progress updates + state: The current state containing the streaming service top_k: Number of top results to retrieve per connector per question Returns: @@ -131,83 +163,237 @@ async def fetch_relevant_documents( """ # Initialize services connector_service = ConnectorService(db_session) - - all_raw_documents = [] # Store all raw documents before reranking - for user_query in research_questions: + # Only use streaming if both writer and state are provided + streaming_service = state.streaming_service if state is not None else None + + # Stream initial status update + if streaming_service and writer: + streaming_service.only_update_terminal(f"Starting research on {len(research_questions)} questions using {len(connectors_to_search)} connectors...") + writer({"yeild_value": streaming_service._format_annotations()}) + + all_raw_documents = [] # Store all raw documents + all_sources = [] # Store all sources + + for i, user_query in enumerate(research_questions): + # Stream question being researched + if streaming_service and writer: + streaming_service.only_update_terminal(f"Researching question {i+1}/{len(research_questions)}: {user_query[:100]}...") + writer({"yeild_value": streaming_service._format_annotations()}) + # Use original research question as the query reformulated_query = user_query # Process each selected connector for connector in connectors_to_search: + # Stream connector being searched + if streaming_service and writer: + streaming_service.only_update_terminal(f"Searching {connector} for relevant information...") + writer({"yeild_value": streaming_service._format_annotations()}) + try: if connector == "YOUTUBE_VIDEO": - _, youtube_chunks = await connector_service.search_youtube( + source_object, youtube_chunks = await connector_service.search_youtube( user_query=reformulated_query, user_id=user_id, search_space_id=search_space_id, top_k=top_k ) + + # Add to sources and raw documents + if source_object: + all_sources.append(source_object) all_raw_documents.extend(youtube_chunks) + # Stream found document count + if streaming_service and writer: + streaming_service.only_update_terminal(f"Found {len(youtube_chunks)} YouTube chunks relevant to the query") + writer({"yeild_value": streaming_service._format_annotations()}) + elif connector == "EXTENSION": - _, extension_chunks = await connector_service.search_extension( + source_object, extension_chunks = await connector_service.search_extension( user_query=reformulated_query, user_id=user_id, search_space_id=search_space_id, top_k=top_k ) + + # Add to sources and raw documents + if source_object: + all_sources.append(source_object) all_raw_documents.extend(extension_chunks) + # Stream found document count + if streaming_service and writer: + streaming_service.only_update_terminal(f"Found {len(extension_chunks)} extension chunks relevant to the query") + writer({"yeild_value": streaming_service._format_annotations()}) + elif connector == "CRAWLED_URL": - _, crawled_urls_chunks = await connector_service.search_crawled_urls( + source_object, crawled_urls_chunks = await connector_service.search_crawled_urls( user_query=reformulated_query, user_id=user_id, search_space_id=search_space_id, top_k=top_k ) + + # Add to sources and raw documents + if source_object: + all_sources.append(source_object) all_raw_documents.extend(crawled_urls_chunks) + # Stream found document count + if streaming_service and writer: + streaming_service.only_update_terminal(f"Found {len(crawled_urls_chunks)} crawled URL chunks relevant to the query") + writer({"yeild_value": streaming_service._format_annotations()}) + elif connector == "FILE": - _, files_chunks = await connector_service.search_files( + source_object, files_chunks = await connector_service.search_files( user_query=reformulated_query, user_id=user_id, search_space_id=search_space_id, top_k=top_k ) + + # Add to sources and raw documents + if source_object: + all_sources.append(source_object) all_raw_documents.extend(files_chunks) + # Stream found document count + if streaming_service and writer: + streaming_service.only_update_terminal(f"Found {len(files_chunks)} file chunks relevant to the query") + writer({"yeild_value": streaming_service._format_annotations()}) + elif connector == "TAVILY_API": - _, tavily_chunks = await connector_service.search_tavily( + source_object, tavily_chunks = await connector_service.search_tavily( user_query=reformulated_query, user_id=user_id, top_k=top_k ) + + # Add to sources and raw documents + if source_object: + all_sources.append(source_object) all_raw_documents.extend(tavily_chunks) + # Stream found document count + if streaming_service and writer: + streaming_service.only_update_terminal(f"Found {len(tavily_chunks)} web search results relevant to the query") + writer({"yeild_value": streaming_service._format_annotations()}) + elif connector == "SLACK_CONNECTOR": - _, slack_chunks = await connector_service.search_slack( + source_object, slack_chunks = await connector_service.search_slack( user_query=reformulated_query, user_id=user_id, search_space_id=search_space_id, top_k=top_k ) + + # Add to sources and raw documents + if source_object: + all_sources.append(source_object) all_raw_documents.extend(slack_chunks) + # Stream found document count + if streaming_service and writer: + streaming_service.only_update_terminal(f"Found {len(slack_chunks)} Slack messages relevant to the query") + writer({"yeild_value": streaming_service._format_annotations()}) + elif connector == "NOTION_CONNECTOR": - _, notion_chunks = await connector_service.search_notion( + source_object, notion_chunks = await connector_service.search_notion( user_query=reformulated_query, user_id=user_id, search_space_id=search_space_id, top_k=top_k ) + + # Add to sources and raw documents + if source_object: + all_sources.append(source_object) all_raw_documents.extend(notion_chunks) + + # Stream found document count + if streaming_service and writer: + streaming_service.only_update_terminal(f"Found {len(notion_chunks)} Notion pages/blocks relevant to the query") + writer({"yeild_value": streaming_service._format_annotations()}) + + elif connector == "GITHUB_CONNECTOR": + source_object, github_chunks = await connector_service.search_github( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k + ) + + # Add to sources and raw documents + if source_object: + all_sources.append(source_object) + all_raw_documents.extend(github_chunks) + + # Stream found document count + if streaming_service and writer: + streaming_service.only_update_terminal(f"Found {len(github_chunks)} GitHub files/issues relevant to the query") + writer({"yeild_value": streaming_service._format_annotations()}) + + elif connector == "LINEAR_CONNECTOR": + source_object, linear_chunks = await connector_service.search_linear( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k + ) + + # Add to sources and raw documents + if source_object: + all_sources.append(source_object) + all_raw_documents.extend(linear_chunks) + + # Stream found document count + if streaming_service and writer: + streaming_service.only_update_terminal(f"Found {len(linear_chunks)} Linear issues relevant to the query") + writer({"yeild_value": streaming_service._format_annotations()}) except Exception as e: - print(f"Error searching connector {connector}: {str(e)}") + error_message = f"Error searching connector {connector}: {str(e)}" + print(error_message) + + # Stream error message + if streaming_service and writer: + streaming_service.only_update_terminal(error_message, "error") + writer({"yeild_value": streaming_service._format_annotations()}) + # Continue with other connectors on error continue - # Deduplicate documents based on chunk_id or content + # Deduplicate source objects by ID before streaming + deduplicated_sources = [] + seen_source_keys = set() + + for source_obj in all_sources: + # Use combination of source ID and type as a unique identifier + # This ensures we don't accidentally deduplicate sources from different connectors + source_id = source_obj.get('id') + source_type = source_obj.get('type') + + if source_id and source_type: + source_key = f"{source_type}_{source_id}" + if source_key not in seen_source_keys: + seen_source_keys.add(source_key) + deduplicated_sources.append(source_obj) + else: + # If there's no ID or type, just add it to be safe + deduplicated_sources.append(source_obj) + + # Stream info about deduplicated sources + if streaming_service and writer: + streaming_service.only_update_terminal(f"Collected {len(deduplicated_sources)} unique sources across all connectors") + writer({"yeild_value": streaming_service._format_annotations()}) + + # After all sources are collected and deduplicated, stream them + if streaming_service and writer: + streaming_service.only_update_sources(deduplicated_sources) + writer({"yeild_value": streaming_service._format_annotations()}) + + # Deduplicate raw documents based on chunk_id or content seen_chunk_ids = set() seen_content_hashes = set() deduplicated_docs = [] @@ -227,11 +413,15 @@ async def fetch_relevant_documents( seen_content_hashes.add(content_hash) deduplicated_docs.append(doc) + # Stream info about deduplicated documents + if streaming_service and writer: + streaming_service.only_update_terminal(f"Found {len(deduplicated_docs)} unique document chunks after deduplication") + writer({"yeild_value": streaming_service._format_annotations()}) + + # Return deduplicated documents return deduplicated_docs - - -async def process_sections(state: State, config: RunnableConfig) -> Dict[str, Any]: +async def process_sections(state: State, config: RunnableConfig, writer: StreamWriter) -> Dict[str, Any]: """ Process all sections in parallel and combine the results. @@ -245,89 +435,97 @@ async def process_sections(state: State, config: RunnableConfig) -> Dict[str, An # Get configuration and answer outline from state configuration = Configuration.from_runnable_config(config) answer_outline = state.answer_outline + streaming_service = state.streaming_service + + streaming_service.only_update_terminal(f"Starting to process research sections...") + writer({"yeild_value": streaming_service._format_annotations()}) print(f"Processing sections from outline: {answer_outline is not None}") if not answer_outline: + streaming_service.only_update_terminal("Error: No answer outline was provided. Cannot generate report.", "error") + writer({"yeild_value": streaming_service._format_annotations()}) return { "final_written_report": "No answer outline was provided. Cannot generate final report." } - # Create session maker from the engine or directly use the session - from sqlalchemy.ext.asyncio import AsyncSession - from sqlalchemy.orm import sessionmaker - - # Use the engine if available, otherwise create a new session for each task - if state.engine: - session_maker = sessionmaker( - state.engine, class_=AsyncSession, expire_on_commit=False - ) - else: - # Fallback to using the same session (less optimal but will work) - print("Warning: No engine available. Using same session for all tasks.") - # Create a mock session maker that returns the same session - async def mock_session_maker(): - class ContextManager: - async def __aenter__(self): - return state.db_session - async def __aexit__(self, exc_type, exc_val, exc_tb): - pass - return ContextManager() - session_maker = mock_session_maker - # Collect all questions from all sections all_questions = [] for section in answer_outline.answer_outline: all_questions.extend(section.questions) print(f"Collected {len(all_questions)} questions from all sections") + streaming_service.only_update_terminal(f"Found {len(all_questions)} research questions across {len(answer_outline.answer_outline)} sections") + writer({"yeild_value": streaming_service._format_annotations()}) # Fetch relevant documents once for all questions + streaming_service.only_update_terminal("Searching for relevant information across all connectors...") + writer({"yeild_value": streaming_service._format_annotations()}) + relevant_documents = [] - async with session_maker() as db_session: - + async with async_session_maker() as db_session: try: relevant_documents = await fetch_relevant_documents( research_questions=all_questions, user_id=configuration.user_id, search_space_id=configuration.search_space_id, db_session=db_session, - connectors_to_search=configuration.connectors_to_search + connectors_to_search=configuration.connectors_to_search, + writer=writer, + state=state ) except Exception as e: - print(f"Error fetching relevant documents: {str(e)}") + error_message = f"Error fetching relevant documents: {str(e)}" + print(error_message) + streaming_service.only_update_terminal(error_message, "error") + writer({"yeild_value": streaming_service._format_annotations()}) # Log the error and continue with an empty list of documents # This allows the process to continue, but the report might lack information relevant_documents = [] # Consider adding more robust error handling or reporting if needed print(f"Fetched {len(relevant_documents)} relevant documents for all sections") + streaming_service.only_update_terminal(f"Starting to draft {len(answer_outline.answer_outline)} sections using {len(relevant_documents)} relevant document chunks") + writer({"yeild_value": streaming_service._format_annotations()}) # Create tasks to process each section in parallel with the same document set section_tasks = [] + streaming_service.only_update_terminal("Creating processing tasks for each section...") + writer({"yeild_value": streaming_service._format_annotations()}) + for section in answer_outline.answer_outline: section_tasks.append( process_section_with_documents( section_title=section.section_title, section_questions=section.questions, + user_query=configuration.user_query, user_id=configuration.user_id, search_space_id=configuration.search_space_id, - session_maker=session_maker, - relevant_documents=relevant_documents + relevant_documents=relevant_documents, + state=state, + writer=writer ) ) # Run all section processing tasks in parallel print(f"Running {len(section_tasks)} section processing tasks in parallel") + streaming_service.only_update_terminal(f"Processing {len(section_tasks)} sections simultaneously...") + writer({"yeild_value": streaming_service._format_annotations()}) + section_results = await asyncio.gather(*section_tasks, return_exceptions=True) # Handle any exceptions in the results + streaming_service.only_update_terminal("Combining section results into final report...") + writer({"yeild_value": streaming_service._format_annotations()}) + processed_results = [] for i, result in enumerate(section_results): if isinstance(result, Exception): section_title = answer_outline.answer_outline[i].section_title error_message = f"Error processing section '{section_title}': {str(result)}" print(error_message) + streaming_service.only_update_terminal(error_message, "error") + writer({"yeild_value": streaming_service._format_annotations()}) processed_results.append(error_message) else: processed_results.append(result) @@ -337,12 +535,33 @@ async def process_sections(state: State, config: RunnableConfig) -> Dict[str, An for i, (section, content) in enumerate(zip(answer_outline.answer_outline, processed_results)): # Skip adding the section header since the content already contains the title final_report.append(content) - final_report.append("\n") # Add spacing between sections + final_report.append("\n") + # Join all sections with newlines final_written_report = "\n".join(final_report) print(f"Generated final report with {len(final_report)} parts") + streaming_service.only_update_terminal("Final research report generated successfully!") + writer({"yeild_value": streaming_service._format_annotations()}) + + if hasattr(state, 'streaming_service') and state.streaming_service: + # Convert the final report to the expected format for UI: + # A list of strings where empty strings represent line breaks + formatted_report = [] + for section in final_report: + if section == "\n": + # Add an empty string for line breaks + formatted_report.append("") + else: + # Split any multiline content by newlines and add each line + section_lines = section.split("\n") + formatted_report.extend(section_lines) + + state.streaming_service.only_update_answer(formatted_report) + writer({"yeild_value": state.streaming_service._format_annotations()}) + + return { "final_written_report": final_written_report } @@ -352,8 +571,10 @@ async def process_section_with_documents( section_questions: List[str], user_id: str, search_space_id: int, - session_maker, - relevant_documents: List[Dict[str, Any]] + relevant_documents: List[Dict[str, Any]], + user_query: str, + state: State = None, + writer: StreamWriter = None ) -> str: """ Process a single section using pre-fetched documents. @@ -363,31 +584,42 @@ async def process_section_with_documents( section_questions: List of research questions for this section user_id: The user ID search_space_id: The search space ID - session_maker: Factory for creating new database sessions relevant_documents: Pre-fetched documents to use for this section + state: The current state + writer: StreamWriter for sending progress updates Returns: The written section content """ try: + # Use the provided documents + documents_to_use = relevant_documents + + # Send status update via streaming if available + if state and state.streaming_service and writer: + state.streaming_service.only_update_terminal(f"Writing section: {section_title} with {len(section_questions)} research questions") + writer({"yeild_value": state.streaming_service._format_annotations()}) + + # Fallback if no documents found + if not documents_to_use: + print(f"No relevant documents found for section: {section_title}") + if state and state.streaming_service and writer: + state.streaming_service.only_update_terminal(f"Warning: No relevant documents found for section: {section_title}", "warning") + writer({"yeild_value": state.streaming_service._format_annotations()}) + + documents_to_use = [ + {"content": f"No specific information was found for: {question}"} + for question in section_questions + ] + # Create a new database session for this section - async with session_maker() as db_session: - # Use the provided documents - documents_to_use = relevant_documents - - # Fallback if no documents found - if not documents_to_use: - print(f"No relevant documents found for section: {section_title}") - documents_to_use = [ - {"content": f"No specific information was found for: {question}"} - for question in section_questions - ] - + async with async_session_maker() as db_session: # Call the sub_section_writer graph with the appropriate config config = { "configurable": { "sub_section_title": section_title, "sub_section_questions": section_questions, + "user_query": user_query, "relevant_documents": documents_to_use, "user_id": user_id, "search_space_id": search_space_id @@ -395,16 +627,32 @@ async def process_section_with_documents( } # Create the initial state with db_session - state = {"db_session": db_session} + sub_state = {"db_session": db_session} # Invoke the sub-section writer graph print(f"Invoking sub_section_writer for: {section_title}") - result = await sub_section_writer_graph.ainvoke(state, config) + if state and state.streaming_service and writer: + state.streaming_service.only_update_terminal(f"Analyzing information and drafting content for section: {section_title}") + writer({"yeild_value": state.streaming_service._format_annotations()}) + + result = await sub_section_writer_graph.ainvoke(sub_state, config) # Return the final answer from the sub_section_writer final_answer = result.get("final_answer", "No content was generated for this section.") + + # Send section content update via streaming if available + if state and state.streaming_service and writer: + state.streaming_service.only_update_terminal(f"Completed writing section: {section_title}") + writer({"yeild_value": state.streaming_service._format_annotations()}) + return final_answer except Exception as e: print(f"Error processing section '{section_title}': {str(e)}") + + # Send error update via streaming if available + if state and state.streaming_service and writer: + state.streaming_service.only_update_terminal(f"Error processing section '{section_title}': {str(e)}", "error") + writer({"yeild_value": state.streaming_service._format_annotations()}) + return f"Error processing section: {section_title}. Details: {str(e)}" diff --git a/surfsense_backend/app/agents/researcher/state.py b/surfsense_backend/app/agents/researcher/state.py index 483e96ac9..dd36163b6 100644 --- a/surfsense_backend/app/agents/researcher/state.py +++ b/surfsense_backend/app/agents/researcher/state.py @@ -3,10 +3,9 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import List, Optional, Any, Dict, Annotated -from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine -from langchain_core.messages import BaseMessage, HumanMessage -from pydantic import BaseModel +from typing import Optional, Any +from sqlalchemy.ext.asyncio import AsyncSession +from app.utils.streaming_service import StreamingService @dataclass class State: @@ -18,7 +17,9 @@ class State: """ # Runtime context (not part of actual graph state) db_session: AsyncSession - engine: Optional[AsyncEngine] = None + + # Streaming service + streaming_service: StreamingService # Intermediate state - populated during workflow # Using field to explicitly mark as part of state diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py b/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py index fbde94d57..9e1ca32b5 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py @@ -15,6 +15,7 @@ class Configuration: # Input parameters provided at invocation sub_section_title: str sub_section_questions: List[str] + user_query: str relevant_documents: List[Any] # Documents provided directly to the agent user_id: str search_space_id: int diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py index af807e386..5b11141ca 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -119,6 +119,7 @@ async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, A # Create the query that uses the section title and questions section_title = configuration.sub_section_title sub_section_questions = configuration.sub_section_questions + user_query = configuration.user_query # Get the original user query documents_text = "\n".join(formatted_documents) # Format the questions as bullet points for clarity @@ -126,17 +127,16 @@ async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, A # Construct a clear, structured query for the LLM human_message_content = f""" - Please write a comprehensive answer for the title: + Now user's query is: + + {user_query} + - + The sub-section title is: + <sub_section_title> {section_title} - + - Focus on answering these specific questions related to the title: - - {questions_text} - - Use the provided documents as your source material and cite them properly using the IEEE citation format [X] where X is the source_id. {documents_text} diff --git a/surfsense_backend/app/agents/researcher/test_researcher.py b/surfsense_backend/app/agents/researcher/test_researcher.py deleted file mode 100644 index 15c993e74..000000000 --- a/surfsense_backend/app/agents/researcher/test_researcher.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for the Researcher LangGraph agent. - -This script demonstrates how to invoke the researcher agent with a sample query. -Run this script directly from VSCode using the "Run Python File" button or -right-click and select "Run Python File in Terminal". - -Before running: -1. Make sure your Python environment has all required dependencies -2. Create a .env file with any required API keys -3. Ensure database connection is properly configured -""" - -import asyncio -import os -import sys -from pathlib import Path - -# Add project root to Python path so that 'app' can be found as a module -# Get the absolute path to the surfsense_backend directory which contains the app module -project_root = str(Path(__file__).resolve().parents[3]) # Go up 3 levels from the script: app/agents/researcher -> app/agents -> app -> surfsense_backend -print(f"Adding to Python path: {project_root}") -sys.path.insert(0, project_root) - -# Now import the modules after fixing the path -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -from sqlalchemy.orm import sessionmaker -from dotenv import load_dotenv - -# These imports should now work with the correct path -from app.agents.researcher.graph import graph -from app.agents.researcher.state import State - - -# Load environment variables -load_dotenv() - -# Database connection string - use a test database or mock -DATABASE_URL = "postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense" - -# Create async engine and session -engine = create_async_engine(DATABASE_URL) -async_session_maker = sessionmaker( - engine, class_=AsyncSession, expire_on_commit=False -) - -async def run_test(): - """Run a test of the researcher agent.""" - print("Starting researcher agent test...") - - # Create a database session - async with async_session_maker() as db_session: - # Sample configuration - config = { - "configurable": { - "user_query": "What are the best clash royale decks recommended by Surgical Goblin?", - "num_sections": 1, - "connectors_to_search": [ - "YOUTUBE_VIDEO", - ], - "user_id": "d6ac2187-7407-4664-8734-af09926d161e", - "search_space_id": 2 - } - } - - try: - # Initialize state with database session and engine - initial_state = State(db_session=db_session, engine=engine) - - # Run the graph directly - print("\nRunning the complete researcher workflow...") - result = await graph.ainvoke(initial_state, config) - - # Extract the answer outline for display - if "answer_outline" in result and result["answer_outline"]: - print(f"\nGenerated answer outline with {len(result['answer_outline'].answer_outline)} sections") - - # Print the outline - print("\nGenerated Answer Outline:") - for section in result["answer_outline"].answer_outline: - print(f"\nSection {section.section_id}: {section.section_title}") - print("Research Questions:") - for q in section.questions: - print(f" - {q}") - - # Check if we got a final report - if "final_written_report" in result and result["final_written_report"]: - final_report = result["final_written_report"] - print("\nFinal Research Report generated successfully!") - print(f"Report length: {len(final_report)} characters") - - # Display the final report - print("\n==== FINAL RESEARCH REPORT ====\n") - print(final_report) - else: - print("\nNo final report was generated.") - print(f"Available result keys: {list(result.keys())}") - - return result - - except Exception as e: - print(f"Error running researcher agent: {str(e)}") - import traceback - traceback.print_exc() - raise - -async def main(): - """Main entry point for the test script.""" - try: - result = await run_test() - print("\nTest completed successfully.") - return result - except Exception as e: - print(f"\nTest failed with error: {e}") - import traceback - traceback.print_exc() - return None - -if __name__ == "__main__": - # Run the async test - result = asyncio.run(main()) - - # Keep terminal open if run directly in VSCode - if 'VSCODE_PID' in os.environ: - input("\nPress Enter to close this window...") \ No newline at end of file diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index e10aa50f6..74ea97b06 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -1,14 +1,15 @@ -from fastapi import APIRouter, Depends, HTTPException, Query -from fastapi.responses import StreamingResponse -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select -from sqlalchemy.exc import IntegrityError, OperationalError from typing import List -from app.db import get_async_session, User, SearchSpace, Chat -from app.schemas import ChatCreate, ChatUpdate, ChatRead, AISDKChatRequest + +from app.db import Chat, SearchSpace, User, get_async_session +from app.schemas import AISDKChatRequest, ChatCreate, ChatRead, ChatUpdate from app.tasks.stream_connector_search_results import stream_connector_search_results from app.users import current_active_user from app.utils.check_ownership import check_ownership +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import StreamingResponse +from sqlalchemy.exc import IntegrityError, OperationalError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select router = APIRouter() diff --git a/surfsense_backend/app/tasks/stream_connector_search_results.py b/surfsense_backend/app/tasks/stream_connector_search_results.py index a1dc0a3e2..c7eb07627 100644 --- a/surfsense_backend/app/tasks/stream_connector_search_results.py +++ b/surfsense_backend/app/tasks/stream_connector_search_results.py @@ -1,20 +1,15 @@ -import json -from sqlalchemy.ext.asyncio import AsyncSession -from typing import List, AsyncGenerator, Dict, Any -import asyncio -import re +from typing import AsyncGenerator, List, Union +from uuid import UUID -from app.utils.connector_service import ConnectorService -from app.utils.research_service import ResearchService +from app.agents.researcher.graph import graph as researcher_graph +from app.agents.researcher.state import State from app.utils.streaming_service import StreamingService -from app.utils.reranker_service import RerankerService -from app.utils.query_service import QueryService -from app.config import config -from app.utils.document_converters import convert_chunks_to_langchain_documents +from sqlalchemy.ext.asyncio import AsyncSession + async def stream_connector_search_results( user_query: str, - user_id: str, + user_id: Union[str, UUID], search_space_id: int, session: AsyncSession, research_mode: str, @@ -25,7 +20,7 @@ async def stream_connector_search_results( Args: user_query: The user's query - user_id: The user's ID + user_id: The user's ID (can be UUID object or string) search_space_id: The search space ID session: The database session research_mode: The research mode @@ -34,418 +29,45 @@ async def stream_connector_search_results( Yields: str: Formatted response strings """ - # Initialize services - connector_service = ConnectorService(session) streaming_service = StreamingService() - - # Reformulate the user query using the strategic LLM - yield streaming_service.add_terminal_message("Reformulating your query for better results...", "info") - reformulated_query = await QueryService.reformulate_query(user_query) - yield streaming_service.add_terminal_message(f"Searching for: {reformulated_query}", "success") - - reranker_service = RerankerService.get_reranker_instance(config) - - all_raw_documents = [] # Store all raw documents before reranking - all_sources = [] - TOP_K = 20 - if research_mode == "GENERAL": - TOP_K = 20 + NUM_SECTIONS = 1 elif research_mode == "DEEP": - TOP_K = 40 + NUM_SECTIONS = 3 elif research_mode == "DEEPER": - TOP_K = 60 + NUM_SECTIONS = 6 - - # Process each selected connector - for connector in selected_connectors: - if connector == "YOUTUBE_VIDEO": - # Send terminal message about starting search - yield streaming_service.add_terminal_message("Starting to search for youtube videos...") - - # Search for YouTube videos using reformulated query - result_object, youtube_chunks = await connector_service.search_youtube( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - - # Send terminal message about search results - yield streaming_service.add_terminal_message( - f"Found {len(result_object['sources'])} relevant YouTube videos", - "success" - ) - - # Update sources - all_sources.append(result_object) - yield streaming_service.update_sources(all_sources) - - # Add documents to collection - all_raw_documents.extend(youtube_chunks) - - - # Extension Docs - if connector == "EXTENSION": - # Send terminal message about starting search - yield streaming_service.add_terminal_message("Starting to search for extension...") - - # Search for crawled URLs using reformulated query - result_object, extension_chunks = await connector_service.search_extension( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - - # Send terminal message about search results - yield streaming_service.add_terminal_message( - f"Found {len(result_object['sources'])} relevant extension documents", - "success" - ) - - # Update sources - all_sources.append(result_object) - yield streaming_service.update_sources(all_sources) - - # Add documents to collection - all_raw_documents.extend(extension_chunks) - - - # Crawled URLs - if connector == "CRAWLED_URL": - # Send terminal message about starting search - yield streaming_service.add_terminal_message("Starting to search for crawled URLs...") - - # Search for crawled URLs using reformulated query - result_object, crawled_urls_chunks = await connector_service.search_crawled_urls( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - - # Send terminal message about search results - yield streaming_service.add_terminal_message( - f"Found {len(result_object['sources'])} relevant crawled URLs", - "success" - ) - - # Update sources - all_sources.append(result_object) - yield streaming_service.update_sources(all_sources) - - # Add documents to collection - all_raw_documents.extend(crawled_urls_chunks) - - - # Files - if connector == "FILE": - # Send terminal message about starting search - yield streaming_service.add_terminal_message("Starting to search for files...") - - # Search for files using reformulated query - result_object, files_chunks = await connector_service.search_files( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - - # Send terminal message about search results - yield streaming_service.add_terminal_message( - f"Found {len(result_object['sources'])} relevant files", - "success" - ) - - # Update sources - all_sources.append(result_object) - yield streaming_service.update_sources(all_sources) - - # Add documents to collection - all_raw_documents.extend(files_chunks) - - # Tavily Connector - if connector == "TAVILY_API": - # Send terminal message about starting search - yield streaming_service.add_terminal_message("Starting to search with Tavily API...") - - # Search using Tavily API with reformulated query - result_object, tavily_chunks = await connector_service.search_tavily( - user_query=reformulated_query, - user_id=user_id, - top_k=TOP_K - ) - - # Send terminal message about search results - yield streaming_service.add_terminal_message( - f"Found {len(result_object['sources'])} relevant results from Tavily", - "success" - ) - - # Update sources - all_sources.append(result_object) - yield streaming_service.update_sources(all_sources) - - # Add documents to collection - all_raw_documents.extend(tavily_chunks) - - # Slack Connector - if connector == "SLACK_CONNECTOR": - # Send terminal message about starting search - yield streaming_service.add_terminal_message("Starting to search for slack connector...") - - # Search using Slack API with reformulated query - result_object, slack_chunks = await connector_service.search_slack( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - - # Send terminal message about search results - yield streaming_service.add_terminal_message( - f"Found {len(result_object['sources'])} relevant results from Slack", - "success" - ) - - # Update sources - all_sources.append(result_object) - yield streaming_service.update_sources(all_sources) - - # Add documents to collection - all_raw_documents.extend(slack_chunks) - - - # Notion Connector - if connector == "NOTION_CONNECTOR": - # Send terminal message about starting search - yield streaming_service.add_terminal_message("Starting to search for notion connector...") - - # Search using Notion API with reformulated query - result_object, notion_chunks = await connector_service.search_notion( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - - # Send terminal message about search results - yield streaming_service.add_terminal_message( - f"Found {len(result_object['sources'])} relevant results from Notion", - "success" - ) - - # Update sources - all_sources.append(result_object) - yield streaming_service.update_sources(all_sources) - - # Add documents to collection - all_raw_documents.extend(notion_chunks) - - - # Github Connector - if connector == "GITHUB_CONNECTOR": - # Send terminal message about starting search - yield streaming_service.add_terminal_message("Starting to search for GitHub connector...") - print("Starting to search for GitHub connector...") - # Search using Github API with reformulated query - result_object, github_chunks = await connector_service.search_github( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - - # Send terminal message about search results - yield streaming_service.add_terminal_message( - f"Found {len(result_object['sources'])} relevant results from Github", - "success" - ) - - # Update sources - all_sources.append(result_object) - yield streaming_service.update_sources(all_sources) - - # Add documents to collection - all_raw_documents.extend(github_chunks) - - # Linear Connector - if connector == "LINEAR_CONNECTOR": - # Send terminal message about starting search - yield streaming_service.add_terminal_message("Starting to search for Linear issues...") - - # Search using Linear API with reformulated query - result_object, linear_chunks = await connector_service.search_linear( - user_query=reformulated_query, - user_id=user_id, - search_space_id=search_space_id, - top_k=TOP_K - ) - - # Send terminal message about search results - yield streaming_service.add_terminal_message( - f"Found {len(result_object['sources'])} relevant results from Linear", - "success" - ) - - # Update sources - all_sources.append(result_object) - yield streaming_service.update_sources(all_sources) - - # Add documents to collection - all_raw_documents.extend(linear_chunks) - - + # Convert UUID to string if needed + user_id_str = str(user_id) if isinstance(user_id, UUID) else user_id - - # If we have documents to research - if all_raw_documents: - # Rerank all documents if reranker is available - if reranker_service: - yield streaming_service.add_terminal_message("Reranking documents for better relevance...", "info") - - # Convert documents to format expected by reranker - reranker_input_docs = [ - { - "chunk_id": doc.get("chunk_id", f"chunk_{i}"), - "content": doc.get("content", ""), - "score": doc.get("score", 0.0), - "document": { - "id": doc.get("document", {}).get("id", ""), - "title": doc.get("document", {}).get("title", ""), - "document_type": doc.get("document", {}).get("document_type", ""), - "metadata": doc.get("document", {}).get("metadata", {}) - } - } for i, doc in enumerate(all_raw_documents) - ] - - # Rerank documents using the reformulated query - reranked_docs = reranker_service.rerank_documents(reformulated_query, reranker_input_docs) - - # Sort by score in descending order - reranked_docs.sort(key=lambda x: x.get("score", 0), reverse=True) - - - - # Convert back to langchain documents format - from langchain.schema import Document as LangchainDocument - all_langchain_documents_to_research = [ - LangchainDocument( - page_content= f"""{doc.get("document", {}).get("id", "")}{doc.get("content", "")}""", - metadata={ - # **doc.get("document", {}).get("metadata", {}), - # "score": doc.get("score", 0.0), - # "rank": doc.get("rank", 0), - # "document_id": doc.get("document", {}).get("id", ""), - # "document_title": doc.get("document", {}).get("title", ""), - # "document_type": doc.get("document", {}).get("document_type", ""), - # # Explicitly set source_id for citation purposes - "source_id": str(doc.get("document", {}).get("id", "")) - } - ) for doc in reranked_docs - ] - - yield streaming_service.add_terminal_message(f"Reranked {len(all_langchain_documents_to_research)} documents", "success") - else: - # Use raw documents if no reranker is available - all_langchain_documents_to_research = convert_chunks_to_langchain_documents(all_raw_documents) - - # Send terminal message about starting research - yield streaming_service.add_terminal_message("Starting to research...", "info") - - # Create a buffer to collect report content - report_buffer = [] - - - # Use the streaming research method - yield streaming_service.add_terminal_message("Generating report...", "info") - - # Create a wrapper to handle the streaming - class StreamHandler: - def __init__(self): - self.queue = asyncio.Queue() - - async def handle_progress(self, data): - result = None - if data.get("type") == "logs": - # Handle log messages - result = streaming_service.add_terminal_message(data.get("output", ""), "info") - elif data.get("type") == "report": - # Handle report content - content = data.get("output", "") - - # Fix incorrect citation formats using regex - - # More specific pattern to match only numeric citations in markdown-style links - # This matches patterns like ([1](https://github.com/...)) but not general links like ([Click here](https://...)) - pattern = r'\(\[(\d+)\]\((https?://[^\)]+)\)\)' - - # Replace with just [X] where X is the number - content = re.sub(pattern, r'[\1]', content) - - # Also match other incorrect formats like ([1]) and convert to [1] - # Only match if the content inside brackets is a number - content = re.sub(r'\(\[(\d+)\]\)', r'[\1]', content) - - report_buffer.append(content) - # Update the answer with the accumulated content - result = streaming_service.update_answer(report_buffer) - - if result: - await self.queue.put(result) - return result - - async def get_next(self): - try: - return await self.queue.get() - except Exception as e: - print(f"Error getting next item from queue: {e}") - return None - - def task_done(self): - self.queue.task_done() - - # Create the stream handler - stream_handler = StreamHandler() - - # Start the research process in a separate task - research_task = asyncio.create_task( - ResearchService.stream_research( - user_query=reformulated_query, - documents=all_langchain_documents_to_research, - on_progress=stream_handler.handle_progress, - research_mode=research_mode - ) - ) - - # Stream results as they become available - while not research_task.done() or not stream_handler.queue.empty(): - try: - # Get the next result with a timeout - result = await asyncio.wait_for(stream_handler.get_next(), timeout=0.1) - stream_handler.task_done() - yield result - except asyncio.TimeoutError: - # No result available yet, check if the research task is done - if research_task.done(): - # If the queue is empty and the task is done, we're finished - if stream_handler.queue.empty(): - break - - # Get the final report - try: - final_report = await research_task - - # Send terminal message about research completion - yield streaming_service.add_terminal_message("Research completed", "success") - - # Update the answer with the final report - final_report_lines = final_report.split('\n') - yield streaming_service.update_answer(final_report_lines) - except Exception as e: - # Handle any exceptions - yield streaming_service.add_terminal_message(f"Error during research: {str(e)}", "error") - - # Send completion message - yield streaming_service.format_completion() + # Sample configuration + config = { + "configurable": { + "user_query": user_query, + "num_sections": NUM_SECTIONS, + "connectors_to_search": selected_connectors, + "user_id": user_id_str, + "search_space_id": search_space_id + } + } + # Initialize state with database session and streaming service + initial_state = State( + db_session=session, + streaming_service=streaming_service + ) + + # Run the graph directly + print("\nRunning the complete researcher workflow...") + + # Use streaming with config parameter + async for chunk in researcher_graph.astream( + initial_state, + config=config, + stream_mode="custom", + ): + # If the chunk contains a 'yeild_value' key, print its value + # Note: there's a typo in 'yeild_value' in the code, but we need to match it + if isinstance(chunk, dict) and 'yeild_value' in chunk: + yield chunk['yeild_value'] + + yield streaming_service.format_completion() \ No newline at end of file diff --git a/surfsense_backend/app/utils/query_service.py b/surfsense_backend/app/utils/query_service.py index b5df744a5..760f0c8fa 100644 --- a/surfsense_backend/app/utils/query_service.py +++ b/surfsense_backend/app/utils/query_service.py @@ -1,5 +1,7 @@ -from typing import Dict, Any -from langchain.schema import LLMResult, HumanMessage, SystemMessage +""" +NOTE: This is not used anymore. Might be removed in the future. +""" +from langchain.schema import HumanMessage, SystemMessage from app.config import config class QueryService: diff --git a/surfsense_backend/app/utils/research_service.py b/surfsense_backend/app/utils/research_service.py deleted file mode 100644 index c0034fd28..000000000 --- a/surfsense_backend/app/utils/research_service.py +++ /dev/null @@ -1,211 +0,0 @@ -import asyncio -import re -from typing import List, Dict, Any, AsyncGenerator, Callable, Optional -from langchain.schema import Document -from gpt_researcher.agent import GPTResearcher -from gpt_researcher.utils.enum import ReportType, Tone, ReportSource -from dotenv import load_dotenv - -load_dotenv() - -class ResearchService: - @staticmethod - async def create_custom_prompt(user_query: str) -> str: - citation_prompt = f""" - You are a research assistant tasked with analyzing documents and providing comprehensive answers with proper citations in IEEE format. - - - 1. Carefully analyze all provided documents in the section's. - 2. Extract relevant information that addresses the user's query. - 3. Synthesize a comprehensive, well-structured answer using information from these documents. - 4. For EVERY piece of information you include from the documents, add an IEEE-style citation in square brackets [X] where X is the source_id from the document's metadata. - 5. Make sure ALL factual statements from the documents have proper citations. - 6. If multiple documents support the same point, include all relevant citations [X], [Y]. - 7. Present information in a logical, coherent flow. - 8. Use your own words to connect ideas, but cite ALL information from the documents. - 9. If documents contain conflicting information, acknowledge this and present both perspectives with appropriate citations. - 10. Do not make up or include information not found in the provided documents. - 11. CRITICAL: You MUST use the exact source_id value from each document's metadata for citations. Do not create your own citation numbers. - 12. CRITICAL: Every citation MUST be in the IEEE format [X] where X is the exact source_id value. - 13. CRITICAL: Never renumber or reorder citations - always use the original source_id values. - 14. CRITICAL: Do not return citations as clickable links. - 15. CRITICAL: Never format citations as markdown links like "([1](https://example.com))". Always use plain square brackets only. - 16. CRITICAL: Citations must ONLY appear as [X] or [X], [Y], [Z] format - never with parentheses, hyperlinks, or other formatting. - 17. CRITICAL: Never make up citation numbers. Only use source_id values that are explicitly provided in the document metadata. - 18. CRITICAL: If you are unsure about a source_id, do not include a citation rather than guessing or making one up. - - - - - Write in clear, professional language suitable for academic or technical audiences - - Organize your response with appropriate paragraphs, headings, and structure - - Every fact from the documents must have an IEEE-style citation in square brackets [X] where X is the EXACT source_id from the document's metadata - - Citations should appear at the end of the sentence containing the information they support - - Multiple citations should be separated by commas: [X], [Y], [Z] - - No need to return references section. Just citation numbers in answer. - - NEVER create your own citation numbering system - use the exact source_id values from the documents. - - NEVER format citations as clickable links or as markdown links like "([1](https://example.com))". Always use plain square brackets only. - - NEVER make up citation numbers if you are unsure about the source_id. It is better to omit the citation than to guess. - - - - - - 1 - - - - The Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia. It comprises over 2,900 individual reefs and 900 islands. - - - - - - - 13 - - - - Climate change poses a significant threat to coral reefs worldwide. Rising ocean temperatures have led to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020. - - - - - - - 21 - - - - The Great Barrier Reef was designated a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity. It is home to over 1,500 species of fish and 400 types of coral. - - - - - - - The Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia [1]. It was designated a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity [21]. The reef is home to over 1,500 species of fish and 400 types of coral [21]. Unfortunately, climate change poses a significant threat to coral reefs worldwide, with rising ocean temperatures leading to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020 [13]. The reef system comprises over 2,900 individual reefs and 900 islands [1], making it an ecological treasure that requires protection from multiple threats [1], [13]. - - - - DO NOT use any of these incorrect citation formats: - - Using parentheses and markdown links: ([1](https://github.com/MODSetter/SurfSense)) - - Using parentheses around brackets: ([1]) - - Using hyperlinked text: [link to source 1](https://example.com) - - Using footnote style: ... reef system¹ - - Making up citation numbers when source_id is unknown - - ONLY use plain square brackets [1] or multiple citations [1], [2], [3] - - - Note that the citation numbers match exactly with the source_id values (1, 13, and 21) and are not renumbered sequentially. Citations follow IEEE style with square brackets and appear at the end of sentences. - - Now, please research the following query: - - - {user_query} - - """ - - return citation_prompt - - - @staticmethod - async def stream_research( - user_query: str, - documents: List[Document] = None, - on_progress: Optional[Callable] = None, - research_mode: str = "GENERAL" - ) -> str: - """ - Stream the research process using GPTResearcher - - Args: - user_query: The user's query - documents: List of Document objects to use for research - on_progress: Optional callback for progress updates - research_mode: Research mode to use - - Returns: - str: The final research report - """ - # Create a custom websocket-like object to capture streaming output - class StreamingWebsocket: - async def send_json(self, data): - if on_progress: - try: - # Filter out excessive logging of the prompt - if data.get("type") == "logs": - output = data.get("output", "") - # Check if this is a verbose prompt log - if "You are a research assistant tasked with analyzing documents" in output and len(output) > 500: - # Replace with a shorter message - data["output"] = f"Processing research for query: {user_query}" - - result = await on_progress(data) - return result - except Exception as e: - print(f"Error in on_progress callback: {e}") - return None - - streaming_websocket = StreamingWebsocket() - - custom_prompt_for_ieee_citations = await ResearchService.create_custom_prompt(user_query) - - if(research_mode == "GENERAL"): - research_report_type = ReportType.CustomReport.value - elif(research_mode == "DEEP"): - research_report_type = ReportType.ResearchReport.value - elif(research_mode == "DEEPER"): - research_report_type = ReportType.DetailedReport.value - # elif(research_mode == "DEEPEST"): - # research_report_type = ReportType.DeepResearch.value - - # Initialize GPTResearcher with the streaming websocket - researcher = GPTResearcher( - query=custom_prompt_for_ieee_citations, - report_type=research_report_type, - report_format="IEEE", - report_source=ReportSource.LangChainDocuments.value, - tone=Tone.Formal, - documents=documents, - verbose=True, - websocket=streaming_websocket - ) - - # Conduct research - await researcher.conduct_research() - - # Generate report with streaming - report = await researcher.write_report() - - # Fix citation format - report = ResearchService.fix_citation_format(report) - - return report - - @staticmethod - def fix_citation_format(text: str) -> str: - """ - Fix any incorrectly formatted citations in the text. - - Args: - text: The text to fix - - Returns: - str: The text with fixed citations - """ - if not text: - return text - - # More specific pattern to match only numeric citations in markdown-style links - # This matches patterns like ([1](https://github.com/...)) but not general links like ([Click here](https://...)) - pattern = r'\(\[(\d+)\]\((https?://[^\)]+)\)\)' - - # Replace with just [X] where X is the number - text = re.sub(pattern, r'[\1]', text) - - # Also match other incorrect formats like ([1]) and convert to [1] - # Only match if the content inside brackets is a number - text = re.sub(r'\(\[(\d+)\]\)', r'[\1]', text) - - return text \ No newline at end of file diff --git a/surfsense_backend/app/utils/streaming_service.py b/surfsense_backend/app/utils/streaming_service.py index 4f2b7d9c2..08a47a943 100644 --- a/surfsense_backend/app/utils/streaming_service.py +++ b/surfsense_backend/app/utils/streaming_service.py @@ -1,5 +1,6 @@ import json -from typing import List, Dict, Any, Generator +from typing import Any, Dict, List + class StreamingService: def __init__(self): @@ -18,55 +19,7 @@ class StreamingService: "content": [] } ] - - def add_terminal_message(self, text: str, message_type: str = "info") -> str: - """ - Add a terminal message to the annotations and return the formatted response - - Args: - text: The message text - message_type: The message type (info, success, error) - - Returns: - str: The formatted response string - """ - self.message_annotations[0]["content"].append({ - "id": self.terminal_idx, - "text": text, - "type": message_type - }) - self.terminal_idx += 1 - return self._format_annotations() - - def update_sources(self, sources: List[Dict[str, Any]]) -> str: - """ - Update the sources in the annotations and return the formatted response - - Args: - sources: List of source objects - - Returns: - str: The formatted response string - """ - self.message_annotations[1]["content"] = sources - return self._format_annotations() - - def update_answer(self, answer_content: List[str]) -> str: - """ - Update the answer in the annotations and return the formatted response - - Args: - answer_content: The answer content as a list of strings - - Returns: - str: The formatted response string - """ - self.message_annotations[2] = { - "type": "ANSWER", - "content": answer_content - } - return self._format_annotations() - + # It is used to send annotations to the frontend def _format_annotations(self) -> str: """ Format the annotations as a string @@ -76,6 +29,7 @@ class StreamingService: """ return f'8:{json.dumps(self.message_annotations)}\n' + # It is used to end Streaming def format_completion(self, prompt_tokens: int = 156, completion_tokens: int = 204) -> str: """ Format a completion message @@ -96,4 +50,23 @@ class StreamingService: "totalTokens": total_tokens } } - return f'd:{json.dumps(completion_data)}\n' \ No newline at end of file + return f'd:{json.dumps(completion_data)}\n' + + def only_update_terminal(self, text: str, message_type: str = "info") -> str: + self.message_annotations[0]["content"].append({ + "id": self.terminal_idx, + "text": text, + "type": message_type + }) + self.terminal_idx += 1 + return self.message_annotations + + def only_update_sources(self, sources: List[Dict[str, Any]]) -> str: + self.message_annotations[1]["content"] = sources + return self.message_annotations + + def only_update_answer(self, answer: List[str]) -> str: + self.message_annotations[2]["content"] = answer + return self.message_annotations + + \ No newline at end of file diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 0d8682eb2..95b111e22 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -11,7 +11,6 @@ dependencies = [ "fastapi>=0.115.8", "fastapi-users[oauth,sqlalchemy]>=14.0.1", "firecrawl-py>=1.12.0", - "gpt-researcher>=0.12.12", "github3.py==4.0.1", "langchain-community>=0.3.17", "langchain-unstructured>=0.1.6", diff --git a/surfsense_web/components/markdown-viewer.tsx b/surfsense_web/components/markdown-viewer.tsx index 4398d1067..03fe7bada 100644 --- a/surfsense_web/components/markdown-viewer.tsx +++ b/surfsense_web/components/markdown-viewer.tsx @@ -117,13 +117,21 @@ function processCitationsInReactChildren(children: React.ReactNode, getCitationS // Process citation references in text content function processCitationsInText(text: string, getCitationSource: (id: number) => Source | null): React.ReactNode[] { + // Use improved regex to catch citation numbers more reliably + // This will match patterns like [1], [42], etc. including when they appear at the end of a line or sentence const citationRegex = /\[(\d+)\]/g; const parts: React.ReactNode[] = []; let lastIndex = 0; let match; let position = 0; + // Debug log for troubleshooting + console.log("Processing citations in text:", text); + while ((match = citationRegex.exec(text)) !== null) { + // Log each match for debugging + console.log("Citation match found:", match[0], "at index", match.index); + // Add text before the citation if (match.index > lastIndex) { parts.push(text.substring(lastIndex, match.index)); @@ -131,13 +139,18 @@ function processCitationsInText(text: string, getCitationSource: (id: number) => // Add the citation component const citationId = parseInt(match[1], 10); + const source = getCitationSource(citationId); + + // Log the citation details + console.log("Citation ID:", citationId, "Source:", source ? "found" : "not found"); + parts.push( ); From a1aad295bb1b8398df63c1f3e247f5931358a02b Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 20 Apr 2025 23:34:21 -0700 Subject: [PATCH 25/31] chore: cleanup --- README.md | 8 ++++---- surfsense_backend/.env.example | 9 ++++++--- .../researcher/sub_section_writer/nodes.py | 5 +++-- surfsense_backend/app/config/__init__.py | 17 +++-------------- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index aa337c7f2..d8c9cad61 100644 --- a/README.md +++ b/README.md @@ -119,13 +119,13 @@ This is the core of SurfSense. Before we begin let's look at `.env` variables' t | EMBEDDING_MODEL| Name of the embedding model to use for vector embeddings. Currently works with Sentence Transformers only. Expect other embeddings soon. Eg. `mixedbread-ai/mxbai-embed-large-v1`| | RERANKERS_MODEL_NAME| Name of the reranker model for search result reranking. Eg. `ms-marco-MiniLM-L-12-v2`| | RERANKERS_MODEL_TYPE| Type of reranker model being used. Eg. `flashrank`| -| FAST_LLM| LiteLLM routed Smaller, faster LLM for quick responses. Eg. `litellm:openai/gpt-4o`| -| STRATEGIC_LLM| LiteLLM routed Advanced LLM for complex reasoning tasks. Eg. `litellm:openai/gpt-4o`| -| LONG_CONTEXT_LLM| LiteLLM routed LLM capable of handling longer context windows. Eg. `litellm:gemini/gemini-2.0-flash`| +| FAST_LLM| LiteLLM routed Smaller, faster LLM for quick responses. Eg. `openai/gpt-4o-mini`, `ollama/deepseek-r1:8b`| +| STRATEGIC_LLM| LiteLLM routed Advanced LLM for complex reasoning tasks. Eg. `openai/gpt-4o`, `ollama/gemma3:12b`| +| LONG_CONTEXT_LLM| LiteLLM routed LLM capable of handling longer context windows. Eg. `gemini/gemini-2.0-flash`, `ollama/deepseek-r1:8b`| | UNSTRUCTURED_API_KEY| API key for Unstructured.io service for document parsing| | FIRECRAWL_API_KEY| API key for Firecrawl service for web crawling and data extraction| -IMPORTANT: Since LLM calls are routed through LiteLLM make sure to include API keys of LLM models you are using. For example if you used `litellm:openai/gpt-4o` make sure to include OpenAI API Key `OPENAI_API_KEY` or if you use `litellm:gemini/gemini-2.0-flash` then you include `GEMINI_API_KEY`. +IMPORTANT: Since LLM calls are routed through LiteLLM make sure to include API keys of LLM models you are using. For example if you used `openai/gpt-4o` make sure to include OpenAI API Key `OPENAI_API_KEY` or if you use `gemini/gemini-2.0-flash` then you include `GEMINI_API_KEY`. You can also integrate any LLM just follow this https://docs.litellm.ai/docs/providers diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index bc3f5cebb..2eee50d08 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -4,15 +4,18 @@ SECRET_KEY="SECRET" GOOGLE_OAUTH_CLIENT_ID="924507538m" GOOGLE_OAUTH_CLIENT_SECRET="GOCSV" NEXT_FRONTEND_URL="http://localhost:3000" + EMBEDDING_MODEL="mixedbread-ai/mxbai-embed-large-v1" RERANKERS_MODEL_NAME="ms-marco-MiniLM-L-12-v2" RERANKERS_MODEL_TYPE="flashrank" -FAST_LLM="litellm:openai/gpt-4o-mini" -STRATEGIC_LLM="litellm:openai/gpt-4o" -LONG_CONTEXT_LLM="litellm:gemini/gemini-2.0-flash" +# https://docs.litellm.ai/docs/providers +FAST_LLM="openai/gpt-4o-mini" +STRATEGIC_LLM="openai/gpt-4o" +LONG_CONTEXT_LLM="gemini/gemini-2.0-flash" +# Chosen LiteLLM Providers Keys OPENAI_API_KEY="sk-proj-iA" GEMINI_API_KEY="AIzaSyB6-1641124124124124124124124124124" diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py index 5b11141ca..0bec4618c 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -37,8 +37,9 @@ async def rerank_documents(state: State, config: RunnableConfig) -> Dict[str, An if reranker_service: try: # Use the sub-section questions for reranking context - rerank_query = "\n".join(sub_section_questions) - + # rerank_query = "\n".join(sub_section_questions) + rerank_query = configuration.user_query + # Convert documents to format expected by reranker if needed reranker_input_docs = [ { diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 82517a8df..c7f842b71 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -15,17 +15,6 @@ env_file = BASE_DIR / ".env" load_dotenv(env_file) -def extract_model_name(llm_string: str) -> str: - """Extract the model name from an LLM string. - Example: "litellm:openai/gpt-4o-mini" -> "openai/gpt-4o-mini" - - Args: - llm_string: The LLM string with optional prefix - - Returns: - str: The extracted model name - """ - return llm_string.split(":", 1)[1] if ":" in llm_string else llm_string class Config: # Database @@ -38,13 +27,13 @@ class Config: # LONG-CONTEXT LLMS LONG_CONTEXT_LLM = os.getenv("LONG_CONTEXT_LLM") - long_context_llm_instance = ChatLiteLLM(model=extract_model_name(LONG_CONTEXT_LLM)) + long_context_llm_instance = ChatLiteLLM(model=LONG_CONTEXT_LLM) # GPT Researcher FAST_LLM = os.getenv("FAST_LLM") STRATEGIC_LLM = os.getenv("STRATEGIC_LLM") - fast_llm_instance = ChatLiteLLM(model=extract_model_name(FAST_LLM)) - strategic_llm_instance = ChatLiteLLM(model=extract_model_name(STRATEGIC_LLM)) + fast_llm_instance = ChatLiteLLM(model=FAST_LLM) + strategic_llm_instance = ChatLiteLLM(model=STRATEGIC_LLM) # Chonkie Configuration | Edit this to your needs From 5fecca945763419f8db58a94b377dda46c025105 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 21 Apr 2025 01:36:19 -0700 Subject: [PATCH 26/31] feat: Added envs for LLM Observability --- surfsense_backend/.env.example | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 2eee50d08..6dfcc9967 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -21,3 +21,9 @@ GEMINI_API_KEY="AIzaSyB6-1641124124124124124124124124124" UNSTRUCTURED_API_KEY="Tpu3P0U8iy" FIRECRAWL_API_KEY="fcr-01J0000000000000000000000" + +#OPTIONAL: Add these for LangSmith Observability +LANGSMITH_TRACING=true +LANGSMITH_ENDPOINT="https://api.smith.langchain.com" +LANGSMITH_API_KEY="lsv2_pt_....." +LANGSMITH_PROJECT="surfsense" From b19e241ca2d174d25c0eb74c6638a80f412a0d7a Mon Sep 17 00:00:00 2001 From: Rohan Verma <122026167+MODSetter@users.noreply.github.com> Date: Mon, 21 Apr 2025 01:42:38 -0700 Subject: [PATCH 27/31] Update README.md Basic LangSmith Info --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index d8c9cad61..4cb51fa42 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,16 @@ SurfSense now only works with Google OAuth. Make sure to set your OAuth Client a ![gauth](https://github.com/user-attachments/assets/80d60fe5-889b-48a6-b947-200fdaf544c1) +#### LLM Observability +One easy way to observe SurfSense Researcher Agent is to use LangSmith. Get its API KEY from https://smith.langchain.com/ + +**Open AI LLMS** +![openai_langraph](https://github.com/user-attachments/assets/b1f4c7a1-0a66-4d21-9053-2e09a5634f95) + + +**Ollama LLMS** +![ollama_langgraph](https://github.com/user-attachments/assets/5b6c870e-095c-4368-86e6-f7488e0fca28) + #### Crawler Support SurfSense currently uses [Firecrawl.py](https://www.firecrawl.dev/) right now. Playwright crawler support will be added soon. From 3155e7b0ac1b51732f7ea3f6ca610a141024007d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 21 Apr 2025 23:58:32 -0700 Subject: [PATCH 28/31] chore: Added Versions --- surfsense_backend/pyproject.toml | 4 +- surfsense_backend/uv.lock | 364 +---------------------- surfsense_browser_extension/package.json | 6 +- surfsense_web/package.json | 5 +- 4 files changed, 9 insertions(+), 370 deletions(-) diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 95b111e22..7b7a6f900 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "surf-new-backend" -version = "0.1.0" -description = "Add your description here" +version = "0.0.6" +description = "SurfSense Backend" readme = "README.md" requires-python = ">=3.12" dependencies = [ diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index 6e9c3b51a..9b485b0df 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -168,19 +168,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/e4/bf8034d25edaa495da3c8a3405627d2e35758e44ff6eaa7948092646fdcc/argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93", size = 53104 }, ] -[[package]] -name = "arxiv" -version = "2.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "feedparser" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fe/59/fe41f54bdfed776c2e9bcd6289e4c71349eb938241d89b4c97d0f33e8013/arxiv-2.1.3.tar.gz", hash = "sha256:32365221994d2cf05657c1fadf63a26efc8ccdec18590281ee03515bfef8bc4e", size = 16747 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/7b/7bf42178d227b26d3daf94cdd22a72a4ed5bf235548c4f5aea49c51c6458/arxiv-2.1.3-py3-none-any.whl", hash = "sha256:6f43673ab770a9e848d7d4fc1894824df55edeac3c3572ea280c9ba2e3c0f39f", size = 11478 }, -] - [[package]] name = "asyncpg" version = "0.30.0" @@ -279,61 +266,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/49/6abb616eb3cbab6a7cca303dc02fdf3836de2e0b834bf966a7f5271a34d8/beautifulsoup4-4.13.3-py3-none-any.whl", hash = "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16", size = 186015 }, ] -[[package]] -name = "brotli" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/c2/f9e977608bdf958650638c3f1e28f85a1b075f075ebbe77db8555463787b/Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724", size = 7372270 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/d0/5373ae13b93fe00095a58efcbce837fd470ca39f703a235d2a999baadfbc/Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28", size = 815693 }, - { url = "https://files.pythonhosted.org/packages/8e/48/f6e1cdf86751300c288c1459724bfa6917a80e30dbfc326f92cea5d3683a/Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f", size = 422489 }, - { url = "https://files.pythonhosted.org/packages/06/88/564958cedce636d0f1bed313381dfc4b4e3d3f6015a63dae6146e1b8c65c/Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409", size = 873081 }, - { url = "https://files.pythonhosted.org/packages/58/79/b7026a8bb65da9a6bb7d14329fd2bd48d2b7f86d7329d5cc8ddc6a90526f/Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2", size = 446244 }, - { url = "https://files.pythonhosted.org/packages/e5/18/c18c32ecea41b6c0004e15606e274006366fe19436b6adccc1ae7b2e50c2/Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451", size = 2906505 }, - { url = "https://files.pythonhosted.org/packages/08/c8/69ec0496b1ada7569b62d85893d928e865df29b90736558d6c98c2031208/Brotli-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91", size = 2944152 }, - { url = "https://files.pythonhosted.org/packages/ab/fb/0517cea182219d6768113a38167ef6d4eb157a033178cc938033a552ed6d/Brotli-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408", size = 2919252 }, - { url = "https://files.pythonhosted.org/packages/c7/53/73a3431662e33ae61a5c80b1b9d2d18f58dfa910ae8dd696e57d39f1a2f5/Brotli-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0", size = 2845955 }, - { url = "https://files.pythonhosted.org/packages/55/ac/bd280708d9c5ebdbf9de01459e625a3e3803cce0784f47d633562cf40e83/Brotli-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc", size = 2914304 }, - { url = "https://files.pythonhosted.org/packages/76/58/5c391b41ecfc4527d2cc3350719b02e87cb424ef8ba2023fb662f9bf743c/Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180", size = 2814452 }, - { url = "https://files.pythonhosted.org/packages/c7/4e/91b8256dfe99c407f174924b65a01f5305e303f486cc7a2e8a5d43c8bec3/Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248", size = 2938751 }, - { url = "https://files.pythonhosted.org/packages/5a/a6/e2a39a5d3b412938362bbbeba5af904092bf3f95b867b4a3eb856104074e/Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966", size = 2933757 }, - { url = "https://files.pythonhosted.org/packages/13/f0/358354786280a509482e0e77c1a5459e439766597d280f28cb097642fc26/Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9", size = 2936146 }, - { url = "https://files.pythonhosted.org/packages/80/f7/daf538c1060d3a88266b80ecc1d1c98b79553b3f117a485653f17070ea2a/Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb", size = 2848055 }, - { url = "https://files.pythonhosted.org/packages/ad/cf/0eaa0585c4077d3c2d1edf322d8e97aabf317941d3a72d7b3ad8bce004b0/Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111", size = 3035102 }, - { url = "https://files.pythonhosted.org/packages/d8/63/1c1585b2aa554fe6dbce30f0c18bdbc877fa9a1bf5ff17677d9cca0ac122/Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839", size = 2930029 }, - { url = "https://files.pythonhosted.org/packages/5f/3b/4e3fd1893eb3bbfef8e5a80d4508bec17a57bb92d586c85c12d28666bb13/Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0", size = 333276 }, - { url = "https://files.pythonhosted.org/packages/3d/d5/942051b45a9e883b5b6e98c041698b1eb2012d25e5948c58d6bf85b1bb43/Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951", size = 357255 }, - { url = "https://files.pythonhosted.org/packages/0a/9f/fb37bb8ffc52a8da37b1c03c459a8cd55df7a57bdccd8831d500e994a0ca/Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5", size = 815681 }, - { url = "https://files.pythonhosted.org/packages/06/b3/dbd332a988586fefb0aa49c779f59f47cae76855c2d00f450364bb574cac/Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8", size = 422475 }, - { url = "https://files.pythonhosted.org/packages/bb/80/6aaddc2f63dbcf2d93c2d204e49c11a9ec93a8c7c63261e2b4bd35198283/Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f", size = 2906173 }, - { url = "https://files.pythonhosted.org/packages/ea/1d/e6ca79c96ff5b641df6097d299347507d39a9604bde8915e76bf026d6c77/Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648", size = 2943803 }, - { url = "https://files.pythonhosted.org/packages/ac/a3/d98d2472e0130b7dd3acdbb7f390d478123dbf62b7d32bda5c830a96116d/Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0", size = 2918946 }, - { url = "https://files.pythonhosted.org/packages/c4/a5/c69e6d272aee3e1423ed005d8915a7eaa0384c7de503da987f2d224d0721/Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089", size = 2845707 }, - { url = "https://files.pythonhosted.org/packages/58/9f/4149d38b52725afa39067350696c09526de0125ebfbaab5acc5af28b42ea/Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368", size = 2936231 }, - { url = "https://files.pythonhosted.org/packages/5a/5a/145de884285611838a16bebfdb060c231c52b8f84dfbe52b852a15780386/Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c", size = 2848157 }, - { url = "https://files.pythonhosted.org/packages/50/ae/408b6bfb8525dadebd3b3dd5b19d631da4f7d46420321db44cd99dcf2f2c/Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284", size = 3035122 }, - { url = "https://files.pythonhosted.org/packages/af/85/a94e5cfaa0ca449d8f91c3d6f78313ebf919a0dbd55a100c711c6e9655bc/Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7", size = 2930206 }, - { url = "https://files.pythonhosted.org/packages/c2/f0/a61d9262cd01351df22e57ad7c34f66794709acab13f34be2675f45bf89d/Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0", size = 333804 }, - { url = "https://files.pythonhosted.org/packages/7e/c1/ec214e9c94000d1c1974ec67ced1c970c148aa6b8d8373066123fc3dbf06/Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b", size = 358517 }, -] - -[[package]] -name = "brotlicffi" -version = "1.1.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/95/9d/70caa61192f570fcf0352766331b735afa931b4c6bc9a348a0925cc13288/brotlicffi-1.1.0.0.tar.gz", hash = "sha256:b77827a689905143f87915310b93b273ab17888fd43ef350d4832c4a71083c13", size = 465192 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/11/7b96009d3dcc2c931e828ce1e157f03824a69fb728d06bfd7b2fc6f93718/brotlicffi-1.1.0.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9b7ae6bd1a3f0df532b6d67ff674099a96d22bc0948955cb338488c31bfb8851", size = 453786 }, - { url = "https://files.pythonhosted.org/packages/d6/e6/a8f46f4a4ee7856fbd6ac0c6fb0dc65ed181ba46cd77875b8d9bbe494d9e/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19ffc919fa4fc6ace69286e0a23b3789b4219058313cf9b45625016bf7ff996b", size = 2911165 }, - { url = "https://files.pythonhosted.org/packages/be/20/201559dff14e83ba345a5ec03335607e47467b6633c210607e693aefac40/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9feb210d932ffe7798ee62e6145d3a757eb6233aa9a4e7db78dd3690d7755814", size = 2927895 }, - { url = "https://files.pythonhosted.org/packages/cd/15/695b1409264143be3c933f708a3f81d53c4a1e1ebbc06f46331decbf6563/brotlicffi-1.1.0.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84763dbdef5dd5c24b75597a77e1b30c66604725707565188ba54bab4f114820", size = 2851834 }, - { url = "https://files.pythonhosted.org/packages/b4/40/b961a702463b6005baf952794c2e9e0099bde657d0d7e007f923883b907f/brotlicffi-1.1.0.0-cp37-abi3-win32.whl", hash = "sha256:1b12b50e07c3911e1efa3a8971543e7648100713d4e0971b13631cce22c587eb", size = 341731 }, - { url = "https://files.pythonhosted.org/packages/1c/fa/5408a03c041114ceab628ce21766a4ea882aa6f6f0a800e04ee3a30ec6b9/brotlicffi-1.1.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:994a4f0681bb6c6c3b0925530a1926b7a189d878e6e5e38fae8efa47c5d9c613", size = 366783 }, -] - [[package]] name = "cachetools" version = "5.5.2" @@ -559,19 +491,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/05/5533d30f53f10239616a357f080892026db2d550a40c393d0a8a7af834a9/cryptography-44.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:e403f7f766ded778ecdb790da786b418a9f2394f36e8cc8b796cc056ab05f44f", size = 3207303 }, ] -[[package]] -name = "cssselect2" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tinycss2" }, - { name = "webencodings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9f/86/fd7f58fc498b3166f3a7e8e0cddb6e620fe1da35b02248b1bd59e95dbaaa/cssselect2-0.8.0.tar.gz", hash = "sha256:7674ffb954a3b46162392aee2a3a0aedb2e14ecf99fcc28644900f4e6e3e9d3a", size = 35716 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/e7/aa315e6a749d9b96c2504a1ba0ba031ba2d0517e972ce22682e3fccecb09/cssselect2-0.8.0-py3-none-any.whl", hash = "sha256:46fc70ebc41ced7a32cd42d58b1884d72ade23d21e5a4eaaf022401c13f0e76e", size = 15454 }, -] - [[package]] name = "cycler" version = "0.12.1" @@ -633,12 +552,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632 }, ] -[[package]] -name = "docopt" -version = "0.6.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/55/8f8cab2afd404cf578136ef2cc5dfb50baa1761b68c9da1fb1e4eed343c9/docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491", size = 25901 } - [[package]] name = "effdet" version = "0.4.1" @@ -747,18 +660,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/08/9968963c1fb8c34627b7f1fbcdfe9438540f87dc7c9bfb59bb4fd19a4ecf/fastapi_users_db_sqlalchemy-7.0.0-py3-none-any.whl", hash = "sha256:5fceac018e7cfa69efc70834dd3035b3de7988eb4274154a0dbe8b14f5aa001e", size = 6891 }, ] -[[package]] -name = "feedparser" -version = "6.0.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sgmllib3k" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ff/aa/7af346ebeb42a76bf108027fe7f3328bb4e57a3a96e53e21fd9ef9dd6dd0/feedparser-6.0.11.tar.gz", hash = "sha256:c9d0407b64c6f2a065d0ebb292c2b35c01050cc0dc33757461aaabdc4c4184d5", size = 286197 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/d4/8c31aad9cc18f451c49f7f9cfb5799dadffc88177f7917bc90a66459b1d7/feedparser-6.0.11-py3-none-any.whl", hash = "sha256:0be7ee7b395572b19ebeb1d6aafb0028dee11169f1c934e0ed67d54992f4ad45", size = 81343 }, -] - [[package]] name = "filelock" version = "3.17.0" @@ -843,13 +744,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/ff/44934a031ce5a39125415eb405b9efb76fe7f9586b75291d66ae5cbfc4e6/fonttools-4.56.0-py3-none-any.whl", hash = "sha256:1088182f68c303b50ca4dc0c82d42083d176cba37af1937e1a976a31149d4d14", size = 1089800 }, ] -[package.optional-dependencies] -woff = [ - { name = "brotli", marker = "platform_python_implementation == 'CPython'" }, - { name = "brotlicffi", marker = "platform_python_implementation != 'CPython'" }, - { name = "zopfli" }, -] - [[package]] name = "frozenlist" version = "1.5.0" @@ -976,42 +870,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/53/d35476d547a286506f0a6a634ccf1e5d288fffd53d48f0bd5fef61d68684/googleapis_common_protos-1.69.2-py3-none-any.whl", hash = "sha256:0b30452ff9c7a27d80bfc5718954063e8ab53dd3697093d3bc99581f5fd24212", size = 293215 }, ] -[[package]] -name = "gpt-researcher" -version = "0.12.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiofiles" }, - { name = "arxiv" }, - { name = "beautifulsoup4" }, - { name = "colorama" }, - { name = "htmldocx" }, - { name = "json-repair" }, - { name = "json5" }, - { name = "langchain" }, - { name = "langchain-community" }, - { name = "langchain-openai" }, - { name = "loguru" }, - { name = "lxml-html-clean" }, - { name = "markdown" }, - { name = "md2pdf" }, - { name = "mistune" }, - { name = "pydantic" }, - { name = "pymupdf" }, - { name = "python-docx" }, - { name = "python-dotenv" }, - { name = "python-multipart" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tiktoken" }, - { name = "unstructured" }, - { name = "websockets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/d2/3a1acfd8b71ead6ab9a6833bcd1725e468e65817e7b04c233cd2d5e0a629/gpt_researcher-0.12.12.tar.gz", hash = "sha256:e3fa6faae4a3dc7e4280521eceb94a081a0aae277eb1ded25152579f65195844", size = 123669 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/09/0b/80ade14566946ca2253d1718746b63059f5bb1a2e489804c87e24332082d/gpt_researcher-0.12.12-py3-none-any.whl", hash = "sha256:3db51994406844d8acb28cb2a4897b9e9aaa34f0127edd7f8ded103fe2e544d4", size = 161671 }, -] - [[package]] name = "greenlet" version = "3.1.1" @@ -1109,19 +967,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", size = 112173 }, ] -[[package]] -name = "htmldocx" -version = "0.0.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "python-docx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8b/61/91a6b70ee576a4b07310d81efd4c688fe2e6f63ea42ec95b8f1d436b887e/htmldocx-0.0.6.tar.gz", hash = "sha256:b4bcec895f86d7a50ffc7133ca24d85c24f3614db2b37d33a30d9d04654a5486", size = 9418 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/da/c70fc2ce54c1d1ce7c16f9656589273a6c94cbbc8867b3a512618d977309/htmldocx-0.0.6-py3-none-any.whl", hash = "sha256:adf5e95ad8ba8121e606cf138c614de13327a1192a5782acdb4a0abdc23db1b7", size = 9490 }, -] - [[package]] name = "httpcore" version = "1.0.7" @@ -1300,24 +1145,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/91/29/df4b9b42f2be0b623cbd5e2140cafcaa2bef0759a00b7b70104dcfe2fb51/joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6", size = 301817 }, ] -[[package]] -name = "json-repair" -version = "0.39.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/60/6d1599bc01070d9fe3840d245ae80fd24b981c732d962842825ce7a9fde6/json_repair-0.39.1.tar.gz", hash = "sha256:e90a489f247e1a8fc86612a5c719872a3dbf9cbaffd6d55f238ec571a77740fa", size = 30040 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/b9/2e445481555422b907dab468b53574bc1e995099ca1a1201d0d876ca05e9/json_repair-0.39.1-py3-none-any.whl", hash = "sha256:3001409a2f319249f13e13d6c622117a5b70ea7e0c6f43864a0233cdffc3a599", size = 20686 }, -] - -[[package]] -name = "json5" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/85/3d/bbe62f3d0c05a689c711cff57b2e3ac3d3e526380adb7c781989f075115c/json5-0.10.0.tar.gz", hash = "sha256:e66941c8f0a02026943c52c2eb34ebeb2a6f819a0be05920a6f5243cd30fd559", size = 48202 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/42/797895b952b682c3dafe23b1834507ee7f02f4d6299b65aaa61425763278/json5-0.10.0-py3-none-any.whl", hash = "sha256:19b23410220a7271e8377f81ba8aacba2fdd56947fbb137ee5977cbe1f5e8dfa", size = 34049 }, -] - [[package]] name = "jsonpatch" version = "1.33" @@ -1479,20 +1306,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/0e/ddf9f5dc46b178df5c101666bb3bc7fc526d68cd81cdd60cbe1b6b438b30/langchain_core-0.3.43-py3-none-any.whl", hash = "sha256:caa6bc1f4c6ab71d3c2e400f8b62e1cd6dc5ac2c37e03f12f3e2c60befd5b273", size = 415421 }, ] -[[package]] -name = "langchain-openai" -version = "0.3.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "langchain-core" }, - { name = "openai" }, - { name = "tiktoken" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2e/04/ae071af0b04d1c3a8040498714091afd21149f6f8ae1dbab584317d9dfd7/langchain_openai-0.3.8.tar.gz", hash = "sha256:4d73727eda8102d1d07a2ca036278fccab0bb5e0abf353cec9c3973eb72550ec", size = 256898 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/43/9c6a1101bcd751d52a3328a06956f85122f9aaa31da1b15a8e0f99a70317/langchain_openai-0.3.8-py3-none-any.whl", hash = "sha256:9004dc8ef853aece0d8f0feca7753dc97f710fa3e53874c8db66466520436dbb", size = 55446 }, -] - [[package]] name = "langchain-text-splitters" version = "0.3.6" @@ -1622,19 +1435,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/c2/1b6c502909b7af9054736af61e27558a3341e8c1ba28e7f82473e6dd936f/litellm-1.61.4-py3-none-any.whl", hash = "sha256:e87e0d397a191795b4217f9299fc9b21eaacaab91409695f0a4780cceccda6e1", size = 6814517 }, ] -[[package]] -name = "loguru" -version = "0.7.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "win32-setctime", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595 }, -] - [[package]] name = "lxml" version = "5.3.1" @@ -1677,18 +1477,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/83/8c54533b3576f4391eebea88454738978669a6cad0d8e23266224007939d/lxml-5.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:91fb6a43d72b4f8863d21f347a9163eecbf36e76e2f51068d59cd004c506f332", size = 3814484 }, ] -[[package]] -name = "lxml-html-clean" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lxml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/81/f2/fe319e3c5cb505a361b95d1e0d0d793fe28d4dcc2fc39d3cae9324dc4233/lxml_html_clean-0.4.1.tar.gz", hash = "sha256:40c838bbcf1fc72ba4ce811fbb3135913017b27820d7c16e8bc412ae1d8bc00b", size = 21378 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/ba/2af7a60b45bf21375e111c1e2d5d721108d06c80e3d9a3cc1d767afe1731/lxml_html_clean-0.4.1-py3-none-any.whl", hash = "sha256:b704f2757e61d793b1c08bf5ad69e4c0b68d6696f4c3c1429982caf90050bcaf", size = 14114 }, -] - [[package]] name = "makefun" version = "1.15.6" @@ -1731,15 +1519,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528 }, ] -[[package]] -name = "markdown2" -version = "2.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/44/52/d7dcc6284d59edb8301b8400435fbb4926a9b0f13a12b5cbaf3a4a54bb7b/markdown2-2.5.3.tar.gz", hash = "sha256:4d502953a4633408b0ab3ec503c5d6984d1b14307e32b325ec7d16ea57524895", size = 141676 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/37/0a13c83ccf5365b8e08ea572dfbc04b8cb87cadd359b2451a567f5248878/markdown2-2.5.3-py3-none-any.whl", hash = "sha256:a8ebb7e84b8519c37bf7382b3db600f1798a22c245bfd754a1f87ca8d7ea63b3", size = 48550 }, -] - [[package]] name = "markdownify" version = "0.14.1" @@ -1840,17 +1619,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/c2/0d5aae823bdcc42cc99327ecdd4d28585e15ccd5218c453b7bcd827f3421/matplotlib-3.10.1-cp313-cp313t-win_amd64.whl", hash = "sha256:bc411ebd5889a78dabbc457b3fa153203e22248bfa6eedc6797be5df0164dbf9", size = 8134832 }, ] -[[package]] -name = "md2pdf" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "docopt" }, - { name = "markdown2" }, - { name = "weasyprint" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/de/b0/adbef5356f97a6d33c7811805b06e3774c7a58ea70dc28039ae4ad1ba1be/md2pdf-1.0.1.tar.gz", hash = "sha256:3d5aab77dcd5b6f5827b193819ab1a8c1cec506ce5f6c777c3411b703352cd98", size = 6377 } - [[package]] name = "mdurl" version = "0.1.2" @@ -1860,15 +1628,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 }, ] -[[package]] -name = "mistune" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/80/f7/f6d06304c61c2a73213c0a4815280f70d985429cda26272f490e42119c1a/mistune-3.1.2.tar.gz", hash = "sha256:733bf018ba007e8b5f2d3a9eb624034f6ee26c4ea769a98ec533ee111d504dff", size = 94613 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/92/30b4e54c4d7c48c06db61595cffbbf4f19588ea177896f9b78f0fbe021fd/mistune-3.1.2-py3-none-any.whl", hash = "sha256:4b47731332315cdca99e0ded46fc0004001c1299ff773dfb48fbe1fd226de319", size = 53696 }, -] - [[package]] name = "model2vec" version = "0.4.0" @@ -2692,15 +2451,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/46/93416fdae86d40879714f72956ac14df9c7b76f7d41a4d68aa9f71a0028b/pydantic_settings-2.7.1-py3-none-any.whl", hash = "sha256:590be9e6e24d06db33a4262829edef682500ef008565a969c73d39d5f8bfb3fd", size = 29718 }, ] -[[package]] -name = "pydyf" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2e/c2/97fc6ce4ce0045080dc99446def812081b57750ed8aa67bfdfafa4561fe5/pydyf-0.11.0.tar.gz", hash = "sha256:394dddf619cca9d0c55715e3c55ea121a9bf9cbc780cdc1201a2427917b86b64", size = 17769 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/ac/d5db977deaf28c6ecbc61bbca269eb3e8f0b3a1f55c8549e5333e606e005/pydyf-0.11.0-py3-none-any.whl", hash = "sha256:0aaf9e2ebbe786ec7a78ec3fbffa4cdcecde53fd6f563221d53c6bc1328848a3", size = 8104 }, -] - [[package]] name = "pyee" version = "12.1.1" @@ -2736,21 +2486,6 @@ crypto = [ { name = "cryptography" }, ] -[[package]] -name = "pymupdf" -version = "1.25.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/47/b61c1c44b87cbdaeecdec3f43ce524ed6b3c72172bc6184eb82c94fbc43d/pymupdf-1.25.3.tar.gz", hash = "sha256:b640187c64c5ac5d97505a92e836da299da79c2f689f3f94a67a37a493492193", size = 67259841 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/61/9b/98ef4b98309e9db3baa9fe572f0e61b6130bb9852d13189970f35b703499/pymupdf-1.25.3-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:96878e1b748f9c2011aecb2028c5f96b5a347a9a91169130ad0133053d97915e", size = 19343576 }, - { url = "https://files.pythonhosted.org/packages/14/62/4e12126db174c8cfbf692281cda971cc4046c5f5226032c2cfaa6f83e08d/pymupdf-1.25.3-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:6ef753005b72ebfd23470f72f7e30f61e21b0b5e748045ec5b8f89e6e3068d62", size = 18580114 }, - { url = "https://files.pythonhosted.org/packages/ec/c5/cf7ecf005e4f8ba3664d6aaa0613adeba4c2ab524832c452c69857e7184f/pymupdf-1.25.3-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cbff443d899f37b17f1e67563cc03673d50b4bf33ccc237e73d34f18f3a07ccf", size = 19442580 }, - { url = "https://files.pythonhosted.org/packages/52/de/bd1418e31f73d37b8381cd5deacfd681e6be702b8890e123e83724569ee1/pymupdf-1.25.3-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:46d90c4f9e62d1856e8db4b9f04a202ff4a7f086a816af73abdc86adb7f5e25a", size = 19999825 }, - { url = "https://files.pythonhosted.org/packages/42/ee/3c449b0de061440ba1ac984aa845315e9e2dca0ff2003c5adfc6febff203/pymupdf-1.25.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5de51efdbe4d486b6c1111c84e8a231cbfb426f3d6ff31ab530ad70e6f39756", size = 21123157 }, - { url = "https://files.pythonhosted.org/packages/83/53/71faaaf91c56f2883b13f3dd849bf2697f012eb35eb7b952d62734cff41f/pymupdf-1.25.3-cp39-abi3-win32.whl", hash = "sha256:bca72e6089f985d800596e22973f79cc08af6cbff1d93e5bda9248326a03857c", size = 15094211 }, - { url = "https://files.pythonhosted.org/packages/09/e0/d72e88a1d5e23aa381fd463057dc3d0fb29090e1e7308a870c334716579c/pymupdf-1.25.3-cp39-abi3-win_amd64.whl", hash = "sha256:4fb357438c9129fbf939b5af85323434df64e36759c399c376b62ad6da95498c", size = 16542949 }, -] - [[package]] name = "pypandoc" version = "1.15" @@ -2798,15 +2533,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/6b/2706497c86e8d69fb76afe5ea857fe1794621aa0f3b1d863feb953fe0f22/pypdfium2-4.30.1-py3-none-win_arm64.whl", hash = "sha256:c2b6d63f6d425d9416c08d2511822b54b8e3ac38e639fc41164b1d75584b3a8c", size = 2814810 }, ] -[[package]] -name = "pyphen" -version = "0.17.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/56/e4d7e1bd70d997713649c5ce530b2d15a5fc2245a74ca820fc2d51d89d4d/pyphen-0.17.2.tar.gz", hash = "sha256:f60647a9c9b30ec6c59910097af82bc5dd2d36576b918e44148d8b07ef3b4aa3", size = 2079470 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/1f/c2142d2edf833a90728e5cdeb10bdbdc094dde8dbac078cee0cf33f5e11b/pyphen-0.17.2-py3-none-any.whl", hash = "sha256:3a07fb017cb2341e1d9ff31b8634efb1ae4dc4b130468c7c39dd3d32e7c3affd", size = 2079358 }, -] - [[package]] name = "pyreadline3" version = "3.5.4" @@ -3255,12 +2981,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/8a/b9dc7678803429e4a3bc9ba462fa3dd9066824d3c607490235c6a796be5a/setuptools-75.8.0-py3-none-any.whl", hash = "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3", size = 1228782 }, ] -[[package]] -name = "sgmllib3k" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9e/bd/3704a8c3e0942d711c1299ebf7b9091930adae6675d7c8f476a7ce48653c/sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9", size = 5750 } - [[package]] name = "six" version = "1.17.0" @@ -3345,7 +3065,7 @@ wheels = [ [[package]] name = "surf-new-backend" -version = "0.1.0" +version = "0.0.6" source = { virtual = "." } dependencies = [ { name = "alembic" }, @@ -3355,7 +3075,6 @@ dependencies = [ { name = "fastapi-users", extra = ["oauth", "sqlalchemy"] }, { name = "firecrawl-py" }, { name = "github3-py" }, - { name = "gpt-researcher" }, { name = "langchain-community" }, { name = "langchain-unstructured" }, { name = "langgraph" }, @@ -3384,7 +3103,6 @@ requires-dist = [ { name = "fastapi-users", extras = ["oauth", "sqlalchemy"], specifier = ">=14.0.1" }, { name = "firecrawl-py", specifier = ">=1.12.0" }, { name = "github3-py", specifier = "==4.0.1" }, - { name = "gpt-researcher", specifier = ">=0.12.12" }, { name = "langchain-community", specifier = ">=0.3.17" }, { name = "langchain-unstructured", specifier = ">=0.1.6" }, { name = "langgraph", specifier = ">=0.3.29" }, @@ -3488,30 +3206,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/d0/179abca8b984b3deefd996f362b612c39da73b60f685921e6cd58b6125b4/timm-1.0.15-py3-none-any.whl", hash = "sha256:5a3dc460c24e322ecc7fd1f3e3eb112423ddee320cb059cc1956fbc9731748ef", size = 2361373 }, ] -[[package]] -name = "tinycss2" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "webencodings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/fd/7a5ee21fd08ff70d3d33a5781c255cbe779659bd03278feb98b19ee550f4/tinycss2-1.4.0.tar.gz", hash = "sha256:10c0972f6fc0fbee87c3edb76549357415e94548c1ae10ebccdea16fb404a9b7", size = 87085 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e6/34/ebdc18bae6aa14fbee1a08b63c015c72b64868ff7dae68808ab500c492e2/tinycss2-1.4.0-py3-none-any.whl", hash = "sha256:3a49cf47b7675da0b15d0c6e1df8df4ebd96e9394bb905a5775adb0d884c5289", size = 26610 }, -] - -[[package]] -name = "tinyhtml5" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "webencodings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fd/03/6111ed99e9bf7dfa1c30baeef0e0fb7e0bd387bd07f8e5b270776fe1de3f/tinyhtml5-2.0.0.tar.gz", hash = "sha256:086f998833da24c300c414d9fe81d9b368fd04cb9d2596a008421cbc705fcfcc", size = 179507 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/de/27c57899297163a4a84104d5cec0af3b1ac5faf62f44667e506373c6b8ce/tinyhtml5-2.0.0-py3-none-any.whl", hash = "sha256:13683277c5b176d070f82d099d977194b7a1e26815b016114f581a74bbfbf47e", size = 39793 }, -] - [[package]] name = "tokenizers" version = "0.21.0" @@ -3891,25 +3585,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/e5/96b8e55271685ddbadc50ce8bc53aa2dff278fb7ac4c2e473df890def2dc/watchfiles-1.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:d6097538b0ae5c1b88c3b55afa245a66793a8fec7ada6755322e465fb1a0e8cc", size = 285216 }, ] -[[package]] -name = "weasyprint" -version = "64.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, - { name = "cssselect2" }, - { name = "fonttools", extra = ["woff"] }, - { name = "pillow" }, - { name = "pydyf" }, - { name = "pyphen" }, - { name = "tinycss2" }, - { name = "tinyhtml5" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6c/a0/f6b3ef688e747488b17b3b39d27fe7438d3ec88d1b79d5524485a5458020/weasyprint-64.1.tar.gz", hash = "sha256:28b02f2c6409bafce1b1220d9d76a7345875bd3bd08c4f6dfbf510bb92a94757", size = 498647 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/95/bf333fbbaf73c1c211b6b801b9ac2563db8e2225f69902d1ba8b25c70e9c/weasyprint-64.1-py3-none-any.whl", hash = "sha256:f7c88ea8ce0ce0c527cbb9c802689e035fae50016d7efc5dfdaba4b75abf68f4", size = 302025 }, -] - [[package]] name = "webencodings" version = "0.5.1" @@ -3950,15 +3625,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/c8/d529f8a32ce40d98309f4470780631e971a5a842b60aec864833b3615786/websockets-14.2-py3-none-any.whl", hash = "sha256:7a6ceec4ea84469f15cf15807a747e9efe57e369c384fa86e022b3bea679b79b", size = 157416 }, ] -[[package]] -name = "win32-setctime" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083 }, -] - [[package]] name = "wrapt" version = "1.17.2" @@ -4125,34 +3791,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630 }, ] -[[package]] -name = "zopfli" -version = "0.2.3.post1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5e/7c/a8f6696e694709e2abcbccd27d05ef761e9b6efae217e11d977471555b62/zopfli-0.2.3.post1.tar.gz", hash = "sha256:96484dc0f48be1c5d7ae9f38ed1ce41e3675fd506b27c11a6607f14b49101e99", size = 175629 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/ce/b6441cc01881d06e0b5883f32c44e7cc9772e0d04e3e59277f59f80b9a19/zopfli-0.2.3.post1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3f0197b6aa6eb3086ae9e66d6dd86c4d502b6c68b0ec490496348ae8c05ecaef", size = 295489 }, - { url = "https://files.pythonhosted.org/packages/93/f0/24dd708f00ae0a925bc5c9edae858641c80f6a81a516810dc4d21688a930/zopfli-0.2.3.post1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fcfc0dc2761e4fcc15ad5d273b4d58c2e8e059d3214a7390d4d3c8e2aee644e", size = 163010 }, - { url = "https://files.pythonhosted.org/packages/65/57/0378eeeb5e3e1e83b1b0958616b2bf954f102ba5b0755b9747dafbd8cb72/zopfli-0.2.3.post1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cac2b37ab21c2b36a10b685b1893ebd6b0f83ae26004838ac817680881576567", size = 823649 }, - { url = "https://files.pythonhosted.org/packages/ab/8a/3ab8a616d4655acf5cf63c40ca84e434289d7d95518a1a42d28b4a7228f8/zopfli-0.2.3.post1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d5ab297d660b75c159190ce6d73035502310e40fd35170aed7d1a1aea7ddd65", size = 826557 }, - { url = "https://files.pythonhosted.org/packages/ed/4d/7f6820af119c4fec6efaf007bffee7bc9052f695853a711a951be7afd26b/zopfli-0.2.3.post1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ba214f4f45bec195ee8559651154d3ac2932470b9d91c5715fc29c013349f8c", size = 851127 }, - { url = "https://files.pythonhosted.org/packages/e1/db/1ef5353ab06f9f2fb0c25ed0cddf1418fe275cc2ee548bc4a29340c44fe1/zopfli-0.2.3.post1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1e0ed5d84ffa2d677cc9582fc01e61dab2e7ef8b8996e055f0a76167b1b94df", size = 1754183 }, - { url = "https://files.pythonhosted.org/packages/39/03/44f8f39950354d330fa798e4bab1ac8e38ec787d3fde25d5b9c7770065a2/zopfli-0.2.3.post1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:bfa1eb759e07d8b7aa7a310a2bc535e127ee70addf90dc8d4b946b593c3e51a8", size = 1905945 }, - { url = "https://files.pythonhosted.org/packages/74/7b/94b920c33cc64255f59e3cfc77c829b5c6e60805d189baeada728854a342/zopfli-0.2.3.post1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cd2c002f160502608dcc822ed2441a0f4509c52e86fcfd1a09e937278ed1ca14", size = 1835885 }, - { url = "https://files.pythonhosted.org/packages/ad/89/c869ac844351e285a6165e2da79b715b0619a122e3160d183805adf8ab45/zopfli-0.2.3.post1-cp312-cp312-win32.whl", hash = "sha256:7be5cc6732eb7b4df17305d8a7b293223f934a31783a874a01164703bc1be6cd", size = 82743 }, - { url = "https://files.pythonhosted.org/packages/29/e6/c98912fd3a589d8a7316c408fd91519f72c237805c4400b753e3942fda0b/zopfli-0.2.3.post1-cp312-cp312-win_amd64.whl", hash = "sha256:4e50ffac74842c1c1018b9b73875a0d0a877c066ab06bf7cccbaa84af97e754f", size = 99403 }, - { url = "https://files.pythonhosted.org/packages/2b/24/0e552e2efce9a20625b56e9609d1e33c2966be33fc008681121ec267daec/zopfli-0.2.3.post1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ecb7572df5372abce8073df078207d9d1749f20b8b136089916a4a0868d56051", size = 295485 }, - { url = "https://files.pythonhosted.org/packages/08/83/b2564369fb98797a617fe2796097b1d719a4937234375757ad2a3febc04b/zopfli-0.2.3.post1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a1cf720896d2ce998bc8e051d4b4ce0d8bec007aab6243102e8e1d22a0b2fb3f", size = 163000 }, - { url = "https://files.pythonhosted.org/packages/3c/55/81d419739c2aab35e19b58bce5498dcb58e6446e5eb69f2d3c748b1c9151/zopfli-0.2.3.post1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5aad740b4d4fcbaaae4887823925166ffd062db3b248b3f432198fc287381d1a", size = 823699 }, - { url = "https://files.pythonhosted.org/packages/9e/91/89f07c8ea3c9bc64099b3461627b07a8384302235ee0f357eaa86f98f509/zopfli-0.2.3.post1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6617fb10f9e4393b331941861d73afb119cd847e88e4974bdbe8068ceef3f73f", size = 826612 }, - { url = "https://files.pythonhosted.org/packages/41/31/46670fc0c7805d42bc89702440fa9b73491d68abbc39e28d687180755178/zopfli-0.2.3.post1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a53b18797cdef27e019db595d66c4b077325afe2fd62145953275f53d84ce40c", size = 851148 }, - { url = "https://files.pythonhosted.org/packages/22/00/71ad39277bbb88f9fd20fb786bd3ff2ea4025c53b31652a0da796fb546cd/zopfli-0.2.3.post1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b78008a69300d929ca2efeffec951b64a312e9a811e265ea4a907ab546d79fa6", size = 1754215 }, - { url = "https://files.pythonhosted.org/packages/d0/4e/e542c508d20c3dfbef1b90fcf726f824f505e725747f777b0b7b7d1deb95/zopfli-0.2.3.post1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0aa5f90d6298bda02a95bc8dc8c3c19004d5a4e44bda00b67ca7431d857b4b54", size = 1905988 }, - { url = "https://files.pythonhosted.org/packages/ba/a5/817ac1ecc888723e91dc172e8c6eeab9f48a1e52285803b965084e11bbd5/zopfli-0.2.3.post1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2768c877f76c8a0e7519b1c86c93757f3c01492ddde55751e9988afb7eff64e1", size = 1835907 }, - { url = "https://files.pythonhosted.org/packages/cd/35/2525f90c972d8aafc39784a8c00244eeee8e8221b26cbc576748ee9dc1cd/zopfli-0.2.3.post1-cp313-cp313-win32.whl", hash = "sha256:71390dbd3fbf6ebea9a5d85ffed8c26ee1453ee09248e9b88486e30e0397b775", size = 82742 }, - { url = "https://files.pythonhosted.org/packages/2f/c6/49b27570923956d52d37363e8f5df3a31a61bd7719bb8718527a9df3ae5f/zopfli-0.2.3.post1-cp313-cp313-win_amd64.whl", hash = "sha256:a86eb88e06bd87e1fff31dac878965c26b0c26db59ddcf78bb0379a954b120de", size = 99408 }, -] - [[package]] name = "zstandard" version = "0.23.0" diff --git a/surfsense_browser_extension/package.json b/surfsense_browser_extension/package.json index 25600ce12..3d2d04002 100644 --- a/surfsense_browser_extension/package.json +++ b/surfsense_browser_extension/package.json @@ -1,7 +1,7 @@ { - "name": "surfsense", - "displayName": "Surfsense", - "version": "0.0.1", + "name": "surfsense_browser_extension", + "displayName": "Surfsense Browser Extension", + "version": "0.0.6", "description": "Extension to collect Browsing History for SurfSense.", "author": "https://github.com/MODSetter", "scripts": { diff --git a/surfsense_web/package.json b/surfsense_web/package.json index bffae6cb6..e2047644a 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -1,7 +1,8 @@ { - "name": "surf_new_frontend", - "version": "0.1.0", + "name": "surfsense_web", + "version": "0.0.6", "private": true, + "description": "SurfSense Frontend", "scripts": { "dev": "next dev --turbopack", "build": "next build", From 0436d2ab64f084dc13264bc97a32e2b6f67d9c99 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 22 Apr 2025 02:24:13 -0700 Subject: [PATCH 29/31] feat: Added FumaDocs --- surfsense_web/.source/index.ts | 5 + surfsense_web/.source/source.config.mjs | 8 + surfsense_web/app/api/search/route.ts | 4 + surfsense_web/app/docs/[[...slug]]/page.tsx | 46 + surfsense_web/app/docs/layout.tsx | 12 + surfsense_web/app/globals.css | 4 +- surfsense_web/app/layout.config.tsx | 7 + surfsense_web/app/layout.tsx | 7 +- surfsense_web/content/docs/index.mdx | 7 + surfsense_web/lib/source.ts | 7 + surfsense_web/mdx-components.tsx | 9 + surfsense_web/next.config.ts | 6 +- surfsense_web/package.json | 10 +- surfsense_web/pnpm-lock.yaml | 1788 ++++++++++++++++++- surfsense_web/source.config.ts | 5 + surfsense_web/tsconfig.json | 2 +- 16 files changed, 1919 insertions(+), 8 deletions(-) create mode 100644 surfsense_web/.source/index.ts create mode 100644 surfsense_web/.source/source.config.mjs create mode 100644 surfsense_web/app/api/search/route.ts create mode 100644 surfsense_web/app/docs/[[...slug]]/page.tsx create mode 100644 surfsense_web/app/docs/layout.tsx create mode 100644 surfsense_web/app/layout.config.tsx create mode 100644 surfsense_web/content/docs/index.mdx create mode 100644 surfsense_web/lib/source.ts create mode 100644 surfsense_web/mdx-components.tsx create mode 100644 surfsense_web/source.config.ts diff --git a/surfsense_web/.source/index.ts b/surfsense_web/.source/index.ts new file mode 100644 index 000000000..0002966e0 --- /dev/null +++ b/surfsense_web/.source/index.ts @@ -0,0 +1,5 @@ +// @ts-nocheck -- skip type checking +import * as docs_0 from "../content/docs/index.mdx?collection=docs&hash=1745310068376" +import { _runtime } from "fumadocs-mdx" +import * as _source from "../source.config" +export const docs = _runtime.docs([{ info: {"path":"index.mdx","absolutePath":"C:/Users/91882/Documents/SurfSense/surfsense_web/content/docs/index.mdx"}, data: docs_0 }], []) \ No newline at end of file diff --git a/surfsense_web/.source/source.config.mjs b/surfsense_web/.source/source.config.mjs new file mode 100644 index 000000000..ae67114fd --- /dev/null +++ b/surfsense_web/.source/source.config.mjs @@ -0,0 +1,8 @@ +// source.config.ts +import { defineDocs } from "fumadocs-mdx/config"; +var docs = defineDocs({ + dir: "content/docs" +}); +export { + docs +}; diff --git a/surfsense_web/app/api/search/route.ts b/surfsense_web/app/api/search/route.ts new file mode 100644 index 000000000..01401b7f5 --- /dev/null +++ b/surfsense_web/app/api/search/route.ts @@ -0,0 +1,4 @@ +import { source } from '@/lib/source'; +import { createFromSource } from 'fumadocs-core/search/server'; + +export const { GET } = createFromSource(source); \ No newline at end of file diff --git a/surfsense_web/app/docs/[[...slug]]/page.tsx b/surfsense_web/app/docs/[[...slug]]/page.tsx new file mode 100644 index 000000000..6c8574d87 --- /dev/null +++ b/surfsense_web/app/docs/[[...slug]]/page.tsx @@ -0,0 +1,46 @@ +import { source } from '@/lib/source'; +import { + DocsBody, + DocsDescription, + DocsPage, + DocsTitle, +} from 'fumadocs-ui/page'; +import { notFound } from 'next/navigation'; +import { getMDXComponents } from '@/mdx-components'; + +export default async function Page(props: { + params: Promise<{ slug?: string[] }>; +}) { + const params = await props.params; + const page = source.getPage(params.slug); + if (!page) notFound(); + + const MDX = page.data.body; + + return ( + + {page.data.title} + {page.data.description} + + + + + ); +} + +export async function generateStaticParams() { + return source.generateParams(); +} + +export async function generateMetadata(props: { + params: Promise<{ slug?: string[] }>; +}) { + const params = await props.params; + const page = source.getPage(params.slug); + if (!page) notFound(); + + return { + title: page.data.title, + description: page.data.description, + }; +} \ No newline at end of file diff --git a/surfsense_web/app/docs/layout.tsx b/surfsense_web/app/docs/layout.tsx new file mode 100644 index 000000000..e818c1f68 --- /dev/null +++ b/surfsense_web/app/docs/layout.tsx @@ -0,0 +1,12 @@ +import { source } from '@/lib/source'; +import { DocsLayout } from 'fumadocs-ui/layouts/docs'; +import type { ReactNode } from 'react'; +import { baseOptions } from '@/app/layout.config'; + +export default function Layout({ children }: { children: ReactNode }) { + return ( + + {children} + + ); +} \ No newline at end of file diff --git a/surfsense_web/app/globals.css b/surfsense_web/app/globals.css index 8fdefeca5..98e4411fb 100644 --- a/surfsense_web/app/globals.css +++ b/surfsense_web/app/globals.css @@ -1,4 +1,6 @@ -@import "tailwindcss"; +@import 'tailwindcss'; +@import 'fumadocs-ui/css/neutral.css'; +@import 'fumadocs-ui/css/preset.css'; @plugin "tailwindcss-animate"; diff --git a/surfsense_web/app/layout.config.tsx b/surfsense_web/app/layout.config.tsx new file mode 100644 index 000000000..ef0500157 --- /dev/null +++ b/surfsense_web/app/layout.config.tsx @@ -0,0 +1,7 @@ +import { BaseLayoutProps } from 'fumadocs-ui/layouts/shared'; + +export const baseOptions: BaseLayoutProps = { + nav: { + title: 'SurfSense Documentation', + }, +}; \ No newline at end of file diff --git a/surfsense_web/app/layout.tsx b/surfsense_web/app/layout.tsx index be0b18079..6b60891a4 100644 --- a/surfsense_web/app/layout.tsx +++ b/surfsense_web/app/layout.tsx @@ -5,6 +5,7 @@ import { Roboto } from "next/font/google"; import { Toaster } from "@/components/ui/sonner"; import { ThemeProvider } from "@/components/theme/theme-provider"; +import { RootProvider } from 'fumadocs-ui/provider'; const roboto = Roboto({ subsets: ["latin"], @@ -64,8 +65,10 @@ export default async function RootLayout({ disableTransitionOnChange defaultTheme="light" > - {children} - + + {children} + + diff --git a/surfsense_web/content/docs/index.mdx b/surfsense_web/content/docs/index.mdx new file mode 100644 index 000000000..e99f597dc --- /dev/null +++ b/surfsense_web/content/docs/index.mdx @@ -0,0 +1,7 @@ +--- +title: Welcome Docs +--- + +## Introduction + +I love Docs. \ No newline at end of file diff --git a/surfsense_web/lib/source.ts b/surfsense_web/lib/source.ts new file mode 100644 index 000000000..d80ab8edd --- /dev/null +++ b/surfsense_web/lib/source.ts @@ -0,0 +1,7 @@ +import { docs } from '@/.source'; +import { loader } from 'fumadocs-core/source'; + +export const source = loader({ + baseUrl: '/docs', + source: docs.toFumadocsSource(), +}); \ No newline at end of file diff --git a/surfsense_web/mdx-components.tsx b/surfsense_web/mdx-components.tsx new file mode 100644 index 000000000..f190f3ced --- /dev/null +++ b/surfsense_web/mdx-components.tsx @@ -0,0 +1,9 @@ +import defaultMdxComponents from 'fumadocs-ui/mdx'; +import type { MDXComponents } from 'mdx/types'; + +export function getMDXComponents(components?: MDXComponents): MDXComponents { + return { + ...defaultMdxComponents, + ...components, + }; +} \ No newline at end of file diff --git a/surfsense_web/next.config.ts b/surfsense_web/next.config.ts index 5cb563cd9..9ba7df975 100644 --- a/surfsense_web/next.config.ts +++ b/surfsense_web/next.config.ts @@ -1,4 +1,5 @@ import type { NextConfig } from "next"; +import { createMDX } from 'fumadocs-mdx/next'; const nextConfig: NextConfig = { typescript: { @@ -9,4 +10,7 @@ const nextConfig: NextConfig = { }, }; -export default nextConfig; +// Wrap the config with createMDX +const withMDX = createMDX({}); + +export default withMDX(nextConfig); diff --git a/surfsense_web/package.json b/surfsense_web/package.json index e2047644a..07fcb5549 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -4,13 +4,15 @@ "private": true, "description": "SurfSense Frontend", "scripts": { - "dev": "next dev --turbopack", + "dev": "next dev", + "dev:turbopack": "next dev --turbopack", "build": "next build", "start": "next start", "lint": "next lint", "debug": "cross-env NODE_OPTIONS=--inspect next dev --turbopack", "debug:browser": "cross-env NODE_OPTIONS=--inspect next dev --turbopack", - "debug:server": "cross-env NODE_OPTIONS=--inspect=0.0.0.0:9229 next dev --turbopack" + "debug:server": "cross-env NODE_OPTIONS=--inspect=0.0.0.0:9229 next dev --turbopack", + "postinstall": "fumadocs-mdx" }, "dependencies": { "@ai-sdk/react": "^1.1.21", @@ -31,12 +33,16 @@ "@radix-ui/react-tooltip": "^1.1.8", "@tabler/icons-react": "^3.30.0", "@tanstack/react-table": "^8.21.2", + "@types/mdx": "^2.0.13", "ai": "^4.1.54", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "date-fns": "^4.1.0", "emblor": "^1.4.7", "framer-motion": "^12.4.7", + "fumadocs-core": "^15.2.9", + "fumadocs-mdx": "^11.6.1", + "fumadocs-ui": "^15.2.9", "geist": "^1.3.1", "lucide-react": "^0.477.0", "next": "15.2.3", diff --git a/surfsense_web/pnpm-lock.yaml b/surfsense_web/pnpm-lock.yaml index 081a1b9dd..a8eb3c82a 100644 --- a/surfsense_web/pnpm-lock.yaml +++ b/surfsense_web/pnpm-lock.yaml @@ -62,6 +62,9 @@ importers: '@tanstack/react-table': specifier: ^8.21.2 version: 8.21.2(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@types/mdx': + specifier: ^2.0.13 + version: 2.0.13 ai: specifier: ^4.1.54 version: 4.1.54(react@19.0.0)(zod@3.24.2) @@ -80,6 +83,15 @@ importers: framer-motion: specifier: ^12.4.7 version: 12.4.7(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + fumadocs-core: + specifier: ^15.2.9 + version: 15.2.9(@types/react@19.0.10)(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0))(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + fumadocs-mdx: + specifier: ^11.6.1 + version: 11.6.1(acorn@8.14.0)(fumadocs-core@15.2.9(@types/react@19.0.10)(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0))(react-dom@19.0.0(react@19.0.0))(react@19.0.0))(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)) + fumadocs-ui: + specifier: ^15.2.9 + version: 15.2.9(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0))(react-dom@19.0.0(react@19.0.0))(react@19.0.0)(tailwindcss@4.0.9) geist: specifier: ^1.3.1 version: 1.3.1(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)) @@ -216,138 +228,288 @@ packages: '@emnapi/runtime@1.3.1': resolution: {integrity: sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==} + '@esbuild/aix-ppc64@0.25.2': + resolution: {integrity: sha512-wCIboOL2yXZym2cgm6mlA742s9QeJ8DjGVaL39dLN4rRwrOgOyYSnOaFPhKZGLb2ngj4EyfAFjsNJwPXZvseag==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/android-arm64@0.17.19': resolution: {integrity: sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==} engines: {node: '>=12'} cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.25.2': + resolution: {integrity: sha512-5ZAX5xOmTligeBaeNEPnPaeEuah53Id2tX4c2CVP3JaROTH+j4fnfHCkr1PjXMd78hMst+TlkfKcW/DlTq0i4w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm@0.17.19': resolution: {integrity: sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A==} engines: {node: '>=12'} cpu: [arm] os: [android] + '@esbuild/android-arm@0.25.2': + resolution: {integrity: sha512-NQhH7jFstVY5x8CKbcfa166GoV0EFkaPkCKBQkdPJFvo5u+nGXLEH/ooniLb3QI8Fk58YAx7nsPLozUWfCBOJA==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-x64@0.17.19': resolution: {integrity: sha512-uUTTc4xGNDT7YSArp/zbtmbhO0uEEK9/ETW29Wk1thYUJBz3IVnvgEiEwEa9IeLyvnpKrWK64Utw2bgUmDveww==} engines: {node: '>=12'} cpu: [x64] os: [android] + '@esbuild/android-x64@0.25.2': + resolution: {integrity: sha512-Ffcx+nnma8Sge4jzddPHCZVRvIfQ0kMsUsCMcJRHkGJ1cDmhe4SsrYIjLUKn1xpHZybmOqCWwB0zQvsjdEHtkg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/darwin-arm64@0.17.19': resolution: {integrity: sha512-80wEoCfF/hFKM6WE1FyBHc9SfUblloAWx6FJkFWTWiCoht9Mc0ARGEM47e67W9rI09YoUxJL68WHfDRYEAvOhg==} engines: {node: '>=12'} cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.25.2': + resolution: {integrity: sha512-MpM6LUVTXAzOvN4KbjzU/q5smzryuoNjlriAIx+06RpecwCkL9JpenNzpKd2YMzLJFOdPqBpuub6eVRP5IgiSA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-x64@0.17.19': resolution: {integrity: sha512-IJM4JJsLhRYr9xdtLytPLSH9k/oxR3boaUIYiHkAawtwNOXKE8KoU8tMvryogdcT8AU+Bflmh81Xn6Q0vTZbQw==} engines: {node: '>=12'} cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.25.2': + resolution: {integrity: sha512-5eRPrTX7wFyuWe8FqEFPG2cU0+butQQVNcT4sVipqjLYQjjh8a8+vUTfgBKM88ObB85ahsnTwF7PSIt6PG+QkA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/freebsd-arm64@0.17.19': resolution: {integrity: sha512-pBwbc7DufluUeGdjSU5Si+P3SoMF5DQ/F/UmTSb8HXO80ZEAJmrykPyzo1IfNbAoaqw48YRpv8shwd1NoI0jcQ==} engines: {node: '>=12'} cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.25.2': + resolution: {integrity: sha512-mLwm4vXKiQ2UTSX4+ImyiPdiHjiZhIaE9QvC7sw0tZ6HoNMjYAqQpGyui5VRIi5sGd+uWq940gdCbY3VLvsO1w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-x64@0.17.19': resolution: {integrity: sha512-4lu+n8Wk0XlajEhbEffdy2xy53dpR06SlzvhGByyg36qJw6Kpfk7cp45DR/62aPH9mtJRmIyrXAS5UWBrJT6TQ==} engines: {node: '>=12'} cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.25.2': + resolution: {integrity: sha512-6qyyn6TjayJSwGpm8J9QYYGQcRgc90nmfdUb0O7pp1s4lTY+9D0H9O02v5JqGApUyiHOtkz6+1hZNvNtEhbwRQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/linux-arm64@0.17.19': resolution: {integrity: sha512-ct1Tg3WGwd3P+oZYqic+YZF4snNl2bsnMKRkb3ozHmnM0dGWuxcPTTntAF6bOP0Sp4x0PjSF+4uHQ1xvxfRKqg==} engines: {node: '>=12'} cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.25.2': + resolution: {integrity: sha512-gq/sjLsOyMT19I8obBISvhoYiZIAaGF8JpeXu1u8yPv8BE5HlWYobmlsfijFIZ9hIVGYkbdFhEqC0NvM4kNO0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm@0.17.19': resolution: {integrity: sha512-cdmT3KxjlOQ/gZ2cjfrQOtmhG4HJs6hhvm3mWSRDPtZ/lP5oe8FWceS10JaSJC13GBd4eH/haHnqf7hhGNLerA==} engines: {node: '>=12'} cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.25.2': + resolution: {integrity: sha512-UHBRgJcmjJv5oeQF8EpTRZs/1knq6loLxTsjc3nxO9eXAPDLcWW55flrMVc97qFPbmZP31ta1AZVUKQzKTzb0g==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-ia32@0.17.19': resolution: {integrity: sha512-w4IRhSy1VbsNxHRQpeGCHEmibqdTUx61Vc38APcsRbuVgK0OPEnQ0YD39Brymn96mOx48Y2laBQGqgZ0j9w6SQ==} engines: {node: '>=12'} cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.25.2': + resolution: {integrity: sha512-bBYCv9obgW2cBP+2ZWfjYTU+f5cxRoGGQ5SeDbYdFCAZpYWrfjjfYwvUpP8MlKbP0nwZ5gyOU/0aUzZ5HWPuvQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-loong64@0.17.19': resolution: {integrity: sha512-2iAngUbBPMq439a+z//gE+9WBldoMp1s5GWsUSgqHLzLJ9WoZLZhpwWuym0u0u/4XmZ3gpHmzV84PonE+9IIdQ==} engines: {node: '>=12'} cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.25.2': + resolution: {integrity: sha512-SHNGiKtvnU2dBlM5D8CXRFdd+6etgZ9dXfaPCeJtz+37PIUlixvlIhI23L5khKXs3DIzAn9V8v+qb1TRKrgT5w==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-mips64el@0.17.19': resolution: {integrity: sha512-LKJltc4LVdMKHsrFe4MGNPp0hqDFA1Wpt3jE1gEyM3nKUvOiO//9PheZZHfYRfYl6AwdTH4aTcXSqBerX0ml4A==} engines: {node: '>=12'} cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.25.2': + resolution: {integrity: sha512-hDDRlzE6rPeoj+5fsADqdUZl1OzqDYow4TB4Y/3PlKBD0ph1e6uPHzIQcv2Z65u2K0kpeByIyAjCmjn1hJgG0Q==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-ppc64@0.17.19': resolution: {integrity: sha512-/c/DGybs95WXNS8y3Ti/ytqETiW7EU44MEKuCAcpPto3YjQbyK3IQVKfF6nbghD7EcLUGl0NbiL5Rt5DMhn5tg==} engines: {node: '>=12'} cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.25.2': + resolution: {integrity: sha512-tsHu2RRSWzipmUi9UBDEzc0nLc4HtpZEI5Ba+Omms5456x5WaNuiG3u7xh5AO6sipnJ9r4cRWQB2tUjPyIkc6g==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-riscv64@0.17.19': resolution: {integrity: sha512-FC3nUAWhvFoutlhAkgHf8f5HwFWUL6bYdvLc/TTuxKlvLi3+pPzdZiFKSWz/PF30TB1K19SuCxDTI5KcqASJqA==} engines: {node: '>=12'} cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.25.2': + resolution: {integrity: sha512-k4LtpgV7NJQOml/10uPU0s4SAXGnowi5qBSjaLWMojNCUICNu7TshqHLAEbkBdAszL5TabfvQ48kK84hyFzjnw==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-s390x@0.17.19': resolution: {integrity: sha512-IbFsFbxMWLuKEbH+7sTkKzL6NJmG2vRyy6K7JJo55w+8xDk7RElYn6xvXtDW8HCfoKBFK69f3pgBJSUSQPr+4Q==} engines: {node: '>=12'} cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.25.2': + resolution: {integrity: sha512-GRa4IshOdvKY7M/rDpRR3gkiTNp34M0eLTaC1a08gNrh4u488aPhuZOCpkF6+2wl3zAN7L7XIpOFBhnaE3/Q8Q==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-x64@0.17.19': resolution: {integrity: sha512-68ngA9lg2H6zkZcyp22tsVt38mlhWde8l3eJLWkyLrp4HwMUr3c1s/M2t7+kHIhvMjglIBrFpncX1SzMckomGw==} engines: {node: '>=12'} cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.25.2': + resolution: {integrity: sha512-QInHERlqpTTZ4FRB0fROQWXcYRD64lAoiegezDunLpalZMjcUcld3YzZmVJ2H/Cp0wJRZ8Xtjtj0cEHhYc/uUg==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.2': + resolution: {integrity: sha512-talAIBoY5M8vHc6EeI2WW9d/CkiO9MQJ0IOWX8hrLhxGbro/vBXJvaQXefW2cP0z0nQVTdQ/eNyGFV1GSKrxfw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + '@esbuild/netbsd-x64@0.17.19': resolution: {integrity: sha512-CwFq42rXCR8TYIjIfpXCbRX0rp1jo6cPIUPSaWwzbVI4aOfX96OXY8M6KNmtPcg7QjYeDmN+DD0Wp3LaBOLf4Q==} engines: {node: '>=12'} cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.25.2': + resolution: {integrity: sha512-voZT9Z+tpOxrvfKFyfDYPc4DO4rk06qamv1a/fkuzHpiVBMOhpjK+vBmWM8J1eiB3OLSMFYNaOaBNLXGChf5tg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.2': + resolution: {integrity: sha512-dcXYOC6NXOqcykeDlwId9kB6OkPUxOEqU+rkrYVqJbK2hagWOMrsTGsMr8+rW02M+d5Op5NNlgMmjzecaRf7Tg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-x64@0.17.19': resolution: {integrity: sha512-cnq5brJYrSZ2CF6c35eCmviIN3k3RczmHz8eYaVlNasVqsNY+JKohZU5MKmaOI+KkllCdzOKKdPs762VCPC20g==} engines: {node: '>=12'} cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.25.2': + resolution: {integrity: sha512-t/TkWwahkH0Tsgoq1Ju7QfgGhArkGLkF1uYz8nQS/PPFlXbP5YgRpqQR3ARRiC2iXoLTWFxc6DJMSK10dVXluw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/sunos-x64@0.17.19': resolution: {integrity: sha512-vCRT7yP3zX+bKWFeP/zdS6SqdWB8OIpaRq/mbXQxTGHnIxspRtigpkUcDMlSCOejlHowLqII7K2JKevwyRP2rg==} engines: {node: '>=12'} cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.25.2': + resolution: {integrity: sha512-cfZH1co2+imVdWCjd+D1gf9NjkchVhhdpgb1q5y6Hcv9TP6Zi9ZG/beI3ig8TvwT9lH9dlxLq5MQBBgwuj4xvA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/win32-arm64@0.17.19': resolution: {integrity: sha512-yYx+8jwowUstVdorcMdNlzklLYhPxjniHWFKgRqH7IFlUEa0Umu3KuYplf1HUZZ422e3NU9F4LGb+4O0Kdcaag==} engines: {node: '>=12'} cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.25.2': + resolution: {integrity: sha512-7Loyjh+D/Nx/sOTzV8vfbB3GJuHdOQyrOryFdZvPHLf42Tk9ivBU5Aedi7iyX+x6rbn2Mh68T4qq1SDqJBQO5Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-ia32@0.17.19': resolution: {integrity: sha512-eggDKanJszUtCdlVs0RB+h35wNlb5v4TWEkq4vZcmVt5u/HiDZrTXe2bWFQUez3RgNHwx/x4sk5++4NSSicKkw==} engines: {node: '>=12'} cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.25.2': + resolution: {integrity: sha512-WRJgsz9un0nqZJ4MfhabxaD9Ft8KioqU3JMinOTvobbX6MOSUigSBlogP8QB3uxpJDsFS6yN+3FDBdqE5lg9kg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-x64@0.17.19': resolution: {integrity: sha512-lAhycmKnVOuRYNtRtatQR1LPQf2oYCkRGkSFnseDAKPl8lu5SOsK/e1sXe5a0Pc5kHIHe6P2I/ilntNv2xf3cA==} engines: {node: '>=12'} cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.25.2': + resolution: {integrity: sha512-kM3HKb16VIXZyIeVrM1ygYmZBKybX8N4p754bw390wGO3Tf2j4L2/WYL+4suWujpgf6GBYs3jv7TyUivdd05JA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@eslint-community/eslint-utils@4.4.1': resolution: {integrity: sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -397,6 +559,9 @@ packages: '@floating-ui/utils@0.2.9': resolution: {integrity: sha512-MDWhGtE+eHw5JW7lq4qhc5yRLS11ERl1c7Z6Xd0a58DozHES6EnNNwUWbMiG4J9Cgj053Bhk8zvlhFYKVhULwg==} + '@formatjs/intl-localematcher@0.6.1': + resolution: {integrity: sha512-ePEgLgVCqi2BBFnTMWPfIghu6FkbZnnBVhO2sSxvLfrdFw7wCHAHiDoM2h4NRgjbaY7+B7HgOLZGkK187pZTZg==} + '@hookform/resolvers@4.1.3': resolution: {integrity: sha512-Jsv6UOWYTrEFJ/01ZrnwVXs7KDvP8XIo115i++5PWvNkNvkrsTfGiLS6w+eJ57CYtUtDQalUWovCZDHFJ8u1VQ==} peerDependencies: @@ -549,6 +714,9 @@ packages: '@jridgewell/trace-mapping@0.3.25': resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} + '@mdx-js/mdx@3.1.0': + resolution: {integrity: sha512-/QxEhPAvGwbQmy1Px8F899L5Uc2KZ6JtXwlCgJmjSTBedwOZkByYcBG4GceIGPXRDsmfxhHazuS+hlOShRLeDw==} + '@next/env@15.2.3': resolution: {integrity: sha512-a26KnbW9DFEUsSxAxKBORR/uD9THoYoKbkpFywMN/AFvboTt94b8+g/07T8J6ACsdLag8/PDU60ov4rPxRAixw==} @@ -623,6 +791,10 @@ packages: resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} + '@orama/orama@3.1.6': + resolution: {integrity: sha512-qtSrqCqRU93SjEBedz987tvWao1YQSELjBhGkHYGVP7Dg0lBWP6d+uZEIt5gxTAYio/YWWlhivmRABvRfPLmnQ==} + engines: {node: '>= 16.0.0'} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -630,6 +802,9 @@ packages: '@radix-ui/number@1.1.0': resolution: {integrity: sha512-V3gRzhVNU1ldS5XhAPTom1fOIo4ccrjjJgmE+LI2h/WaFpHmx0MQApT+KZHnx8abG6Avtfcz4WoEciMnpFT3HQ==} + '@radix-ui/number@1.1.1': + resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==} + '@radix-ui/primitive@1.0.0': resolution: {integrity: sha512-3e7rn8FDMin4CgeL7Z/49smCA3rFYY3Ha2rUQ7HRWFadS5iCRw08ZgVT1LaNTCNqgvrUiyczLflrVrF0SRQtNA==} @@ -639,6 +814,9 @@ packages: '@radix-ui/primitive@1.1.1': resolution: {integrity: sha512-SJ31y+Q/zAyShtXJc8x83i9TYdbAfHZ++tUZnvjJJqFjzsdUnKsxPL6IEtBlxKkU7yzer//GQtZSV4GbldL3YA==} + '@radix-ui/primitive@1.1.2': + resolution: {integrity: sha512-XnbHrrprsNqZKQhStrSwgRUQzoCI1glLzdw79xiZPoofhGICeZRSQ3dIxAKH1gb3OHfNf4d6f+vAv3kil2eggA==} + '@radix-ui/react-accordion@1.2.3': resolution: {integrity: sha512-RIQ15mrcvqIkDARJeERSuXSry2N8uYnxkdDetpfmalT/+0ntOXLkFOsh9iwlAsCv+qcmhZjbdJogIm6WBa6c4A==} peerDependencies: @@ -652,6 +830,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-accordion@1.2.7': + resolution: {integrity: sha512-stDPylBV/3kFHBAFQK/GeyIFaN7q60zWaXthA5/p6egu8AclIN79zG+bv+Ps+exB4JE5rtW/u3Z7SDvmFuTzgA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-alert-dialog@1.1.6': resolution: {integrity: sha512-p4XnPqgej8sZAAReCAKgz1REYZEBLR8hU9Pg27wFnCWIMc8g1ccCs0FjBcy05V15VTu8pAePw/VDYeOm/uZ6yQ==} peerDependencies: @@ -678,6 +869,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-arrow@1.1.4': + resolution: {integrity: sha512-qz+fxrqgNxG0dYew5l7qR3c7wdgRu1XVUHGnGYX7rg5HM4p9SWaRmJwfgR3J0SgyUKayLmzQIun+N6rWRgiRKw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-avatar@1.1.3': resolution: {integrity: sha512-Paen00T4P8L8gd9bNsRMw7Cbaz85oxiv+hzomsRZgFm2byltPFDtfcoqlWJ8GyZlIBWgLssJlzLCnKU0G0302g==} peerDependencies: @@ -717,6 +921,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-collapsible@1.1.7': + resolution: {integrity: sha512-zGFsPcFJNdQa/UNd6MOgF40BS054FIGj32oOWBllixz42f+AkQg3QJ1YT9pw7vs+Ai+EgWkh839h69GEK8oH2A==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-collection@1.1.2': resolution: {integrity: sha512-9z54IEKRxIa9VityapoEYMuByaG42iSy1ZXlY2KcuLSEtq8x4987/N6m15ppoMffgZX72gER2uHe1D9Y6Unlcw==} peerDependencies: @@ -730,6 +947,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-collection@1.1.4': + resolution: {integrity: sha512-cv4vSf7HttqXilDnAnvINd53OTl1/bjUYVZrkFnA7nwmY9Ob2POUy0WY0sfqBAe1s5FyKsyceQlqiEGPYNTadg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-compose-refs@1.0.0': resolution: {integrity: sha512-0KaSv6sx787/hK3eF53iOkiSLwAGlFMx5lotrqD2pTjB18KbybKoEIgkNZTKC60YECDQTKGTRcDBILwZVqVKvA==} peerDependencies: @@ -753,6 +983,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-compose-refs@1.1.2': + resolution: {integrity: sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-context@1.0.0': resolution: {integrity: sha512-1pVM9RfOQ+n/N5PJK33kRSKsr1glNxomxONs5c49MliinBY6Yw2Q995qfBUUo0/Mbg05B/sGA0gkgPI7kmSHBg==} peerDependencies: @@ -776,6 +1015,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-context@1.1.2': + resolution: {integrity: sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-dialog@1.0.0': resolution: {integrity: sha512-Yn9YU+QlHYLWwV1XfKiqnGVpWYWk6MeBVM6x/bcoyPvxgjQGoeT35482viLPctTMWoMw0PoHgqfSox7Ig+957Q==} peerDependencies: @@ -795,6 +1043,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-dialog@1.1.10': + resolution: {integrity: sha512-m6pZb0gEM5uHPSb+i2nKKGQi/HMSVjARMsLMWQfKDP+eJ6B+uqryHnXhpnohTWElw+vEcMk/o4wJODtdRKHwqg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-dialog@1.1.6': resolution: {integrity: sha512-/IVhJV5AceX620DUJ4uYVMymzsipdKBzo3edo+omeskCKGm9FRHM0ebIdbPnlQVJqyuHbuBltQUOG2mOTq2IYw==} peerDependencies: @@ -817,6 +1078,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-direction@1.1.1': + resolution: {integrity: sha512-1UEWRX6jnOA2y4H5WczZ44gOOjTEmlqv1uNW4GAJEO5+bauCBhv8snY65Iw5/VOS/ghKN9gr2KjnLKxrsvoMVw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-dismissable-layer@1.0.0': resolution: {integrity: sha512-n7kDRfx+LB1zLueRDvZ1Pd0bxdJWDUZNQ/GWoxDn2prnuJKRdxsjulejX/ePkOsLi2tTm6P24mDqlMSgQpsT6g==} peerDependencies: @@ -849,6 +1119,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-dismissable-layer@1.1.7': + resolution: {integrity: sha512-j5+WBUdhccJsmH5/H0K6RncjDtoALSEr6jbkaZu+bjw6hOPOhHycr6vEUujl+HBK8kjUfWcoCJXxP6e4lUlMZw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-dropdown-menu@2.1.6': resolution: {integrity: sha512-no3X7V5fD487wab/ZYSHXq3H37u4NVeLDKI/Ks724X/eEFSSEFYZxWgsIlr1UBeEyDaM29HM5x9p1Nv8DuTYPA==} peerDependencies: @@ -885,6 +1168,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-focus-guards@1.1.2': + resolution: {integrity: sha512-fyjAACV62oPV925xFCrH8DR5xWhg9KYtJT4s3u54jxp+L/hbpTY2kIeEFFbFe+a/HCE94zGQMZLIpVTPVZDhaA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-focus-scope@1.0.0': resolution: {integrity: sha512-C4SWtsULLGf/2L4oGeIHlvWQx7Rf+7cX/vKOAD2dXW0A1b5QXwi3wWeaEgW+wn+SEVrraMUk05vLU9fZZz5HbQ==} peerDependencies: @@ -917,6 +1209,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-focus-scope@1.1.4': + resolution: {integrity: sha512-r2annK27lIW5w9Ho5NyQgqs0MmgZSTIKXWpVCJaLC1q2kZrZkcqnmHkCHMEmv8XLvsLlurKMPT+kbKkRkm/xVA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-id@1.0.0': resolution: {integrity: sha512-Q6iAB/U7Tq3NTolBBQbHTgclPmGWE3OlktGGqrClPozSw4vkQ1DfQAOtzgRPecKsMdJINE05iaoDUG8tRzCBjw==} peerDependencies: @@ -940,6 +1245,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-id@1.1.1': + resolution: {integrity: sha512-kGkGegYIdQsOb4XjsfM97rXsiHaBwco+hFI66oO4s9LU+PLAC5oJ7khdOVFxkhsmlbpUqDAvXw11CluXP+jkHg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-label@2.1.2': resolution: {integrity: sha512-zo1uGMTaNlHehDyFQcDZXRJhUPDuukcnHz0/jnrup0JA6qL+AFpAnty+7VKa9esuU5xTblAZzTGYJKSKaBxBhw==} peerDependencies: @@ -966,6 +1280,32 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-navigation-menu@1.2.9': + resolution: {integrity: sha512-Z7lefjA5VAmEB5ZClxeHGWGQAqhGWgEc6u0MYviUmIVrgGCVLv5mv/jsfUY3tJWI71cVhpQ7dnf/Q6RtM3ylVA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-popover@1.1.10': + resolution: {integrity: sha512-IZN7b3sXqajiPsOzKuNJBSP9obF4MX5/5UhTgWNofw4r1H+eATWb0SyMlaxPD/kzA4vadFgy1s7Z1AEJ6WMyHQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-popover@1.1.6': resolution: {integrity: sha512-NQouW0x4/GnkFJ/pRqsIS3rM/k97VzKnVb2jB7Gq7VEGPy5g7uNV1ykySFt7eWSp3i2uSGFwaJcvIRJBAHmmFg==} peerDependencies: @@ -992,6 +1332,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-popper@1.2.4': + resolution: {integrity: sha512-3p2Rgm/a1cK0r/UVkx5F/K9v/EplfjAeIFCGOPYPO4lZ0jtg4iSQXt/YGTSLWaf4x7NG6Z4+uKFcylcTZjeqDA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-portal@1.0.0': resolution: {integrity: sha512-a8qyFO/Xb99d8wQdu4o7qnigNjTPG123uADNecz0eX4usnQEj7o+cG4ZX4zkqq98NYekT7UoEQIjxBNWIFuqTA==} peerDependencies: @@ -1024,6 +1377,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-portal@1.1.6': + resolution: {integrity: sha512-XmsIl2z1n/TsYFLIdYam2rmFwf9OC/Sh2avkbmVMDuBZIe7hSpM0cYnWPAo7nHOVx8zTuwDZGByfcqLdnzp3Vw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-presence@1.0.0': resolution: {integrity: sha512-A+6XEvN01NfVWiKu38ybawfHsBjWum42MRPnEuqPsBZ4eV7e/7K321B5VgYMPv3Xx5An6o1/l9ZuDBgmcmWK3w==} peerDependencies: @@ -1056,6 +1422,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-presence@1.1.3': + resolution: {integrity: sha512-IrVLIhskYhH3nLvtcBLQFZr61tBG7wx7O3kEmdzcYwRGAEBmBicGGL7ATzNgruYJ3xBTbuzEEq9OXJM3PAX3tA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-primitive@1.0.0': resolution: {integrity: sha512-EyXe6mnRlHZ8b6f4ilTDrXmkLShICIuOTTj0GX4w1rp+wSxf3+TD05u1UOITC8VsJ2a9nwHvdXtOXEOl0Cw/zQ==} peerDependencies: @@ -1088,6 +1467,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-primitive@2.1.0': + resolution: {integrity: sha512-/J/FhLdK0zVcILOwt5g+dH4KnkonCtkVJsa2G6JmvbbtZfBEI1gMsO3QMjseL4F/SwfAMt1Vc/0XKYKq+xJ1sw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-roving-focus@1.1.2': resolution: {integrity: sha512-zgMQWkNO169GtGqRvYrzb0Zf8NhMHS2DuEB/TiEmVnpr5OqPU3i8lfbxaAmC2J/KYuIQxyoQQ6DxepyXp61/xw==} peerDependencies: @@ -1101,6 +1493,32 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-roving-focus@1.1.7': + resolution: {integrity: sha512-C6oAg451/fQT3EGbWHbCQjYTtbyjNO1uzQgMzwyivcHT3GKNEmu1q3UuREhN+HzHAVtv3ivMVK08QlC+PkYw9Q==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-scroll-area@1.2.5': + resolution: {integrity: sha512-VyLjxI8/gXYn+Wij1FLpXjZp6Z/uNklUFQQ75tOpJNESeNaZ2kCRfjiEDmHgWmLeUPeJGwrqbgRmcdFjtYEkMA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-select@2.1.6': resolution: {integrity: sha512-T6ajELxRvTuAMWH0YmRJ1qez+x4/7Nq7QIx7zJ0VK3qaEWdnWpNbEDnmWldG1zBDwqrLy5aLMUWcoGirVj5kMg==} peerDependencies: @@ -1150,6 +1568,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-slot@1.2.0': + resolution: {integrity: sha512-ujc+V6r0HNDviYqIK3rW4ffgYiZ8g5DEHrGJVk4x7kTlLXRDILnKX9vAUYeIsLOoDpDJ0ujpqMkjH4w2ofuo6w==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-tabs@1.1.3': resolution: {integrity: sha512-9mFyI30cuRDImbmFF6O2KUJdgEOsGh9Vmx9x/Dh9tOhL7BngmQPQfwW4aejKm5OHpfWIdmeV6ySyuxoOGjtNng==} peerDependencies: @@ -1163,6 +1590,19 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-tabs@1.1.8': + resolution: {integrity: sha512-4iUaN9SYtG+/E+hJ7jRks/Nv90f+uAsRHbLYA6BcA9EsR6GNWgsvtS4iwU2SP0tOZfDGAyqIT0yz7ckgohEIFA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-tooltip@1.1.8': resolution: {integrity: sha512-YAA2cu48EkJZdAMHC0dqo9kialOcRStbtiY4nJPaht7Ptrhcvpo+eDChaM6BIs8kL6a8Z5l5poiqLnXcNduOkA==} peerDependencies: @@ -1199,6 +1639,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-use-callback-ref@1.1.1': + resolution: {integrity: sha512-FkBMwD+qbGQeMu1cOHnuGB6x4yzPjho8ap5WtbEJ26umhgqVXbhekKUQO+hZEL1vU92a3wHwdp0HAcqAUF5iDg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-use-controllable-state@1.0.0': resolution: {integrity: sha512-FohDoZvk3mEXh9AWAVyRTYR4Sq7/gavuofglmiXB2g1aKyboUD4YtgWxKj8O5n+Uak52gXQ4wKz5IFST4vtJHg==} peerDependencies: @@ -1222,6 +1671,24 @@ packages: '@types/react': optional: true + '@radix-ui/react-use-controllable-state@1.2.2': + resolution: {integrity: sha512-BjasUjixPFdS+NKkypcyyN5Pmg83Olst0+c6vGov0diwTEo6mgdqVR6hxcEgFuh4QrAs7Rc+9KuGJ9TVCj0Zzg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-effect-event@0.0.2': + resolution: {integrity: sha512-Qp8WbZOBe+blgpuUT+lw2xheLP8q0oatc9UpmiemEICxGvFLYmHm9QowVZGHtJlGbS6A6yJ3iViad/2cVjnOiA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-use-escape-keydown@1.0.0': resolution: {integrity: sha512-JwfBCUIfhXRxKExgIqGa4CQsiMemo1Xt0W/B4ei3fpzpvPENKpMKQ8mZSB6Acj3ebrAEgi2xiQvcI1PAAodvyg==} peerDependencies: @@ -1245,6 +1712,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-use-escape-keydown@1.1.1': + resolution: {integrity: sha512-Il0+boE7w/XebUHyBjroE+DbByORGR9KKmITzbR7MyQ4akpORYP/ZmbhAr0DG7RmmBqoOnZdy2QlvajJ2QA59g==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-use-layout-effect@1.0.0': resolution: {integrity: sha512-6Tpkq+R6LOlmQb1R5NNETLG0B4YP0wc+klfXafpUCj6JGyaUc8il7/kUZ7m59rGbXGczE9Bs+iz2qloqsZBduQ==} peerDependencies: @@ -1268,6 +1744,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-use-layout-effect@1.1.1': + resolution: {integrity: sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-use-previous@1.1.0': resolution: {integrity: sha512-Z/e78qg2YFnnXcW88A4JmTtm4ADckLno6F7OXotmkQfeuCVaKuYzqAATPhVzl3delXE7CxIV8shofPn3jPc5Og==} peerDependencies: @@ -1277,6 +1762,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-use-previous@1.1.1': + resolution: {integrity: sha512-2dHfToCj/pzca2Ck724OZ5L0EVrr3eHRNsG/b3xQJLA2hZpVCS99bLAX+hm1IHXDEnzU6by5z/5MIY794/a8NQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-use-rect@1.1.0': resolution: {integrity: sha512-0Fmkebhr6PiseyZlYAOtLS+nb7jLmpqTrJyv61Pe68MKYW6OWdRE2kI70TaYY27u7H0lajqM3hSMMLFq18Z7nQ==} peerDependencies: @@ -1286,6 +1780,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-use-rect@1.1.1': + resolution: {integrity: sha512-QTYuDesS0VtuHNNvMh+CjlKJ4LJickCMUAqjlE3+j8w+RlRpwyX3apEQKGFzbZGdo7XNG1tXa+bQqIE7HIXT2w==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-use-size@1.1.0': resolution: {integrity: sha512-XW3/vWuIXHa+2Uwcc2ABSfcCledmXhhQPlGbfcRXbiUQI5Icjcg19BGCZVKKInYbvUCut/ufbbLLPFC5cbb1hw==} peerDependencies: @@ -1295,6 +1798,15 @@ packages: '@types/react': optional: true + '@radix-ui/react-use-size@1.1.1': + resolution: {integrity: sha512-ewrXRDTAqAXlkl6t/fkXWNAhFX9I+CkKlw6zjEwk86RSPKwZr3xpBRso655aqYafwtnbpHLj6toFzmd6xdVptQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@radix-ui/react-visually-hidden@1.1.2': resolution: {integrity: sha512-1SzA4ns2M1aRlvxErqhLHsBHoS5eI5UUcI2awAMgGUp4LoaoWOKYmvqDY2s/tltuPkh3Yk77YF/r3IRj+Amx4Q==} peerDependencies: @@ -1308,15 +1820,61 @@ packages: '@types/react-dom': optional: true + '@radix-ui/react-visually-hidden@1.2.0': + resolution: {integrity: sha512-rQj0aAWOpCdCMRbI6pLQm8r7S2BM3YhTa0SzOYD55k+hJA8oo9J+H+9wLM9oMlZWOX/wJWPTzfDfmZkf7LvCfg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/rect@1.1.0': resolution: {integrity: sha512-A9+lCBZoaMJlVKcRBz2YByCG+Cp2t6nAnMnNba+XiWxnj6r4JUFqfsgwocMBZU9LPtdxC6wB56ySYpc7LQIoJg==} + '@radix-ui/rect@1.1.1': + resolution: {integrity: sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw==} + '@rtsao/scc@1.1.0': resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==} '@rushstack/eslint-patch@1.10.5': resolution: {integrity: sha512-kkKUDVlII2DQiKy7UstOR1ErJP8kUKAQ4oa+SQtM0K+lPdmmjj0YnnxBgtTVYH7mUKtbsxeFC9y0AmK7Yb78/A==} + '@shikijs/core@3.3.0': + resolution: {integrity: sha512-CovkFL2WVaHk6PCrwv6ctlmD4SS1qtIfN8yEyDXDYWh4ONvomdM9MaFw20qHuqJOcb8/xrkqoWQRJ//X10phOQ==} + + '@shikijs/engine-javascript@3.3.0': + resolution: {integrity: sha512-XlhnFGv0glq7pfsoN0KyBCz9FJU678LZdQ2LqlIdAj6JKsg5xpYKay3DkazXWExp3DTJJK9rMOuGzU2911pg7Q==} + + '@shikijs/engine-oniguruma@3.3.0': + resolution: {integrity: sha512-l0vIw+GxeNU7uGnsu6B+Crpeqf+WTQ2Va71cHb5ZYWEVEPdfYwY5kXwYqRJwHrxz9WH+pjSpXQz+TJgAsrkA5A==} + + '@shikijs/langs@3.3.0': + resolution: {integrity: sha512-zt6Kf/7XpBQKSI9eqku+arLkAcDQ3NHJO6zFjiChI8w0Oz6Jjjay7pToottjQGjSDCFk++R85643WbyINcuL+g==} + + '@shikijs/rehype@3.3.0': + resolution: {integrity: sha512-m9clrxedJHyKDwYoAkIUJ7thWGSZwZbA0PeGDST7NHCTGeS227BFn8Hoq2olAtxXo14k5T1JcUCDgyaRZfI4Hw==} + + '@shikijs/themes@3.3.0': + resolution: {integrity: sha512-tXeCvLXBnqq34B0YZUEaAD1lD4lmN6TOHAhnHacj4Owh7Ptb/rf5XCDeROZt2rEOk5yuka3OOW2zLqClV7/SOg==} + + '@shikijs/transformers@3.3.0': + resolution: {integrity: sha512-PIknEyxfkT7i7at/78ynVmuZEv4+7IcS37f6abxMjQ0pVIPEya8n+KNl7XtfbhNL+U9ElR3UzfSzuD5l5Iu+nw==} + + '@shikijs/types@3.3.0': + resolution: {integrity: sha512-KPCGnHG6k06QG/2pnYGbFtFvpVJmC3uIpXrAiPrawETifujPBv0Se2oUxm5qYgjCvGJS9InKvjytOdN+bGuX+Q==} + + '@shikijs/vscode-textmate@10.0.2': + resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} + + '@standard-schema/spec@1.0.0': + resolution: {integrity: sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA==} + '@standard-schema/utils@0.3.0': resolution: {integrity: sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==} @@ -1450,6 +2008,9 @@ packages: '@types/mdast@4.0.4': resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} + '@types/mdx@2.0.13': + resolution: {integrity: sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw==} + '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} @@ -1568,6 +2129,9 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -1625,6 +2189,10 @@ packages: ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} + astring@1.9.0: + resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} + hasBin: true + async-function@1.0.0: resolution: {integrity: sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==} engines: {node: '>= 0.4'} @@ -1728,6 +2296,10 @@ packages: resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} engines: {node: '>= 8.10.0'} + chokidar@4.0.3: + resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} + engines: {node: '>= 14.16.0'} + class-variance-authority@0.7.1: resolution: {integrity: sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg==} @@ -1744,6 +2316,9 @@ packages: react: ^18.0.0 react-dom: ^18.0.0 + collapse-white-space@2.1.0: + resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -1765,6 +2340,9 @@ packages: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} + compute-scroll-into-view@3.1.1: + resolution: {integrity: sha512-VRhuHOLoKYOy4UbilLbUzbYg93XLjv2PncJC50EuTWPA3gaja1UjBsUP/D/9/juV3vQFr6XBEzn9KCAHdUvOHw==} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -1880,6 +2458,9 @@ packages: react: ^18.0.0 react-dom: ^18.0.0 + emoji-regex-xs@1.0.0: + resolution: {integrity: sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg==} + emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} @@ -1926,11 +2507,22 @@ packages: resolution: {integrity: sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==} engines: {node: '>= 0.4'} + esast-util-from-estree@2.0.0: + resolution: {integrity: sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==} + + esast-util-from-js@2.0.1: + resolution: {integrity: sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw==} + esbuild@0.17.19: resolution: {integrity: sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw==} engines: {node: '>=12'} hasBin: true + esbuild@0.25.2: + resolution: {integrity: sha512-16854zccKPnC+toMywC+uKNeYSv+/eXkevRAfwRD/G9Cleq66m8XFIrigkbvauLLlCfDL45Q2cWegSg53gGBnQ==} + engines: {node: '>=18'} + hasBin: true + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -2039,6 +2631,11 @@ packages: resolution: {integrity: sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + esquery@1.6.0: resolution: {integrity: sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==} engines: {node: '>=0.10'} @@ -2051,9 +2648,30 @@ packages: resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} engines: {node: '>=4.0'} + estree-util-attach-comments@3.0.0: + resolution: {integrity: sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw==} + + estree-util-build-jsx@3.0.1: + resolution: {integrity: sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ==} + estree-util-is-identifier-name@3.0.0: resolution: {integrity: sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==} + estree-util-scope@1.0.0: + resolution: {integrity: sha512-2CAASclonf+JFWBNJPndcOpA8EMJwa0Q8LUFJEKqXLW6+qBvbFZuF5gItbQOs/umBUkjviCSDCbBwU2cXbmrhQ==} + + estree-util-to-js@2.0.0: + resolution: {integrity: sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg==} + + estree-util-value-to-estree@3.3.3: + resolution: {integrity: sha512-Db+m1WSD4+mUO7UgMeKkAwdbfNWwIxLt48XF2oFU9emPfXkIu+k5/nlOj313v7wqtAPo0f9REhUvznFrPkG8CQ==} + + estree-util-visit@2.0.0: + resolution: {integrity: sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==} + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + esutils@2.0.3: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} @@ -2066,6 +2684,10 @@ packages: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} + extend-shallow@2.0.1: + resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} + engines: {node: '>=0.10.0'} + extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} @@ -2161,6 +2783,48 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + fumadocs-core@15.2.9: + resolution: {integrity: sha512-CMclwgXDDdqIA6ZZxEfFvFtkWTN5+X75vIvBlW7BAV4On8oCUOtBl5Whr02rpc7glyKQfOvdKXvHX5xbMHV5WA==} + peerDependencies: + '@oramacloud/client': 1.x.x || 2.x.x + algoliasearch: 4.24.0 + next: 14.x.x || 15.x.x + react: 18.x.x || 19.x.x + react-dom: 18.x.x || 19.x.x + peerDependenciesMeta: + '@oramacloud/client': + optional: true + algoliasearch: + optional: true + next: + optional: true + react: + optional: true + react-dom: + optional: true + + fumadocs-mdx@11.6.1: + resolution: {integrity: sha512-z+H/eOJC4II0VW7rgf6btqeEkD9DEG1SNToNCYKMklCJAc9Y6l+NuQozKuknP2Ey6NK+Qqhvwhi2MOq38YLSeQ==} + hasBin: true + peerDependencies: + '@fumadocs/mdx-remote': ^1.2.0 + fumadocs-core: ^14.0.0 || ^15.0.0 + next: ^15.3.0 + peerDependenciesMeta: + '@fumadocs/mdx-remote': + optional: true + + fumadocs-ui@15.2.9: + resolution: {integrity: sha512-6bCbWEFc19x6sMi1k+bEKWumOhaPN7N//ih0fNF5OUh6GY3Dfa7NUuGHCfyxFRNAz2dcQQ6YL89L9E35fxoHvg==} + peerDependencies: + next: 14.x.x || 15.x.x + react: 18.x.x || 19.x.x + react-dom: 18.x.x || 19.x.x + tailwindcss: ^3.4.14 || ^4.0.0 + peerDependenciesMeta: + tailwindcss: + optional: true + function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} @@ -2199,6 +2863,9 @@ packages: get-tsconfig@4.10.0: resolution: {integrity: sha512-kGzZ3LWWQcGIAmg6iWvXn0ei6WDtV26wzHRMwDSzmAbcXrTEXxHy6IehI6/4eT6VRKyMP1eF1VqwrVUmE/LR7A==} + github-slugger@2.0.0: + resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} + glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -2233,6 +2900,10 @@ packages: graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} + gray-matter@4.0.3: + resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} + engines: {node: '>=6.0'} + has-bigints@1.1.0: resolution: {integrity: sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==} engines: {node: '>= 0.4'} @@ -2272,12 +2943,21 @@ packages: hast-util-sanitize@5.0.2: resolution: {integrity: sha512-3yTWghByc50aGS7JlGhk61SPenfE/p1oaFeNwkOOyrscaOkMGrcW9+Cy/QAIOBpZxP1yqDIzFMR0+Np0i0+usg==} + hast-util-to-estree@3.1.3: + resolution: {integrity: sha512-48+B/rJWAp0jamNbAAf9M7Uf//UVqAoMmgXhBdxTDJLGKY+LRnZ99qcG+Qjl5HfMpYNzS5v4EAwVEF34LeAj7w==} + + hast-util-to-html@9.0.5: + resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==} + hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} hast-util-to-parse5@8.0.0: resolution: {integrity: sha512-3KKrV5ZVI8if87DVSi1vDeByYrkGzg4mEfeu4alwgmmIeARiBLKCZS2uw5Gb6nU9x9Yufyj3iudm6i7nl52PFw==} + hast-util-to-string@3.0.1: + resolution: {integrity: sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==} + hast-util-whitespace@3.0.0: resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} @@ -2298,6 +2978,11 @@ packages: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} + image-size@2.0.2: + resolution: {integrity: sha512-IRqXKlaXwgSMAMtpNzZa1ZAe8m+Sa1770Dhk8VkSsP9LS+iHD62Zd8FQKs8fbPiagBE7BzoFX23cxFnwshpV6w==} + engines: {node: '>=16.x'} + hasBin: true + import-fresh@3.3.1: resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} engines: {node: '>=6'} @@ -2364,6 +3049,10 @@ packages: is-decimal@2.0.1: resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} + is-extendable@0.1.1: + resolution: {integrity: sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==} + engines: {node: '>=0.10.0'} + is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -2467,6 +3156,10 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-yaml@3.14.1: + resolution: {integrity: sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==} + hasBin: true + js-yaml@4.1.0: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} hasBin: true @@ -2499,6 +3192,10 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + kind-of@6.0.3: + resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} + engines: {node: '>=0.10.0'} + language-subtag-registry@0.3.23: resolution: {integrity: sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==} @@ -2617,11 +3314,24 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.1.0: + resolution: {integrity: sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==} + engines: {node: 20 || >=22} + lucide-react@0.477.0: resolution: {integrity: sha512-yCf7aYxerFZAbd8jHJxjwe1j7jEMPptjnaOqdYeirFnEy85cNR3/L+o0I875CYFYya+eEVzZSbNuRk8BZPDpVw==} peerDependencies: react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 + lucide-react@0.488.0: + resolution: {integrity: sha512-ronlL0MyKut4CEzBY/ai2ZpKPxyWO4jUqdAkm2GNK5Zn3Rj+swDz+3lvyAUXN0PNqPKIX6XM9Xadwz/skLs/pQ==} + peerDependencies: + react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + markdown-extensions@2.0.0: + resolution: {integrity: sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==} + engines: {node: '>=16'} + markdown-table@3.0.4: resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} @@ -2659,6 +3369,9 @@ packages: mdast-util-mdx-jsx@3.2.0: resolution: {integrity: sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==} + mdast-util-mdx@3.0.0: + resolution: {integrity: sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==} + mdast-util-mdxjs-esm@2.0.1: resolution: {integrity: sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==} @@ -2705,12 +3418,30 @@ packages: micromark-extension-gfm@3.0.0: resolution: {integrity: sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==} + micromark-extension-mdx-expression@3.0.1: + resolution: {integrity: sha512-dD/ADLJ1AeMvSAKBwO22zG22N4ybhe7kFIZ3LsDI0GlsNr2A3KYxb0LdC1u5rj4Nw+CHKY0RVdnHX8vj8ejm4Q==} + + micromark-extension-mdx-jsx@3.0.2: + resolution: {integrity: sha512-e5+q1DjMh62LZAJOnDraSSbDMvGJ8x3cbjygy2qFEi7HCeUT4BDKCvMozPozcD6WmOt6sVvYDNBKhFSz3kjOVQ==} + + micromark-extension-mdx-md@2.0.0: + resolution: {integrity: sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==} + + micromark-extension-mdxjs-esm@3.0.0: + resolution: {integrity: sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A==} + + micromark-extension-mdxjs@3.0.0: + resolution: {integrity: sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==} + micromark-factory-destination@2.0.1: resolution: {integrity: sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==} micromark-factory-label@2.0.1: resolution: {integrity: sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==} + micromark-factory-mdx-expression@2.0.3: + resolution: {integrity: sha512-kQnEtA3vzucU2BkrIa8/VaSAsP+EJ3CKOvhMuJgOEGg9KDC6OAY6nSnNDVRiVNRqj7Y4SlSzcStaH/5jge8JdQ==} + micromark-factory-space@2.0.1: resolution: {integrity: sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==} @@ -2741,6 +3472,9 @@ packages: micromark-util-encode@2.0.1: resolution: {integrity: sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==} + micromark-util-events-to-acorn@2.0.3: + resolution: {integrity: sha512-jmsiEIiZ1n7X1Rr5k8wVExBQCg5jy4UXVADItHmNk1zkwEVhBuIUKRu3fqv+hs4nxLISi2DQGlqIOGiFxgbfHg==} + micromark-util-html-tag-name@2.0.1: resolution: {integrity: sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==} @@ -2812,12 +3546,22 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + negotiator@1.0.0: + resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} + engines: {node: '>= 0.6'} + next-themes@0.4.4: resolution: {integrity: sha512-LDQ2qIOJF0VnuVrrMSMLrWGjRMkq+0mpgl6e0juCLqdJ+oo8Q84JRWT6Wh11VDQKkMMe+dVzDKLWs5n87T+PkQ==} peerDependencies: react: ^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc react-dom: ^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc + next-themes@0.4.6: + resolution: {integrity: sha512-pZvgD5L0IEvX5/9GWyHMf3m8BKiVQwsCMHfoFosXtXBMnaS0ZnIJ9ST4b4NqLVKDEm8QBxoNNGNaBv2JNF6XNA==} + peerDependencies: + react: ^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc + react-dom: ^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc + next@15.2.3: resolution: {integrity: sha512-x6eDkZxk2rPpu46E1ZVUWIBhYCLszmUY6fvHBFcbzJ9dD+qRX6vcHusaqqDlnY+VngKzKbAiG2iRCkPbmi8f7w==} engines: {node: ^18.18.0 || ^19.8.0 || >= 20.0.0} @@ -2892,6 +3636,12 @@ packages: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} + oniguruma-parser@0.11.2: + resolution: {integrity: sha512-F7Ld4oDZJCI5/wCZ8AOffQbqjSzIRpKH7I/iuSs1SkhZeCj0wS6PMZ4W6VA16TWHrAo0Y9bBKEJOe7tvwcTXnw==} + + oniguruma-to-es@4.2.0: + resolution: {integrity: sha512-MDPs6KSOLS0tKQ7joqg44dRIRZUyotfTy0r+7oEEs6VwWWP0+E2PPDYWMFN0aqOjRyWHBYq7RfKw9GQk2S2z5g==} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -2975,6 +3725,10 @@ packages: resolution: {integrity: sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==} engines: {node: '>=4'} + postcss-selector-parser@7.1.0: + resolution: {integrity: sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==} + engines: {node: '>=4'} + postcss@8.4.31: resolution: {integrity: sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==} engines: {node: ^10 || ^12 || >=14} @@ -3060,6 +3814,12 @@ packages: '@types/react': '>=18' react: '>=18' + react-medium-image-zoom@5.2.14: + resolution: {integrity: sha512-nfTVYcAUnBzXQpPDcZL+cG/e6UceYUIG+zDcnemL7jtAqbJjVVkA85RgneGtJeni12dTyiRPZVM6Szkmwd/o8w==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-remove-scroll-bar@2.3.8: resolution: {integrity: sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==} engines: {node: '>=10'} @@ -3124,6 +3884,22 @@ packages: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} + readdirp@4.1.2: + resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} + engines: {node: '>= 14.18.0'} + + recma-build-jsx@1.0.0: + resolution: {integrity: sha512-8GtdyqaBcDfva+GUKDr3nev3VpKAhup1+RvkMvUxURHpW7QyIvk9F5wz7Vzo06CEMSilw6uArgRqhpiUcWp8ew==} + + recma-jsx@1.0.0: + resolution: {integrity: sha512-5vwkv65qWwYxg+Atz95acp8DMu1JDSqdGkA2Of1j6rCreyFUE/gp15fC8MnGEuG1W68UKjM6x6+YTWIh7hZM/Q==} + + recma-parse@1.0.0: + resolution: {integrity: sha512-OYLsIGBB5Y5wjnSnQW6t3Xg7q3fQ7FWbw/vcXtORTnyaSFscOtABg+7Pnz6YZ6c27fG1/aN8CjfwoUEUIdwqWQ==} + + recma-stringify@1.0.0: + resolution: {integrity: sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g==} + reflect.getprototypeof@1.0.10: resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} engines: {node: '>= 0.4'} @@ -3131,6 +3907,15 @@ packages: regenerator-runtime@0.14.1: resolution: {integrity: sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==} + regex-recursion@6.0.2: + resolution: {integrity: sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==} + + regex-utilities@2.3.0: + resolution: {integrity: sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==} + + regex@6.0.1: + resolution: {integrity: sha512-uorlqlzAKjKQZ5P+kTJr3eeJGSVroLKoHmquUj4zHWuR+hEyNqlXsSKlYYF5F4NI6nl7tWCs0apKJ0lmfsXAPA==} + regexp.prototype.flags@1.5.4: resolution: {integrity: sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==} engines: {node: '>= 0.4'} @@ -3138,12 +3923,18 @@ packages: rehype-raw@7.0.0: resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==} + rehype-recma@1.0.0: + resolution: {integrity: sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==} + rehype-sanitize@6.0.0: resolution: {integrity: sha512-CsnhKNsyI8Tub6L4sm5ZFsme4puGfc6pYylvXo1AeqaGbjOYyzNv3qZPwvs0oMJ39eryyeOdmxwUIo94IpEhqg==} remark-gfm@4.0.1: resolution: {integrity: sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==} + remark-mdx@3.1.0: + resolution: {integrity: sha512-Ngl/H3YXyBV9RcRNdlYsZujAmhsxwzxpDzpDEhFBVAGthS4GDgnctpDjgFl/ULx5UEDzqtW1cyBSNKqYYrqLBA==} + remark-parse@11.0.0: resolution: {integrity: sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==} @@ -3153,6 +3944,9 @@ packages: remark-stringify@11.0.0: resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==} + remark@15.0.1: + resolution: {integrity: sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A==} + resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -3200,6 +3994,13 @@ packages: scheduler@0.25.0: resolution: {integrity: sha512-xFVuu11jh+xcO7JOAGJNOXld8/TcEHK/4CituBUeUb5hqxJLj9YuemAEuvm9gQ/+pgXYfbQuqAkiYu+u7YEsNA==} + scroll-into-view-if-needed@3.1.0: + resolution: {integrity: sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ==} + + section-matter@1.0.0: + resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} + engines: {node: '>=4'} + secure-json-parse@2.7.0: resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==} @@ -3239,6 +4040,9 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + shiki@3.3.0: + resolution: {integrity: sha512-j0Z1tG5vlOFGW8JVj0Cpuatzvshes7VJy5ncDmmMaYcmnGW0Js1N81TOW98ivTFNZfKRn9uwEg/aIm638o368g==} + side-channel-list@1.0.0: resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} engines: {node: '>= 0.4'} @@ -3279,6 +4083,10 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + source-map@0.7.4: + resolution: {integrity: sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==} + engines: {node: '>= 8'} + source-map@0.8.0-beta.0: resolution: {integrity: sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==} engines: {node: '>= 8'} @@ -3286,6 +4094,9 @@ packages: space-separated-tokens@2.0.2: resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + stable-hash@0.0.4: resolution: {integrity: sha512-LjdcbuBeLcdETCrPn9i8AYAZ1eCtu4ECAWtP7UleOiZ9LzVxRzzUZEoZ8zB24nhkQnDWyET0I+3sWokSDS3E7g==} @@ -3335,6 +4146,10 @@ packages: resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} engines: {node: '>=12'} + strip-bom-string@1.0.0: + resolution: {integrity: sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==} + engines: {node: '>=0.10.0'} + strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -3387,6 +4202,9 @@ packages: tailwind-merge@3.0.2: resolution: {integrity: sha512-l7z+OYZ7mu3DTqrL88RiKrKIqO3NcpEO8V/Od04bNpvk0kiIFndGEoqfuzvj4yuhRkHKjRkII2z+KS2HfPcSxw==} + tailwind-merge@3.2.0: + resolution: {integrity: sha512-FQT/OVqCD+7edmmJpsgCsY820RTD5AkBryuG5IUqR5YQZSdj5xlH5nLgH7YPths7WsLPSpSBNneJdM8aS8aeFA==} + tailwindcss-animate@1.0.7: resolution: {integrity: sha512-bl6mpH3T7I3UFxuvDEXLxy/VuFxBk5bbzplh7tXI68mwMokNYd1t9qPBHlnyTwfa4JGC4zP516I1hYYtQ/vspA==} peerDependencies: @@ -3510,6 +4328,9 @@ packages: unist-util-is@6.0.0: resolution: {integrity: sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==} + unist-util-position-from-estree@2.0.0: + resolution: {integrity: sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==} + unist-util-position@5.0.0: resolution: {integrity: sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==} @@ -3653,6 +4474,9 @@ packages: zod@3.24.2: resolution: {integrity: sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==} + zod@3.24.3: + resolution: {integrity: sha512-HhY1oqzWCQWuUqvBFnsyrtZRhyPeR7SUGv+C4+MsisMuVfSPx8HpwWqH8tRahSlt6M3PiFAcoeFhZAqIXTxoSg==} + zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} @@ -3700,72 +4524,147 @@ snapshots: tslib: 2.8.1 optional: true + '@esbuild/aix-ppc64@0.25.2': + optional: true + '@esbuild/android-arm64@0.17.19': optional: true + '@esbuild/android-arm64@0.25.2': + optional: true + '@esbuild/android-arm@0.17.19': optional: true + '@esbuild/android-arm@0.25.2': + optional: true + '@esbuild/android-x64@0.17.19': optional: true + '@esbuild/android-x64@0.25.2': + optional: true + '@esbuild/darwin-arm64@0.17.19': optional: true + '@esbuild/darwin-arm64@0.25.2': + optional: true + '@esbuild/darwin-x64@0.17.19': optional: true + '@esbuild/darwin-x64@0.25.2': + optional: true + '@esbuild/freebsd-arm64@0.17.19': optional: true + '@esbuild/freebsd-arm64@0.25.2': + optional: true + '@esbuild/freebsd-x64@0.17.19': optional: true + '@esbuild/freebsd-x64@0.25.2': + optional: true + '@esbuild/linux-arm64@0.17.19': optional: true + '@esbuild/linux-arm64@0.25.2': + optional: true + '@esbuild/linux-arm@0.17.19': optional: true + '@esbuild/linux-arm@0.25.2': + optional: true + '@esbuild/linux-ia32@0.17.19': optional: true + '@esbuild/linux-ia32@0.25.2': + optional: true + '@esbuild/linux-loong64@0.17.19': optional: true + '@esbuild/linux-loong64@0.25.2': + optional: true + '@esbuild/linux-mips64el@0.17.19': optional: true + '@esbuild/linux-mips64el@0.25.2': + optional: true + '@esbuild/linux-ppc64@0.17.19': optional: true + '@esbuild/linux-ppc64@0.25.2': + optional: true + '@esbuild/linux-riscv64@0.17.19': optional: true + '@esbuild/linux-riscv64@0.25.2': + optional: true + '@esbuild/linux-s390x@0.17.19': optional: true + '@esbuild/linux-s390x@0.25.2': + optional: true + '@esbuild/linux-x64@0.17.19': optional: true + '@esbuild/linux-x64@0.25.2': + optional: true + + '@esbuild/netbsd-arm64@0.25.2': + optional: true + '@esbuild/netbsd-x64@0.17.19': optional: true + '@esbuild/netbsd-x64@0.25.2': + optional: true + + '@esbuild/openbsd-arm64@0.25.2': + optional: true + '@esbuild/openbsd-x64@0.17.19': optional: true + '@esbuild/openbsd-x64@0.25.2': + optional: true + '@esbuild/sunos-x64@0.17.19': optional: true + '@esbuild/sunos-x64@0.25.2': + optional: true + '@esbuild/win32-arm64@0.17.19': optional: true + '@esbuild/win32-arm64@0.25.2': + optional: true + '@esbuild/win32-ia32@0.17.19': optional: true + '@esbuild/win32-ia32@0.25.2': + optional: true + '@esbuild/win32-x64@0.17.19': optional: true + '@esbuild/win32-x64@0.25.2': + optional: true + '@eslint-community/eslint-utils@4.4.1(eslint@9.21.0(jiti@2.4.2))': dependencies: eslint: 9.21.0(jiti@2.4.2) @@ -3825,6 +4724,10 @@ snapshots: '@floating-ui/utils@0.2.9': {} + '@formatjs/intl-localematcher@0.6.1': + dependencies: + tslib: 2.8.1 + '@hookform/resolvers@4.1.3(react-hook-form@7.54.2(react@19.0.0))': dependencies: '@standard-schema/utils': 0.3.0 @@ -3944,6 +4847,36 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.0 + '@mdx-js/mdx@3.1.0(acorn@8.14.0)': + dependencies: + '@types/estree': 1.0.6 + '@types/estree-jsx': 1.0.5 + '@types/hast': 3.0.4 + '@types/mdx': 2.0.13 + collapse-white-space: 2.1.0 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + estree-util-scope: 1.0.0 + estree-walker: 3.0.3 + hast-util-to-jsx-runtime: 2.3.6 + markdown-extensions: 2.0.0 + recma-build-jsx: 1.0.0 + recma-jsx: 1.0.0(acorn@8.14.0) + recma-stringify: 1.0.0 + rehype-recma: 1.0.0 + remark-mdx: 3.1.0 + remark-parse: 11.0.0 + remark-rehype: 11.1.1 + source-map: 0.7.4 + unified: 11.0.5 + unist-util-position-from-estree: 2.0.0 + unist-util-stringify-position: 4.0.0 + unist-util-visit: 5.0.0 + vfile: 6.0.3 + transitivePeerDependencies: + - acorn + - supports-color + '@next/env@15.2.3': {} '@next/eslint-plugin-next@15.2.0': @@ -3990,11 +4923,15 @@ snapshots: '@opentelemetry/api@1.9.0': {} + '@orama/orama@3.1.6': {} + '@pkgjs/parseargs@0.11.0': optional: true '@radix-ui/number@1.1.0': {} + '@radix-ui/number@1.1.1': {} + '@radix-ui/primitive@1.0.0': dependencies: '@babel/runtime': 7.26.9 @@ -4005,6 +4942,8 @@ snapshots: '@radix-ui/primitive@1.1.1': {} + '@radix-ui/primitive@1.1.2': {} + '@radix-ui/react-accordion@1.2.3(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/primitive': 1.1.1 @@ -4022,6 +4961,23 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-accordion@1.2.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/primitive': 1.1.2 + '@radix-ui/react-collapsible': 1.1.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-collection': 1.1.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-direction': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-id': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-alert-dialog@1.1.6(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/primitive': 1.1.1 @@ -4045,6 +5001,15 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-arrow@1.1.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-avatar@1.1.3(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/react-context': 1.1.1(@types/react@19.0.10)(react@19.0.0) @@ -4089,6 +5054,22 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-collapsible@1.1.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/primitive': 1.1.2 + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-id': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-presence': 1.1.3(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-collection@1.1.2(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/react-compose-refs': 1.1.1(@types/react@19.0.10)(react@19.0.0) @@ -4101,6 +5082,18 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-collection@1.1.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-slot': 1.2.0(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-compose-refs@1.0.0(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4119,6 +5112,12 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-compose-refs@1.1.2(@types/react@19.0.10)(react@19.0.0)': + dependencies: + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-context@1.0.0(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4137,6 +5136,12 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-context@1.1.2(@types/react@19.0.10)(react@19.0.0)': + dependencies: + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-dialog@1.0.0(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4182,6 +5187,28 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-dialog@1.1.10(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/primitive': 1.1.2 + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-dismissable-layer': 1.1.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-focus-guards': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-focus-scope': 1.1.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-id': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-portal': 1.1.6(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-presence': 1.1.3(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-slot': 1.2.0(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.0.10)(react@19.0.0) + aria-hidden: 1.2.4 + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + react-remove-scroll: 2.6.3(@types/react@19.0.10)(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-dialog@1.1.6(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/primitive': 1.1.1 @@ -4210,6 +5237,12 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-direction@1.1.1(@types/react@19.0.10)(react@19.0.0)': + dependencies: + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-dismissable-layer@1.0.0(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4248,6 +5281,19 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-dismissable-layer@1.1.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/primitive': 1.1.2 + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-escape-keydown': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-dropdown-menu@2.1.6(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/primitive': 1.1.1 @@ -4281,6 +5327,12 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-focus-guards@1.1.2(@types/react@19.0.10)(react@19.0.0)': + dependencies: + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-focus-scope@1.0.0(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4313,6 +5365,17 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-focus-scope@1.1.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-id@1.0.0(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4334,6 +5397,13 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-id@1.1.1(@types/react@19.0.10)(react@19.0.0)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-label@2.1.2(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/react-primitive': 2.0.2(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) @@ -4369,6 +5439,51 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-navigation-menu@1.2.9(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/primitive': 1.1.2 + '@radix-ui/react-collection': 1.1.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-direction': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-dismissable-layer': 1.1.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-id': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-presence': 1.1.3(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-previous': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-visually-hidden': 1.2.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + + '@radix-ui/react-popover@1.1.10(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/primitive': 1.1.2 + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-dismissable-layer': 1.1.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-focus-guards': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-focus-scope': 1.1.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-id': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-popper': 1.2.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-portal': 1.1.6(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-presence': 1.1.3(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-slot': 1.2.0(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.0.10)(react@19.0.0) + aria-hidden: 1.2.4 + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + react-remove-scroll: 2.6.3(@types/react@19.0.10)(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-popover@1.1.6(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/primitive': 1.1.1 @@ -4410,6 +5525,24 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-popper@1.2.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@floating-ui/react-dom': 2.1.2(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-arrow': 1.1.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-rect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-size': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/rect': 1.1.1 + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-portal@1.0.0(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4437,6 +5570,16 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-portal@1.1.6(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-presence@1.0.0(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4466,6 +5609,16 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-presence@1.1.3(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-primitive@1.0.0(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4492,6 +5645,15 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-primitive@2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/react-slot': 1.2.0(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-roving-focus@1.1.2(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/primitive': 1.1.1 @@ -4509,6 +5671,40 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-roving-focus@1.1.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/primitive': 1.1.2 + '@radix-ui/react-collection': 1.1.4(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-direction': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-id': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + + '@radix-ui/react-scroll-area@1.2.5(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/number': 1.1.1 + '@radix-ui/primitive': 1.1.2 + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-context': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-direction': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-presence': 1.1.3(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-select@2.1.6(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/number': 1.1.0 @@ -4568,6 +5764,13 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-slot@1.2.0(@types/react@19.0.10)(react@19.0.0)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-tabs@1.1.3(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/primitive': 1.1.1 @@ -4584,6 +5787,22 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-tabs@1.1.8(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/primitive': 1.1.2 + '@radix-ui/react-context': 1.1.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-direction': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-id': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-presence': 1.1.3(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-roving-focus': 1.1.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-tooltip@1.1.8(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/primitive': 1.1.1 @@ -4622,6 +5841,12 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-use-callback-ref@1.1.1(@types/react@19.0.10)(react@19.0.0)': + dependencies: + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-use-controllable-state@1.0.0(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4643,6 +5868,21 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-use-controllable-state@1.2.2(@types/react@19.0.10)(react@19.0.0)': + dependencies: + '@radix-ui/react-use-effect-event': 0.0.2(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + + '@radix-ui/react-use-effect-event@0.0.2(@types/react@19.0.10)(react@19.0.0)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-use-escape-keydown@1.0.0(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4664,6 +5904,13 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-use-escape-keydown@1.1.1(@types/react@19.0.10)(react@19.0.0)': + dependencies: + '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-use-layout-effect@1.0.0(react@19.0.0)': dependencies: '@babel/runtime': 7.26.9 @@ -4682,12 +5929,24 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-use-layout-effect@1.1.1(@types/react@19.0.10)(react@19.0.0)': + dependencies: + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-use-previous@1.1.0(@types/react@19.0.10)(react@19.0.0)': dependencies: react: 19.0.0 optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-use-previous@1.1.1(@types/react@19.0.10)(react@19.0.0)': + dependencies: + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-use-rect@1.1.0(@types/react@19.0.10)(react@19.0.0)': dependencies: '@radix-ui/rect': 1.1.0 @@ -4695,6 +5954,13 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-use-rect@1.1.1(@types/react@19.0.10)(react@19.0.0)': + dependencies: + '@radix-ui/rect': 1.1.1 + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-use-size@1.1.0(@types/react@19.0.10)(react@19.0.0)': dependencies: '@radix-ui/react-use-layout-effect': 1.1.0(@types/react@19.0.10)(react@19.0.0) @@ -4702,6 +5968,13 @@ snapshots: optionalDependencies: '@types/react': 19.0.10 + '@radix-ui/react-use-size@1.1.1(@types/react@19.0.10)(react@19.0.0)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.0.10)(react@19.0.0) + react: 19.0.0 + optionalDependencies: + '@types/react': 19.0.10 + '@radix-ui/react-visually-hidden@1.1.2(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': dependencies: '@radix-ui/react-primitive': 2.0.2(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) @@ -4711,12 +5984,72 @@ snapshots: '@types/react': 19.0.10 '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/react-visually-hidden@1.2.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)': + dependencies: + '@radix-ui/react-primitive': 2.1.0(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + optionalDependencies: + '@types/react': 19.0.10 + '@types/react-dom': 19.0.4(@types/react@19.0.10) + '@radix-ui/rect@1.1.0': {} + '@radix-ui/rect@1.1.1': {} + '@rtsao/scc@1.1.0': {} '@rushstack/eslint-patch@1.10.5': {} + '@shikijs/core@3.3.0': + dependencies: + '@shikijs/types': 3.3.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.4 + hast-util-to-html: 9.0.5 + + '@shikijs/engine-javascript@3.3.0': + dependencies: + '@shikijs/types': 3.3.0 + '@shikijs/vscode-textmate': 10.0.2 + oniguruma-to-es: 4.2.0 + + '@shikijs/engine-oniguruma@3.3.0': + dependencies: + '@shikijs/types': 3.3.0 + '@shikijs/vscode-textmate': 10.0.2 + + '@shikijs/langs@3.3.0': + dependencies: + '@shikijs/types': 3.3.0 + + '@shikijs/rehype@3.3.0': + dependencies: + '@shikijs/types': 3.3.0 + '@types/hast': 3.0.4 + hast-util-to-string: 3.0.1 + shiki: 3.3.0 + unified: 11.0.5 + unist-util-visit: 5.0.0 + + '@shikijs/themes@3.3.0': + dependencies: + '@shikijs/types': 3.3.0 + + '@shikijs/transformers@3.3.0': + dependencies: + '@shikijs/core': 3.3.0 + '@shikijs/types': 3.3.0 + + '@shikijs/types@3.3.0': + dependencies: + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.4 + + '@shikijs/vscode-textmate@10.0.2': {} + + '@standard-schema/spec@1.0.0': {} + '@standard-schema/utils@0.3.0': {} '@swc/counter@0.1.3': {} @@ -4834,6 +6167,8 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/mdx@2.0.13': {} + '@types/ms@2.1.0': {} '@types/node@20.17.22': @@ -4973,6 +6308,10 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.1 + argparse@1.0.10: + dependencies: + sprintf-js: 1.0.3 + argparse@2.0.1: {} aria-hidden@1.2.4: @@ -5053,6 +6392,8 @@ snapshots: ast-types-flow@0.0.8: {} + astring@1.9.0: {} + async-function@1.0.0: {} attr-accept@2.2.5: {} @@ -5147,6 +6488,10 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + chokidar@4.0.3: + dependencies: + readdirp: 4.1.2 + class-variance-authority@0.7.1: dependencies: clsx: 2.1.1 @@ -5163,6 +6508,8 @@ snapshots: transitivePeerDependencies: - '@types/react' + collapse-white-space@2.1.0: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -5185,6 +6532,8 @@ snapshots: commander@4.1.1: {} + compute-scroll-into-view@3.1.1: {} + concat-map@0.0.1: {} cross-env@7.0.3: @@ -5297,7 +6646,7 @@ snapshots: react: 19.0.0 react-dom: 19.0.0(react@19.0.0) react-easy-sort: 1.6.0(react-dom@19.0.0(react@19.0.0))(react@19.0.0) - tailwind-merge: 3.0.2 + tailwind-merge: 3.2.0 tsup: 6.7.0(postcss@8.5.3)(typescript@5.8.2) transitivePeerDependencies: - '@swc/core' @@ -5308,6 +6657,8 @@ snapshots: - ts-node - typescript + emoji-regex-xs@1.0.0: {} + emoji-regex@8.0.0: {} emoji-regex@9.2.2: {} @@ -5417,6 +6768,20 @@ snapshots: is-date-object: 1.1.0 is-symbol: 1.1.1 + esast-util-from-estree@2.0.0: + dependencies: + '@types/estree-jsx': 1.0.5 + devlop: 1.1.0 + estree-util-visit: 2.0.0 + unist-util-position-from-estree: 2.0.0 + + esast-util-from-js@2.0.1: + dependencies: + '@types/estree-jsx': 1.0.5 + acorn: 8.14.0 + esast-util-from-estree: 2.0.0 + vfile-message: 4.0.2 + esbuild@0.17.19: optionalDependencies: '@esbuild/android-arm': 0.17.19 @@ -5442,6 +6807,34 @@ snapshots: '@esbuild/win32-ia32': 0.17.19 '@esbuild/win32-x64': 0.17.19 + esbuild@0.25.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.2 + '@esbuild/android-arm': 0.25.2 + '@esbuild/android-arm64': 0.25.2 + '@esbuild/android-x64': 0.25.2 + '@esbuild/darwin-arm64': 0.25.2 + '@esbuild/darwin-x64': 0.25.2 + '@esbuild/freebsd-arm64': 0.25.2 + '@esbuild/freebsd-x64': 0.25.2 + '@esbuild/linux-arm': 0.25.2 + '@esbuild/linux-arm64': 0.25.2 + '@esbuild/linux-ia32': 0.25.2 + '@esbuild/linux-loong64': 0.25.2 + '@esbuild/linux-mips64el': 0.25.2 + '@esbuild/linux-ppc64': 0.25.2 + '@esbuild/linux-riscv64': 0.25.2 + '@esbuild/linux-s390x': 0.25.2 + '@esbuild/linux-x64': 0.25.2 + '@esbuild/netbsd-arm64': 0.25.2 + '@esbuild/netbsd-x64': 0.25.2 + '@esbuild/openbsd-arm64': 0.25.2 + '@esbuild/openbsd-x64': 0.25.2 + '@esbuild/sunos-x64': 0.25.2 + '@esbuild/win32-arm64': 0.25.2 + '@esbuild/win32-ia32': 0.25.2 + '@esbuild/win32-x64': 0.25.2 + escape-string-regexp@4.0.0: {} escape-string-regexp@5.0.0: {} @@ -5630,6 +7023,8 @@ snapshots: acorn-jsx: 5.3.2(acorn@8.14.0) eslint-visitor-keys: 4.2.0 + esprima@4.0.1: {} + esquery@1.6.0: dependencies: estraverse: 5.3.0 @@ -5640,8 +7035,43 @@ snapshots: estraverse@5.3.0: {} + estree-util-attach-comments@3.0.0: + dependencies: + '@types/estree': 1.0.6 + + estree-util-build-jsx@3.0.1: + dependencies: + '@types/estree-jsx': 1.0.5 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + estree-walker: 3.0.3 + estree-util-is-identifier-name@3.0.0: {} + estree-util-scope@1.0.0: + dependencies: + '@types/estree': 1.0.6 + devlop: 1.1.0 + + estree-util-to-js@2.0.0: + dependencies: + '@types/estree-jsx': 1.0.5 + astring: 1.9.0 + source-map: 0.7.4 + + estree-util-value-to-estree@3.3.3: + dependencies: + '@types/estree': 1.0.6 + + estree-util-visit@2.0.0: + dependencies: + '@types/estree-jsx': 1.0.5 + '@types/unist': 3.0.3 + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.6 + esutils@2.0.3: {} eventsource-parser@3.0.0: {} @@ -5658,6 +7088,10 @@ snapshots: signal-exit: 3.0.7 strip-final-newline: 2.0.0 + extend-shallow@2.0.1: + dependencies: + is-extendable: 0.1.1 + extend@3.0.2: {} fast-deep-equal@3.1.3: {} @@ -5763,6 +7197,82 @@ snapshots: fsevents@2.3.3: optional: true + fumadocs-core@15.2.9(@types/react@19.0.10)(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0))(react-dom@19.0.0(react@19.0.0))(react@19.0.0): + dependencies: + '@formatjs/intl-localematcher': 0.6.1 + '@orama/orama': 3.1.6 + '@shikijs/rehype': 3.3.0 + '@shikijs/transformers': 3.3.0 + github-slugger: 2.0.0 + hast-util-to-estree: 3.1.3 + hast-util-to-jsx-runtime: 2.3.6 + image-size: 2.0.2 + negotiator: 1.0.0 + react-remove-scroll: 2.6.3(@types/react@19.0.10)(react@19.0.0) + remark: 15.0.1 + remark-gfm: 4.0.1 + scroll-into-view-if-needed: 3.1.0 + shiki: 3.3.0 + unist-util-visit: 5.0.0 + optionalDependencies: + next: 15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + transitivePeerDependencies: + - '@types/react' + - supports-color + + fumadocs-mdx@11.6.1(acorn@8.14.0)(fumadocs-core@15.2.9(@types/react@19.0.10)(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0))(react-dom@19.0.0(react@19.0.0))(react@19.0.0))(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)): + dependencies: + '@mdx-js/mdx': 3.1.0(acorn@8.14.0) + '@standard-schema/spec': 1.0.0 + chokidar: 4.0.3 + cross-spawn: 7.0.6 + esbuild: 0.25.2 + estree-util-value-to-estree: 3.3.3 + fast-glob: 3.3.3 + fumadocs-core: 15.2.9(@types/react@19.0.10)(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0))(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + gray-matter: 4.0.3 + lru-cache: 11.1.0 + next: 15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + picocolors: 1.1.1 + unist-util-visit: 5.0.0 + zod: 3.24.3 + transitivePeerDependencies: + - acorn + - supports-color + + fumadocs-ui@15.2.9(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0))(react-dom@19.0.0(react@19.0.0))(react@19.0.0)(tailwindcss@4.0.9): + dependencies: + '@radix-ui/react-accordion': 1.2.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-collapsible': 1.1.7(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-dialog': 1.1.10(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-direction': 1.1.1(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-navigation-menu': 1.2.9(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-popover': 1.1.10(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-scroll-area': 1.2.5(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@radix-ui/react-slot': 1.2.0(@types/react@19.0.10)(react@19.0.0) + '@radix-ui/react-tabs': 1.1.8(@types/react-dom@19.0.4(@types/react@19.0.10))(@types/react@19.0.10)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + class-variance-authority: 0.7.1 + fumadocs-core: 15.2.9(@types/react@19.0.10)(next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0))(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + lodash.merge: 4.6.2 + lucide-react: 0.488.0(react@19.0.0) + next: 15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + next-themes: 0.4.6(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + postcss-selector-parser: 7.1.0 + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + react-medium-image-zoom: 5.2.14(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + tailwind-merge: 3.2.0 + optionalDependencies: + tailwindcss: 4.0.9 + transitivePeerDependencies: + - '@oramacloud/client' + - '@types/react' + - '@types/react-dom' + - algoliasearch + - supports-color + function-bind@1.1.2: {} function.prototype.name@1.1.8: @@ -5812,6 +7322,8 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 + github-slugger@2.0.0: {} + glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -5851,6 +7363,13 @@ snapshots: graphemer@1.4.0: {} + gray-matter@4.0.3: + dependencies: + js-yaml: 3.14.1 + kind-of: 6.0.3 + section-matter: 1.0.0 + strip-bom-string: 1.0.0 + has-bigints@1.1.0: {} has-flag@4.0.0: {} @@ -5910,6 +7429,41 @@ snapshots: '@ungap/structured-clone': 1.3.0 unist-util-position: 5.0.0 + hast-util-to-estree@3.1.3: + dependencies: + '@types/estree': 1.0.6 + '@types/estree-jsx': 1.0.5 + '@types/hast': 3.0.4 + comma-separated-tokens: 2.0.3 + devlop: 1.1.0 + estree-util-attach-comments: 3.0.0 + estree-util-is-identifier-name: 3.0.0 + hast-util-whitespace: 3.0.0 + mdast-util-mdx-expression: 2.0.1 + mdast-util-mdx-jsx: 3.2.0 + mdast-util-mdxjs-esm: 2.0.1 + property-information: 7.0.0 + space-separated-tokens: 2.0.2 + style-to-js: 1.1.16 + unist-util-position: 5.0.0 + zwitch: 2.0.4 + transitivePeerDependencies: + - supports-color + + hast-util-to-html@9.0.5: + dependencies: + '@types/hast': 3.0.4 + '@types/unist': 3.0.3 + ccount: 2.0.1 + comma-separated-tokens: 2.0.3 + hast-util-whitespace: 3.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.2.0 + property-information: 7.0.0 + space-separated-tokens: 2.0.2 + stringify-entities: 4.0.4 + zwitch: 2.0.4 + hast-util-to-jsx-runtime@2.3.6: dependencies: '@types/estree': 1.0.6 @@ -5940,6 +7494,10 @@ snapshots: web-namespaces: 2.0.1 zwitch: 2.0.4 + hast-util-to-string@3.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-whitespace@3.0.0: dependencies: '@types/hast': 3.0.4 @@ -5960,6 +7518,8 @@ snapshots: ignore@5.3.2: {} + image-size@2.0.2: {} + import-fresh@3.3.1: dependencies: parent-module: 1.0.1 @@ -6035,6 +7595,8 @@ snapshots: is-decimal@2.0.1: {} + is-extendable@0.1.1: {} + is-extglob@2.1.1: {} is-finalizationregistry@1.1.1: @@ -6133,6 +7695,11 @@ snapshots: js-tokens@4.0.0: {} + js-yaml@3.14.1: + dependencies: + argparse: 1.0.10 + esprima: 4.0.1 + js-yaml@4.1.0: dependencies: argparse: 2.0.1 @@ -6166,6 +7733,8 @@ snapshots: dependencies: json-buffer: 3.0.1 + kind-of@6.0.3: {} + language-subtag-registry@0.3.23: {} language-tags@1.0.9: @@ -6252,10 +7821,18 @@ snapshots: lru-cache@10.4.3: {} + lru-cache@11.1.0: {} + lucide-react@0.477.0(react@19.0.0): dependencies: react: 19.0.0 + lucide-react@0.488.0(react@19.0.0): + dependencies: + react: 19.0.0 + + markdown-extensions@2.0.0: {} + markdown-table@3.0.4: {} math-intrinsics@1.1.0: {} @@ -6369,6 +7946,16 @@ snapshots: transitivePeerDependencies: - supports-color + mdast-util-mdx@3.0.0: + dependencies: + mdast-util-from-markdown: 2.0.2 + mdast-util-mdx-expression: 2.0.1 + mdast-util-mdx-jsx: 3.2.0 + mdast-util-mdxjs-esm: 2.0.1 + mdast-util-to-markdown: 2.1.2 + transitivePeerDependencies: + - supports-color + mdast-util-mdxjs-esm@2.0.1: dependencies: '@types/estree-jsx': 1.0.5 @@ -6494,6 +8081,57 @@ snapshots: micromark-util-combine-extensions: 2.0.1 micromark-util-types: 2.0.2 + micromark-extension-mdx-expression@3.0.1: + dependencies: + '@types/estree': 1.0.6 + devlop: 1.1.0 + micromark-factory-mdx-expression: 2.0.3 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-events-to-acorn: 2.0.3 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + + micromark-extension-mdx-jsx@3.0.2: + dependencies: + '@types/estree': 1.0.6 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + micromark-factory-mdx-expression: 2.0.3 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-events-to-acorn: 2.0.3 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + vfile-message: 4.0.2 + + micromark-extension-mdx-md@2.0.0: + dependencies: + micromark-util-types: 2.0.2 + + micromark-extension-mdxjs-esm@3.0.0: + dependencies: + '@types/estree': 1.0.6 + devlop: 1.1.0 + micromark-core-commonmark: 2.0.3 + micromark-util-character: 2.1.1 + micromark-util-events-to-acorn: 2.0.3 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + unist-util-position-from-estree: 2.0.0 + vfile-message: 4.0.2 + + micromark-extension-mdxjs@3.0.0: + dependencies: + acorn: 8.14.0 + acorn-jsx: 5.3.2(acorn@8.14.0) + micromark-extension-mdx-expression: 3.0.1 + micromark-extension-mdx-jsx: 3.0.2 + micromark-extension-mdx-md: 2.0.0 + micromark-extension-mdxjs-esm: 3.0.0 + micromark-util-combine-extensions: 2.0.1 + micromark-util-types: 2.0.2 + micromark-factory-destination@2.0.1: dependencies: micromark-util-character: 2.1.1 @@ -6507,6 +8145,18 @@ snapshots: micromark-util-symbol: 2.0.1 micromark-util-types: 2.0.2 + micromark-factory-mdx-expression@2.0.3: + dependencies: + '@types/estree': 1.0.6 + devlop: 1.1.0 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-events-to-acorn: 2.0.3 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + unist-util-position-from-estree: 2.0.0 + vfile-message: 4.0.2 + micromark-factory-space@2.0.1: dependencies: micromark-util-character: 2.1.1 @@ -6559,6 +8209,16 @@ snapshots: micromark-util-encode@2.0.1: {} + micromark-util-events-to-acorn@2.0.3: + dependencies: + '@types/estree': 1.0.6 + '@types/unist': 3.0.3 + devlop: 1.1.0 + estree-util-visit: 2.0.0 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + vfile-message: 4.0.2 + micromark-util-html-tag-name@2.0.1: {} micromark-util-normalize-identifier@2.0.1: @@ -6647,11 +8307,18 @@ snapshots: natural-compare@1.4.0: {} + negotiator@1.0.0: {} + next-themes@0.4.4(react-dom@19.0.0(react@19.0.0))(react@19.0.0): dependencies: react: 19.0.0 react-dom: 19.0.0(react@19.0.0) + next-themes@0.4.6(react-dom@19.0.0(react@19.0.0))(react@19.0.0): + dependencies: + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + next@15.2.3(@opentelemetry/api@1.9.0)(react-dom@19.0.0(react@19.0.0))(react@19.0.0): dependencies: '@next/env': 15.2.3 @@ -6733,6 +8400,15 @@ snapshots: dependencies: mimic-fn: 2.1.0 + oniguruma-parser@0.11.2: {} + + oniguruma-to-es@4.2.0: + dependencies: + emoji-regex-xs: 1.0.0 + oniguruma-parser: 0.11.2 + regex: 6.0.1 + regex-recursion: 6.0.2 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -6811,6 +8487,11 @@ snapshots: cssesc: 3.0.0 util-deprecate: 1.0.2 + postcss-selector-parser@7.1.0: + dependencies: + cssesc: 3.0.0 + util-deprecate: 1.0.2 + postcss@8.4.31: dependencies: nanoid: 3.3.11 @@ -6913,6 +8594,11 @@ snapshots: transitivePeerDependencies: - supports-color + react-medium-image-zoom@5.2.14(react-dom@19.0.0(react@19.0.0))(react@19.0.0): + dependencies: + react: 19.0.0 + react-dom: 19.0.0(react@19.0.0) + react-remove-scroll-bar@2.3.8(@types/react@19.0.10)(react@19.0.0): dependencies: react: 19.0.0 @@ -6977,6 +8663,38 @@ snapshots: dependencies: picomatch: 2.3.1 + readdirp@4.1.2: {} + + recma-build-jsx@1.0.0: + dependencies: + '@types/estree': 1.0.6 + estree-util-build-jsx: 3.0.1 + vfile: 6.0.3 + + recma-jsx@1.0.0(acorn@8.14.0): + dependencies: + acorn-jsx: 5.3.2(acorn@8.14.0) + estree-util-to-js: 2.0.0 + recma-parse: 1.0.0 + recma-stringify: 1.0.0 + unified: 11.0.5 + transitivePeerDependencies: + - acorn + + recma-parse@1.0.0: + dependencies: + '@types/estree': 1.0.6 + esast-util-from-js: 2.0.1 + unified: 11.0.5 + vfile: 6.0.3 + + recma-stringify@1.0.0: + dependencies: + '@types/estree': 1.0.6 + estree-util-to-js: 2.0.0 + unified: 11.0.5 + vfile: 6.0.3 + reflect.getprototypeof@1.0.10: dependencies: call-bind: 1.0.8 @@ -6990,6 +8708,16 @@ snapshots: regenerator-runtime@0.14.1: {} + regex-recursion@6.0.2: + dependencies: + regex-utilities: 2.3.0 + + regex-utilities@2.3.0: {} + + regex@6.0.1: + dependencies: + regex-utilities: 2.3.0 + regexp.prototype.flags@1.5.4: dependencies: call-bind: 1.0.8 @@ -7005,6 +8733,14 @@ snapshots: hast-util-raw: 9.1.0 vfile: 6.0.3 + rehype-recma@1.0.0: + dependencies: + '@types/estree': 1.0.6 + '@types/hast': 3.0.4 + hast-util-to-estree: 3.1.3 + transitivePeerDependencies: + - supports-color + rehype-sanitize@6.0.0: dependencies: '@types/hast': 3.0.4 @@ -7021,6 +8757,13 @@ snapshots: transitivePeerDependencies: - supports-color + remark-mdx@3.1.0: + dependencies: + mdast-util-mdx: 3.0.0 + micromark-extension-mdxjs: 3.0.0 + transitivePeerDependencies: + - supports-color + remark-parse@11.0.0: dependencies: '@types/mdast': 4.0.4 @@ -7044,6 +8787,15 @@ snapshots: mdast-util-to-markdown: 2.1.2 unified: 11.0.5 + remark@15.0.1: + dependencies: + '@types/mdast': 4.0.4 + remark-parse: 11.0.0 + remark-stringify: 11.0.0 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + resolve-from@4.0.0: {} resolve-from@5.0.0: {} @@ -7093,6 +8845,15 @@ snapshots: scheduler@0.25.0: {} + scroll-into-view-if-needed@3.1.0: + dependencies: + compute-scroll-into-view: 3.1.1 + + section-matter@1.0.0: + dependencies: + extend-shallow: 2.0.1 + kind-of: 6.0.3 + secure-json-parse@2.7.0: {} semver@6.3.1: {} @@ -7156,6 +8917,17 @@ snapshots: shebang-regex@3.0.0: {} + shiki@3.3.0: + dependencies: + '@shikijs/core': 3.3.0 + '@shikijs/engine-javascript': 3.3.0 + '@shikijs/engine-oniguruma': 3.3.0 + '@shikijs/langs': 3.3.0 + '@shikijs/themes': 3.3.0 + '@shikijs/types': 3.3.0 + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.4 + side-channel-list@1.0.0: dependencies: es-errors: 1.3.0 @@ -7202,12 +8974,16 @@ snapshots: source-map-js@1.2.1: {} + source-map@0.7.4: {} + source-map@0.8.0-beta.0: dependencies: whatwg-url: 7.1.0 space-separated-tokens@2.0.2: {} + sprintf-js@1.0.3: {} + stable-hash@0.0.4: {} streamsearch@1.1.0: {} @@ -7287,6 +9063,8 @@ snapshots: dependencies: ansi-regex: 6.1.0 + strip-bom-string@1.0.0: {} + strip-bom@3.0.0: {} strip-final-newline@2.0.0: {} @@ -7330,6 +9108,8 @@ snapshots: tailwind-merge@3.0.2: {} + tailwind-merge@3.2.0: {} + tailwindcss-animate@1.0.7(tailwindcss@4.0.9): dependencies: tailwindcss: 4.0.9 @@ -7473,6 +9253,10 @@ snapshots: dependencies: '@types/unist': 3.0.3 + unist-util-position-from-estree@2.0.0: + dependencies: + '@types/unist': 3.0.3 + unist-util-position@5.0.0: dependencies: '@types/unist': 3.0.3 @@ -7636,4 +9420,6 @@ snapshots: zod@3.24.2: {} + zod@3.24.3: {} + zwitch@2.0.4: {} diff --git a/surfsense_web/source.config.ts b/surfsense_web/source.config.ts new file mode 100644 index 000000000..250c23a07 --- /dev/null +++ b/surfsense_web/source.config.ts @@ -0,0 +1,5 @@ +import { defineDocs } from 'fumadocs-mdx/config'; + +export const docs = defineDocs({ + dir: 'content/docs', +}); \ No newline at end of file diff --git a/surfsense_web/tsconfig.json b/surfsense_web/tsconfig.json index d8b93235f..b3de9e41f 100644 --- a/surfsense_web/tsconfig.json +++ b/surfsense_web/tsconfig.json @@ -22,6 +22,6 @@ "@/*": ["./*"] } }, - "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", "next.config.mjs"], "exclude": ["node_modules"] } From 02cf7d9828801f2edc9a76c02a9d08ca197c3fc3 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 22 Apr 2025 02:37:49 -0700 Subject: [PATCH 30/31] chore: Stop tracking .source directory --- surfsense_web/.source/index.ts | 5 ----- surfsense_web/.source/source.config.mjs | 8 -------- 2 files changed, 13 deletions(-) delete mode 100644 surfsense_web/.source/index.ts delete mode 100644 surfsense_web/.source/source.config.mjs diff --git a/surfsense_web/.source/index.ts b/surfsense_web/.source/index.ts deleted file mode 100644 index 0002966e0..000000000 --- a/surfsense_web/.source/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -// @ts-nocheck -- skip type checking -import * as docs_0 from "../content/docs/index.mdx?collection=docs&hash=1745310068376" -import { _runtime } from "fumadocs-mdx" -import * as _source from "../source.config" -export const docs = _runtime.docs([{ info: {"path":"index.mdx","absolutePath":"C:/Users/91882/Documents/SurfSense/surfsense_web/content/docs/index.mdx"}, data: docs_0 }], []) \ No newline at end of file diff --git a/surfsense_web/.source/source.config.mjs b/surfsense_web/.source/source.config.mjs deleted file mode 100644 index ae67114fd..000000000 --- a/surfsense_web/.source/source.config.mjs +++ /dev/null @@ -1,8 +0,0 @@ -// source.config.ts -import { defineDocs } from "fumadocs-mdx/config"; -var docs = defineDocs({ - dir: "content/docs" -}); -export { - docs -}; From b2f47d28030f0cb57a0b494560d246eaa0867116 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 22 Apr 2025 02:40:45 -0700 Subject: [PATCH 31/31] updated .gitignore --- surfsense_web/.gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surfsense_web/.gitignore b/surfsense_web/.gitignore index e72b4d6a4..c78e9de6c 100644 --- a/surfsense_web/.gitignore +++ b/surfsense_web/.gitignore @@ -39,3 +39,6 @@ yarn-error.log* # typescript *.tsbuildinfo next-env.d.ts + +# source +/.source/