mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat(workspace): remove AI file sorting feature and related components
This commit is contained in:
Anish Sarkar
parent
72c2dd0e1b
commit
cda232101f
18 changed files with 40 additions and 1005 deletions
|
|
@ -41,7 +41,6 @@ NotebookLM is one of the best and most useful AI platforms out there, but once y
|
|||
- **No Vendor Lock-in** - Configure any LLM, image, TTS, and STT models to use.
|
||||
- **25+ External Data Sources** - Add your sources from Google Drive, OneDrive, Dropbox, Notion, and many other external services.
|
||||
- **Real-Time Multiplayer Support** - Work easily with your team members in a shared notebook.
|
||||
- **AI File Sorting** - Automatically organize your documents into a smart folder hierarchy using AI-powered categorization by source, date, and topic.
|
||||
- **AI Automations & Agents** - Run AI agents on a schedule or trigger them the moment a document lands in a folder, then write results back to Notion, Slack, Linear, and Drive. Build no-code automations just by describing them in chat.
|
||||
- **Desktop App** - Get AI assistance in any application with Quick Assist, General Assist, Screenshot Assist, and local folder sync.
|
||||
|
||||
|
|
@ -271,7 +270,6 @@ All features operate against your chosen search space, so your answers are alway
|
|||
| **Video Generation** | Cinematic Video Overviews via Veo 3 (Ultra only) | Available (NotebookLM is better here, actively improving) |
|
||||
| **Presentation Generation** | Better looking slides but not editable | Create editable, slide-based presentations |
|
||||
| **Podcast Generation** | Audio Overviews with customizable hosts and languages | Available with multiple TTS providers (NotebookLM is better here, actively improving) |
|
||||
| **AI File Sorting** | No | LLM-powered auto-categorization into source, date, category, and subcategory folders |
|
||||
| **AI Automations & Agents** | No | Scheduled AI workflows, event triggers on new documents, and chat-built no-code automations with connector write-back to Notion, Slack, Linear & Jira |
|
||||
| **Desktop App** | No | Native app with General Assist, Quick Assist, Screenshot Assist, and local folder sync |
|
||||
| **Browser Extension** | No | Cross-browser extension to save any webpage, including auth-protected pages |
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
"""Remove AI file sort flag.
|
||||
|
||||
Revision ID: 172
|
||||
Revises: 171
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "172"
|
||||
down_revision: str | None = "171"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_column("workspaces", "ai_file_sort_enabled")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column(
|
||||
"workspaces",
|
||||
sa.Column(
|
||||
"ai_file_sort_enabled",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.text("false"),
|
||||
),
|
||||
)
|
||||
|
|
@ -1732,10 +1732,6 @@ class Workspace(BaseModel, TimestampMixin):
|
|||
Integer, nullable=True, default=0, server_default="0"
|
||||
) # For vision/screenshot analysis, defaults to Auto mode
|
||||
|
||||
ai_file_sort_enabled = Column(
|
||||
Boolean, nullable=False, default=False, server_default="false"
|
||||
)
|
||||
|
||||
user_id = Column(
|
||||
UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""``document.entered_folder``: a document became a member of a folder.
|
||||
|
||||
Fires once per arrival, however the document got there (upload, AI sort, move).
|
||||
Fires once per arrival, however the document got there (upload or move).
|
||||
The payload carries the fields a user can filter a trigger on.
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -427,8 +427,6 @@ class IndexingPipelineService:
|
|||
log_index_success(ctx, chunk_count=chunk_count)
|
||||
outcome_status = "success"
|
||||
|
||||
await self._enqueue_ai_sort_if_enabled(document)
|
||||
|
||||
except RETRYABLE_LLM_ERRORS as e:
|
||||
ot.record_error(persist_span, e)
|
||||
log_retryable_llm_error(ctx, e)
|
||||
|
|
@ -564,29 +562,6 @@ class IndexingPipelineService:
|
|||
)
|
||||
return len(new_texts)
|
||||
|
||||
async def _enqueue_ai_sort_if_enabled(self, document: Document) -> None:
|
||||
"""Fire-and-forget: enqueue incremental AI sort if the workspace has it enabled."""
|
||||
try:
|
||||
from app.db import Workspace
|
||||
|
||||
result = await self.session.execute(
|
||||
select(Workspace.ai_file_sort_enabled).where(
|
||||
Workspace.id == document.workspace_id
|
||||
)
|
||||
)
|
||||
enabled = result.scalar()
|
||||
if not enabled:
|
||||
return
|
||||
|
||||
from app.tasks.celery_tasks.document_tasks import ai_sort_document_task
|
||||
|
||||
user_id = str(document.created_by_id) if document.created_by_id else ""
|
||||
ai_sort_document_task.delay(document.workspace_id, user_id, document.id)
|
||||
except Exception:
|
||||
logging.getLogger(__name__).warning(
|
||||
"Failed to enqueue AI sort for document %s", document.id, exc_info=True
|
||||
)
|
||||
|
||||
async def index_batch_parallel(
|
||||
self,
|
||||
connector_docs: list[ConnectorDocument],
|
||||
|
|
|
|||
|
|
@ -189,7 +189,6 @@ async def read_workspaces(
|
|||
citations_enabled=space.citations_enabled,
|
||||
api_access_enabled=space.api_access_enabled,
|
||||
qna_custom_instructions=space.qna_custom_instructions,
|
||||
ai_file_sort_enabled=space.ai_file_sort_enabled,
|
||||
member_count=member_count,
|
||||
is_owner=is_owner,
|
||||
)
|
||||
|
|
@ -326,43 +325,6 @@ async def update_workspace_api_access(
|
|||
) from e
|
||||
|
||||
|
||||
@router.post("/workspaces/{workspace_id}/ai-sort")
|
||||
async def trigger_ai_sort(
|
||||
workspace_id: int,
|
||||
session: AsyncSession = Depends(get_async_session),
|
||||
auth: AuthContext = Depends(get_auth_context),
|
||||
):
|
||||
user = auth.user
|
||||
"""Trigger a full AI file sort for all documents in the workspace."""
|
||||
try:
|
||||
await check_permission(
|
||||
session,
|
||||
auth,
|
||||
workspace_id,
|
||||
Permission.SETTINGS_UPDATE.value,
|
||||
"You don't have permission to trigger AI sort on this workspace",
|
||||
)
|
||||
|
||||
result = await session.execute(
|
||||
select(Workspace).filter(Workspace.id == workspace_id)
|
||||
)
|
||||
db_workspace = result.scalars().first()
|
||||
if not db_workspace:
|
||||
raise HTTPException(status_code=404, detail="Workspace not found")
|
||||
|
||||
from app.tasks.celery_tasks.document_tasks import ai_sort_workspace_task
|
||||
|
||||
ai_sort_workspace_task.delay(workspace_id, str(user.id))
|
||||
return {"message": "AI sort started"}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to trigger AI sort: {e!s}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Failed to trigger AI sort: {e!s}"
|
||||
) from e
|
||||
|
||||
|
||||
@router.delete("/workspaces/{workspace_id}", response_model=dict)
|
||||
async def delete_workspace(
|
||||
workspace_id: int,
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ class WorkspaceUpdate(BaseModel):
|
|||
description: str | None = None
|
||||
citations_enabled: bool | None = None
|
||||
qna_custom_instructions: str | None = None
|
||||
ai_file_sort_enabled: bool | None = None
|
||||
|
||||
|
||||
class WorkspaceApiAccessUpdate(BaseModel):
|
||||
|
|
@ -36,7 +35,6 @@ class WorkspaceRead(WorkspaceBase, IDModel, TimestampModel):
|
|||
api_access_enabled: bool = False
|
||||
qna_custom_instructions: str | None = None
|
||||
shared_memory_md: str | None = None
|
||||
ai_file_sort_enabled: bool = False
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,329 +0,0 @@
|
|||
"""AI File Sort Service: builds connector-type/date/category/subcategory folder paths."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from langchain_core.messages import HumanMessage
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.db import (
|
||||
Chunk,
|
||||
Document,
|
||||
DocumentType,
|
||||
SearchSourceConnector,
|
||||
SearchSourceConnectorType,
|
||||
)
|
||||
from app.services.folder_service import ensure_folder_hierarchy_with_depth_validation
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DOCTYPE_TO_CONNECTOR_LABEL: dict[str, str] = {
|
||||
DocumentType.EXTENSION: "Browser Extension",
|
||||
DocumentType.CRAWLED_URL: "Web Crawl",
|
||||
DocumentType.FILE: "File Upload",
|
||||
DocumentType.SLACK_CONNECTOR: "Slack",
|
||||
DocumentType.TEAMS_CONNECTOR: "Teams",
|
||||
DocumentType.ONEDRIVE_FILE: "OneDrive",
|
||||
DocumentType.NOTION_CONNECTOR: "Notion",
|
||||
DocumentType.YOUTUBE_VIDEO: "YouTube",
|
||||
DocumentType.GITHUB_CONNECTOR: "GitHub",
|
||||
DocumentType.LINEAR_CONNECTOR: "Linear",
|
||||
DocumentType.DISCORD_CONNECTOR: "Discord",
|
||||
DocumentType.JIRA_CONNECTOR: "Jira",
|
||||
DocumentType.CONFLUENCE_CONNECTOR: "Confluence",
|
||||
DocumentType.CLICKUP_CONNECTOR: "ClickUp",
|
||||
DocumentType.GOOGLE_CALENDAR_CONNECTOR: "Google Calendar",
|
||||
DocumentType.GOOGLE_GMAIL_CONNECTOR: "Gmail",
|
||||
DocumentType.GOOGLE_DRIVE_FILE: "Google Drive",
|
||||
DocumentType.AIRTABLE_CONNECTOR: "Airtable",
|
||||
DocumentType.LUMA_CONNECTOR: "Luma",
|
||||
DocumentType.ELASTICSEARCH_CONNECTOR: "Elasticsearch",
|
||||
DocumentType.BOOKSTACK_CONNECTOR: "BookStack",
|
||||
DocumentType.CIRCLEBACK: "Circleback",
|
||||
DocumentType.OBSIDIAN_CONNECTOR: "Obsidian",
|
||||
DocumentType.NOTE: "Notes",
|
||||
DocumentType.DROPBOX_FILE: "Dropbox",
|
||||
DocumentType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR: "Google Drive (Composio)",
|
||||
DocumentType.COMPOSIO_GMAIL_CONNECTOR: "Gmail (Composio)",
|
||||
DocumentType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: "Google Calendar (Composio)",
|
||||
DocumentType.LOCAL_FOLDER_FILE: "Local Folder",
|
||||
}
|
||||
|
||||
_CONNECTOR_TYPE_LABEL: dict[str, str] = {
|
||||
SearchSourceConnectorType.SERPER_API: "Serper Search",
|
||||
SearchSourceConnectorType.TAVILY_API: "Tavily Search",
|
||||
SearchSourceConnectorType.SEARXNG_API: "SearXNG Search",
|
||||
SearchSourceConnectorType.LINKUP_API: "Linkup Search",
|
||||
SearchSourceConnectorType.BAIDU_SEARCH_API: "Baidu Search",
|
||||
SearchSourceConnectorType.SLACK_CONNECTOR: "Slack",
|
||||
SearchSourceConnectorType.TEAMS_CONNECTOR: "Teams",
|
||||
SearchSourceConnectorType.ONEDRIVE_CONNECTOR: "OneDrive",
|
||||
SearchSourceConnectorType.NOTION_CONNECTOR: "Notion",
|
||||
SearchSourceConnectorType.GITHUB_CONNECTOR: "GitHub",
|
||||
SearchSourceConnectorType.LINEAR_CONNECTOR: "Linear",
|
||||
SearchSourceConnectorType.DISCORD_CONNECTOR: "Discord",
|
||||
SearchSourceConnectorType.JIRA_CONNECTOR: "Jira",
|
||||
SearchSourceConnectorType.CONFLUENCE_CONNECTOR: "Confluence",
|
||||
SearchSourceConnectorType.CLICKUP_CONNECTOR: "ClickUp",
|
||||
SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR: "Google Calendar",
|
||||
SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR: "Gmail",
|
||||
SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR: "Google Drive",
|
||||
SearchSourceConnectorType.AIRTABLE_CONNECTOR: "Airtable",
|
||||
SearchSourceConnectorType.LUMA_CONNECTOR: "Luma",
|
||||
SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR: "Elasticsearch",
|
||||
SearchSourceConnectorType.WEBCRAWLER_CONNECTOR: "Web Crawl",
|
||||
SearchSourceConnectorType.BOOKSTACK_CONNECTOR: "BookStack",
|
||||
SearchSourceConnectorType.CIRCLEBACK_CONNECTOR: "Circleback",
|
||||
SearchSourceConnectorType.OBSIDIAN_CONNECTOR: "Obsidian",
|
||||
SearchSourceConnectorType.MCP_CONNECTOR: "MCP",
|
||||
SearchSourceConnectorType.DROPBOX_CONNECTOR: "Dropbox",
|
||||
SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR: "Google Drive (Composio)",
|
||||
SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR: "Gmail (Composio)",
|
||||
SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: "Google Calendar (Composio)",
|
||||
}
|
||||
|
||||
_MAX_CONTENT_CHARS = 4000
|
||||
_MAX_CHUNKS_FOR_CONTEXT = 5
|
||||
|
||||
_CATEGORY_PROMPT = (
|
||||
"Based on the document information below, classify it into a broad category "
|
||||
"and a more specific subcategory.\n\n"
|
||||
"Rules:\n"
|
||||
"- category: 1-2 word broad theme (e.g. Science, Finance, Engineering, Communication, Media)\n"
|
||||
"- subcategory: 1-2 word specific topic within the category "
|
||||
"(e.g. Physics, Tax Reports, Backend, Team Updates)\n"
|
||||
"- Use nouns only. Do not include generic terms like 'General' or 'Miscellaneous'.\n\n"
|
||||
"Title: {title}\n\n"
|
||||
"Content: {summary}\n\n"
|
||||
'Respond with ONLY a JSON object: {{"category": "...", "subcategory": "..."}}'
|
||||
)
|
||||
|
||||
_SAFE_NAME_RE = re.compile(r"[^a-zA-Z0-9 _\-()]")
|
||||
_FALLBACK_CATEGORY = "Uncategorized"
|
||||
_FALLBACK_SUBCATEGORY = "General"
|
||||
|
||||
|
||||
def resolve_root_folder_label(
|
||||
document: Document, connector: SearchSourceConnector | None
|
||||
) -> str:
|
||||
if connector is not None:
|
||||
return _CONNECTOR_TYPE_LABEL.get(
|
||||
connector.connector_type, str(connector.connector_type)
|
||||
)
|
||||
return _DOCTYPE_TO_CONNECTOR_LABEL.get(
|
||||
document.document_type, str(document.document_type)
|
||||
)
|
||||
|
||||
|
||||
def resolve_date_folder(document: Document) -> str:
|
||||
ts = document.updated_at or document.created_at
|
||||
if ts is None:
|
||||
ts = datetime.now(UTC)
|
||||
return ts.strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def sanitize_category_folder_name(
|
||||
value: str | None, fallback: str = _FALLBACK_CATEGORY
|
||||
) -> str:
|
||||
if not value or not value.strip():
|
||||
return fallback
|
||||
cleaned = _SAFE_NAME_RE.sub("", value.strip())
|
||||
cleaned = " ".join(cleaned.split())
|
||||
if not cleaned:
|
||||
return fallback
|
||||
return cleaned[:50]
|
||||
|
||||
|
||||
async def _resolve_document_text(
|
||||
session: AsyncSession,
|
||||
document: Document,
|
||||
) -> str:
|
||||
"""Build the best available text representation for taxonomy generation.
|
||||
|
||||
Prefers ``document.content``; falls back to joining the first few chunks
|
||||
when content is empty or too short to be useful.
|
||||
"""
|
||||
text = (document.content or "").strip()
|
||||
if len(text) >= 100:
|
||||
return text[:_MAX_CONTENT_CHARS]
|
||||
|
||||
stmt = (
|
||||
select(Chunk.content)
|
||||
.where(Chunk.document_id == document.id)
|
||||
.order_by(Chunk.position, Chunk.id)
|
||||
.limit(_MAX_CHUNKS_FOR_CONTEXT)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
chunk_texts = [row[0] for row in result.all() if row[0]]
|
||||
if chunk_texts:
|
||||
combined = "\n\n".join(chunk_texts)
|
||||
return combined[:_MAX_CONTENT_CHARS]
|
||||
|
||||
return text[:_MAX_CONTENT_CHARS]
|
||||
|
||||
|
||||
def _get_cached_taxonomy(document: Document) -> tuple[str, str] | None:
|
||||
"""Return (category, subcategory) from document metadata cache, or None."""
|
||||
meta = document.document_metadata
|
||||
if not isinstance(meta, dict):
|
||||
return None
|
||||
cat = meta.get("ai_sort_category")
|
||||
subcat = meta.get("ai_sort_subcategory")
|
||||
if cat and subcat and isinstance(cat, str) and isinstance(subcat, str):
|
||||
return cat, subcat
|
||||
return None
|
||||
|
||||
|
||||
def _set_cached_taxonomy(document: Document, category: str, subcategory: str) -> None:
|
||||
"""Persist the AI taxonomy on document metadata for deterministic re-sorts."""
|
||||
meta = dict(document.document_metadata or {})
|
||||
meta["ai_sort_category"] = category
|
||||
meta["ai_sort_subcategory"] = subcategory
|
||||
document.document_metadata = meta
|
||||
|
||||
|
||||
async def generate_ai_taxonomy(
|
||||
title: str,
|
||||
summary_or_content: str,
|
||||
llm,
|
||||
) -> tuple[str, str]:
|
||||
"""Return (category, subcategory) using a single structured LLM call."""
|
||||
text = (summary_or_content or "").strip()
|
||||
if not text:
|
||||
return _FALLBACK_CATEGORY, _FALLBACK_SUBCATEGORY
|
||||
|
||||
if len(text) > _MAX_CONTENT_CHARS:
|
||||
text = text[:_MAX_CONTENT_CHARS]
|
||||
|
||||
prompt = _CATEGORY_PROMPT.format(title=title or "Untitled", summary=text)
|
||||
try:
|
||||
result = await llm.ainvoke([HumanMessage(content=prompt)])
|
||||
raw = result.content.strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
|
||||
parsed = json.loads(raw)
|
||||
category = sanitize_category_folder_name(
|
||||
parsed.get("category"), _FALLBACK_CATEGORY
|
||||
)
|
||||
subcategory = sanitize_category_folder_name(
|
||||
parsed.get("subcategory"), _FALLBACK_SUBCATEGORY
|
||||
)
|
||||
return category, subcategory
|
||||
except Exception:
|
||||
logger.warning("AI taxonomy generation failed, using fallback", exc_info=True)
|
||||
return _FALLBACK_CATEGORY, _FALLBACK_SUBCATEGORY
|
||||
|
||||
|
||||
def _build_path_segments(
|
||||
root_label: str,
|
||||
date_label: str,
|
||||
category: str,
|
||||
subcategory: str,
|
||||
) -> list[dict]:
|
||||
return [
|
||||
{"name": root_label, "metadata": {"ai_sort": True, "ai_sort_level": 1}},
|
||||
{"name": date_label, "metadata": {"ai_sort": True, "ai_sort_level": 2}},
|
||||
{"name": category, "metadata": {"ai_sort": True, "ai_sort_level": 3}},
|
||||
{"name": subcategory, "metadata": {"ai_sort": True, "ai_sort_level": 4}},
|
||||
]
|
||||
|
||||
|
||||
async def _resolve_taxonomy(
|
||||
session: AsyncSession,
|
||||
document: Document,
|
||||
llm,
|
||||
) -> tuple[str, str]:
|
||||
"""Return (category, subcategory), reusing cached values when available."""
|
||||
cached = _get_cached_taxonomy(document)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
content_text = await _resolve_document_text(session, document)
|
||||
category, subcategory = await generate_ai_taxonomy(
|
||||
document.title, content_text, llm
|
||||
)
|
||||
_set_cached_taxonomy(document, category, subcategory)
|
||||
return category, subcategory
|
||||
|
||||
|
||||
async def ai_sort_document(
|
||||
session: AsyncSession,
|
||||
document: Document,
|
||||
llm,
|
||||
) -> Document:
|
||||
"""Sort a single document into the 4-level AI folder hierarchy."""
|
||||
connector: SearchSourceConnector | None = None
|
||||
if document.connector_id is not None:
|
||||
connector = await session.get(SearchSourceConnector, document.connector_id)
|
||||
|
||||
root_label = resolve_root_folder_label(document, connector)
|
||||
date_label = resolve_date_folder(document)
|
||||
|
||||
category, subcategory = await _resolve_taxonomy(session, document, llm)
|
||||
|
||||
segments = _build_path_segments(root_label, date_label, category, subcategory)
|
||||
|
||||
leaf_folder = await ensure_folder_hierarchy_with_depth_validation(
|
||||
session,
|
||||
document.workspace_id,
|
||||
segments,
|
||||
)
|
||||
|
||||
document.folder_id = leaf_folder.id
|
||||
await session.flush()
|
||||
return document
|
||||
|
||||
|
||||
async def ai_sort_all_documents(
|
||||
session: AsyncSession,
|
||||
workspace_id: int,
|
||||
llm,
|
||||
) -> tuple[int, int]:
|
||||
"""Sort all documents in a workspace. Returns (sorted_count, failed_count)."""
|
||||
stmt = (
|
||||
select(Document)
|
||||
.where(Document.workspace_id == workspace_id)
|
||||
.options(selectinload(Document.connector))
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
documents = list(result.scalars().all())
|
||||
|
||||
sorted_count = 0
|
||||
failed_count = 0
|
||||
|
||||
for doc in documents:
|
||||
try:
|
||||
connector = doc.connector
|
||||
root_label = resolve_root_folder_label(doc, connector)
|
||||
date_label = resolve_date_folder(doc)
|
||||
|
||||
category, subcategory = await _resolve_taxonomy(session, doc, llm)
|
||||
segments = _build_path_segments(
|
||||
root_label, date_label, category, subcategory
|
||||
)
|
||||
|
||||
leaf_folder = await ensure_folder_hierarchy_with_depth_validation(
|
||||
session,
|
||||
workspace_id,
|
||||
segments,
|
||||
)
|
||||
doc.folder_id = leaf_folder.id
|
||||
sorted_count += 1
|
||||
except Exception:
|
||||
logger.error("Failed to AI-sort document %s", doc.id, exc_info=True)
|
||||
failed_count += 1
|
||||
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"AI sort complete for workspace=%d: sorted=%d, failed=%d",
|
||||
workspace_id,
|
||||
sorted_count,
|
||||
failed_count,
|
||||
)
|
||||
return sorted_count, failed_count
|
||||
|
|
@ -4,7 +4,6 @@ import asyncio
|
|||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from uuid import UUID
|
||||
|
||||
from app.celery_app import celery_app
|
||||
|
|
@ -1395,108 +1394,3 @@ async def _index_uploaded_folder_files_async(
|
|||
if notification_id is not None:
|
||||
_stop_heartbeat(notification_id)
|
||||
|
||||
|
||||
# ===== AI File Sort tasks =====
|
||||
|
||||
AI_SORT_LOCK_TTL_SECONDS = 600 # 10 minutes
|
||||
_ai_sort_redis = None
|
||||
|
||||
|
||||
def _get_ai_sort_redis():
|
||||
import redis
|
||||
|
||||
global _ai_sort_redis
|
||||
if _ai_sort_redis is None:
|
||||
_ai_sort_redis = redis.from_url(config.REDIS_APP_URL, decode_responses=True)
|
||||
return _ai_sort_redis
|
||||
|
||||
|
||||
def _ai_sort_lock_key(workspace_id: int) -> str:
|
||||
return f"ai_sort:workspace:{workspace_id}:lock"
|
||||
|
||||
|
||||
@celery_app.task(name="ai_sort_search_space", bind=True, max_retries=1)
|
||||
def ai_sort_workspace_task(self, workspace_id: int, user_id: str):
|
||||
"""Full AI sort for all documents in a workspace."""
|
||||
return run_async_celery_task(
|
||||
lambda: _ai_sort_workspace_async(workspace_id, user_id)
|
||||
)
|
||||
|
||||
|
||||
async def _ai_sort_workspace_async(workspace_id: int, user_id: str):
|
||||
r = _get_ai_sort_redis()
|
||||
lock_key = _ai_sort_lock_key(workspace_id)
|
||||
|
||||
if not r.set(lock_key, "running", nx=True, ex=AI_SORT_LOCK_TTL_SECONDS):
|
||||
logger.info(
|
||||
"AI sort already running for workspace=%d, skipping",
|
||||
workspace_id,
|
||||
)
|
||||
return
|
||||
|
||||
t_start = time.perf_counter()
|
||||
try:
|
||||
from app.services.ai_file_sort_service import ai_sort_all_documents
|
||||
from app.services.llm_service import get_agent_llm
|
||||
|
||||
async with get_celery_session_maker()() as session:
|
||||
llm = await get_agent_llm(session, workspace_id, disable_streaming=True)
|
||||
if llm is None:
|
||||
logger.warning(
|
||||
"No LLM configured for workspace=%d, skipping AI sort",
|
||||
workspace_id,
|
||||
)
|
||||
return
|
||||
|
||||
sorted_count, failed_count = await ai_sort_all_documents(
|
||||
session, workspace_id, llm
|
||||
)
|
||||
elapsed = time.perf_counter() - t_start
|
||||
logger.info(
|
||||
"AI sort workspace=%d done in %.1fs: sorted=%d failed=%d",
|
||||
workspace_id,
|
||||
elapsed,
|
||||
sorted_count,
|
||||
failed_count,
|
||||
)
|
||||
finally:
|
||||
r.delete(lock_key)
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
name="ai_sort_document", bind=True, max_retries=2, default_retry_delay=10
|
||||
)
|
||||
def ai_sort_document_task(self, workspace_id: int, user_id: str, document_id: int):
|
||||
"""Incremental AI sort for a single document after indexing."""
|
||||
return run_async_celery_task(
|
||||
lambda: _ai_sort_document_async(workspace_id, user_id, document_id)
|
||||
)
|
||||
|
||||
|
||||
async def _ai_sort_document_async(workspace_id: int, user_id: str, document_id: int):
|
||||
from app.db import Document
|
||||
from app.services.ai_file_sort_service import ai_sort_document
|
||||
from app.services.llm_service import get_agent_llm
|
||||
|
||||
async with get_celery_session_maker()() as session:
|
||||
document = await session.get(Document, document_id)
|
||||
if document is None:
|
||||
logger.warning("Document %d not found, skipping AI sort", document_id)
|
||||
return
|
||||
|
||||
llm = await get_agent_llm(session, workspace_id, disable_streaming=True)
|
||||
if llm is None:
|
||||
logger.warning(
|
||||
"No LLM for workspace=%d, skipping AI sort of doc=%d",
|
||||
workspace_id,
|
||||
document_id,
|
||||
)
|
||||
return
|
||||
|
||||
await ai_sort_document(session, document, llm)
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"AI sorted document=%d into workspace=%d",
|
||||
document_id,
|
||||
workspace_id,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ def _disable_otel(monkeypatch: pytest.MonkeyPatch):
|
|||
("cleanup_stale_indexing_notifications", "cleanup"),
|
||||
("reconcile_pending_stripe_credit_purchases", "reconcile"),
|
||||
("check_periodic_schedules", "check"),
|
||||
("ai_sort_search_space", "ai"),
|
||||
("index_notion_pages", "index"),
|
||||
("index_github_repos", "index"),
|
||||
("index_google_drive_files", "index"),
|
||||
|
|
|
|||
|
|
@ -1,275 +0,0 @@
|
|||
"""Unit tests for AI file sort service: folder label resolution, date extraction, category sanitization."""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
# ── resolve_root_folder_label ──
|
||||
|
||||
|
||||
def _make_document(document_type: str, connector_id=None):
|
||||
doc = MagicMock()
|
||||
doc.document_type = document_type
|
||||
doc.connector_id = connector_id
|
||||
return doc
|
||||
|
||||
|
||||
def _make_connector(connector_type: str):
|
||||
conn = MagicMock()
|
||||
conn.connector_type = connector_type
|
||||
return conn
|
||||
|
||||
|
||||
def test_root_label_uses_connector_type_when_available():
|
||||
from app.services.ai_file_sort_service import resolve_root_folder_label
|
||||
|
||||
doc = _make_document("FILE", connector_id=1)
|
||||
conn = _make_connector("GOOGLE_DRIVE_CONNECTOR")
|
||||
assert resolve_root_folder_label(doc, conn) == "Google Drive"
|
||||
|
||||
|
||||
def test_root_label_falls_back_to_document_type():
|
||||
from app.services.ai_file_sort_service import resolve_root_folder_label
|
||||
|
||||
doc = _make_document("SLACK_CONNECTOR")
|
||||
assert resolve_root_folder_label(doc, None) == "Slack"
|
||||
|
||||
|
||||
def test_root_label_unknown_doctype_returns_raw_value():
|
||||
from app.services.ai_file_sort_service import resolve_root_folder_label
|
||||
|
||||
doc = _make_document("UNKNOWN_TYPE")
|
||||
assert resolve_root_folder_label(doc, None) == "UNKNOWN_TYPE"
|
||||
|
||||
|
||||
# ── resolve_date_folder ──
|
||||
|
||||
|
||||
def test_date_folder_from_updated_at():
|
||||
from app.services.ai_file_sort_service import resolve_date_folder
|
||||
|
||||
doc = MagicMock()
|
||||
doc.updated_at = datetime(2025, 3, 15, 10, 30, 0, tzinfo=UTC)
|
||||
doc.created_at = datetime(2025, 1, 1, 0, 0, 0, tzinfo=UTC)
|
||||
assert resolve_date_folder(doc) == "2025-03-15"
|
||||
|
||||
|
||||
def test_date_folder_falls_back_to_created_at():
|
||||
from app.services.ai_file_sort_service import resolve_date_folder
|
||||
|
||||
doc = MagicMock()
|
||||
doc.updated_at = None
|
||||
doc.created_at = datetime(2024, 12, 25, 23, 59, 0, tzinfo=UTC)
|
||||
assert resolve_date_folder(doc) == "2024-12-25"
|
||||
|
||||
|
||||
def test_date_folder_both_none_uses_today():
|
||||
from app.services.ai_file_sort_service import resolve_date_folder
|
||||
|
||||
doc = MagicMock()
|
||||
doc.updated_at = None
|
||||
doc.created_at = None
|
||||
result = resolve_date_folder(doc)
|
||||
today = datetime.now(UTC).strftime("%Y-%m-%d")
|
||||
assert result == today
|
||||
|
||||
|
||||
# ── sanitize_category_folder_name ──
|
||||
|
||||
|
||||
def test_sanitize_normal_value():
|
||||
from app.services.ai_file_sort_service import sanitize_category_folder_name
|
||||
|
||||
assert sanitize_category_folder_name("Machine Learning") == "Machine Learning"
|
||||
|
||||
|
||||
def test_sanitize_strips_special_chars():
|
||||
from app.services.ai_file_sort_service import sanitize_category_folder_name
|
||||
|
||||
assert sanitize_category_folder_name("Tax/Reports!") == "TaxReports"
|
||||
|
||||
|
||||
def test_sanitize_empty_returns_fallback():
|
||||
from app.services.ai_file_sort_service import sanitize_category_folder_name
|
||||
|
||||
assert sanitize_category_folder_name("") == "Uncategorized"
|
||||
assert sanitize_category_folder_name(None) == "Uncategorized"
|
||||
|
||||
|
||||
def test_sanitize_truncates_long_names():
|
||||
from app.services.ai_file_sort_service import sanitize_category_folder_name
|
||||
|
||||
long_name = "A" * 100
|
||||
result = sanitize_category_folder_name(long_name)
|
||||
assert len(result) <= 50
|
||||
|
||||
|
||||
# ── generate_ai_taxonomy ──
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_ai_taxonomy_parses_json():
|
||||
from app.services.ai_file_sort_service import generate_ai_taxonomy
|
||||
|
||||
mock_llm = AsyncMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.content = '{"category": "Science", "subcategory": "Physics"}'
|
||||
mock_llm.ainvoke.return_value = mock_result
|
||||
|
||||
cat, sub = await generate_ai_taxonomy(
|
||||
"Physics Paper", "Some science document about physics", mock_llm
|
||||
)
|
||||
assert cat == "Science"
|
||||
assert sub == "Physics"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_ai_taxonomy_handles_markdown_code_block():
|
||||
from app.services.ai_file_sort_service import generate_ai_taxonomy
|
||||
|
||||
mock_llm = AsyncMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.content = (
|
||||
'```json\n{"category": "Finance", "subcategory": "Tax Reports"}\n```'
|
||||
)
|
||||
mock_llm.ainvoke.return_value = mock_result
|
||||
|
||||
cat, sub = await generate_ai_taxonomy("Tax Doc", "A tax report document", mock_llm)
|
||||
assert cat == "Finance"
|
||||
assert sub == "Tax Reports"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_ai_taxonomy_includes_title_in_prompt():
|
||||
from app.services.ai_file_sort_service import generate_ai_taxonomy
|
||||
|
||||
mock_llm = AsyncMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.content = '{"category": "Engineering", "subcategory": "Backend"}'
|
||||
mock_llm.ainvoke.return_value = mock_result
|
||||
|
||||
await generate_ai_taxonomy("API Design Guide", "content about REST APIs", mock_llm)
|
||||
|
||||
prompt_text = mock_llm.ainvoke.call_args[0][0][0].content
|
||||
assert "API Design Guide" in prompt_text
|
||||
assert "content about REST APIs" in prompt_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_ai_taxonomy_fallback_on_error():
|
||||
from app.services.ai_file_sort_service import generate_ai_taxonomy
|
||||
|
||||
mock_llm = AsyncMock()
|
||||
mock_llm.ainvoke.side_effect = RuntimeError("LLM down")
|
||||
|
||||
cat, sub = await generate_ai_taxonomy("Title", "some content", mock_llm)
|
||||
assert cat == "Uncategorized"
|
||||
assert sub == "General"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_ai_taxonomy_fallback_on_empty_content():
|
||||
from app.services.ai_file_sort_service import generate_ai_taxonomy
|
||||
|
||||
mock_llm = AsyncMock()
|
||||
cat, sub = await generate_ai_taxonomy("Title", "", mock_llm)
|
||||
assert cat == "Uncategorized"
|
||||
assert sub == "General"
|
||||
mock_llm.ainvoke.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_ai_taxonomy_fallback_on_invalid_json():
|
||||
from app.services.ai_file_sort_service import generate_ai_taxonomy
|
||||
|
||||
mock_llm = AsyncMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.content = "not valid json at all"
|
||||
mock_llm.ainvoke.return_value = mock_result
|
||||
|
||||
cat, sub = await generate_ai_taxonomy("Title", "some content", mock_llm)
|
||||
assert cat == "Uncategorized"
|
||||
assert sub == "General"
|
||||
|
||||
|
||||
# ── taxonomy caching ──
|
||||
|
||||
|
||||
def test_get_cached_taxonomy_returns_none_when_no_metadata():
|
||||
from app.services.ai_file_sort_service import _get_cached_taxonomy
|
||||
|
||||
doc = MagicMock()
|
||||
doc.document_metadata = None
|
||||
assert _get_cached_taxonomy(doc) is None
|
||||
|
||||
|
||||
def test_get_cached_taxonomy_returns_none_when_keys_missing():
|
||||
from app.services.ai_file_sort_service import _get_cached_taxonomy
|
||||
|
||||
doc = MagicMock()
|
||||
doc.document_metadata = {"some_other_key": "value"}
|
||||
assert _get_cached_taxonomy(doc) is None
|
||||
|
||||
|
||||
def test_get_cached_taxonomy_returns_cached_values():
|
||||
from app.services.ai_file_sort_service import _get_cached_taxonomy
|
||||
|
||||
doc = MagicMock()
|
||||
doc.document_metadata = {
|
||||
"ai_sort_category": "Finance",
|
||||
"ai_sort_subcategory": "Tax Reports",
|
||||
}
|
||||
assert _get_cached_taxonomy(doc) == ("Finance", "Tax Reports")
|
||||
|
||||
|
||||
def test_set_cached_taxonomy_persists_on_metadata():
|
||||
from app.services.ai_file_sort_service import _set_cached_taxonomy
|
||||
|
||||
doc = MagicMock()
|
||||
doc.document_metadata = {"existing_key": "keep_me"}
|
||||
_set_cached_taxonomy(doc, "Science", "Physics")
|
||||
assert doc.document_metadata["ai_sort_category"] == "Science"
|
||||
assert doc.document_metadata["ai_sort_subcategory"] == "Physics"
|
||||
assert doc.document_metadata["existing_key"] == "keep_me"
|
||||
|
||||
|
||||
def test_set_cached_taxonomy_creates_metadata_when_none():
|
||||
from app.services.ai_file_sort_service import _set_cached_taxonomy
|
||||
|
||||
doc = MagicMock()
|
||||
doc.document_metadata = None
|
||||
_set_cached_taxonomy(doc, "Engineering", "Backend")
|
||||
assert doc.document_metadata == {
|
||||
"ai_sort_category": "Engineering",
|
||||
"ai_sort_subcategory": "Backend",
|
||||
}
|
||||
|
||||
|
||||
# ── _build_path_segments ──
|
||||
|
||||
|
||||
def test_build_path_segments_structure():
|
||||
from app.services.ai_file_sort_service import _build_path_segments
|
||||
|
||||
segments = _build_path_segments("Google Drive", "2025-03-15", "Science", "Physics")
|
||||
assert len(segments) == 4
|
||||
assert segments[0] == {
|
||||
"name": "Google Drive",
|
||||
"metadata": {"ai_sort": True, "ai_sort_level": 1},
|
||||
}
|
||||
assert segments[1] == {
|
||||
"name": "2025-03-15",
|
||||
"metadata": {"ai_sort": True, "ai_sort_level": 2},
|
||||
}
|
||||
assert segments[2] == {
|
||||
"name": "Science",
|
||||
"metadata": {"ai_sort": True, "ai_sort_level": 3},
|
||||
}
|
||||
assert segments[3] == {
|
||||
"name": "Physics",
|
||||
"metadata": {"ai_sort": True, "ai_sort_level": 4},
|
||||
}
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
"""Unit tests for AI sort task Redis deduplication lock."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_lock_key_format():
|
||||
from app.tasks.celery_tasks.document_tasks import _ai_sort_lock_key
|
||||
|
||||
key = _ai_sort_lock_key(42)
|
||||
assert key == "ai_sort:workspace:42:lock"
|
||||
|
||||
|
||||
def test_lock_prevents_duplicate_run():
|
||||
"""When the Redis lock already exists, the task should skip execution."""
|
||||
|
||||
mock_redis = MagicMock()
|
||||
mock_redis.set.return_value = False # Lock already held
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.tasks.celery_tasks.document_tasks._get_ai_sort_redis",
|
||||
return_value=mock_redis,
|
||||
),
|
||||
patch(
|
||||
"app.tasks.celery_tasks.document_tasks.get_celery_session_maker"
|
||||
) as mock_session_maker,
|
||||
):
|
||||
import asyncio
|
||||
|
||||
from app.tasks.celery_tasks.document_tasks import _ai_sort_workspace_async
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
loop.run_until_complete(_ai_sort_workspace_async(1, "user-123"))
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
# Session maker should never be called since lock was not acquired
|
||||
mock_session_maker.assert_not_called()
|
||||
|
|
@ -49,8 +49,8 @@ async def test_creates_missing_folders_in_chain():
|
|||
session.flush = mock_flush
|
||||
|
||||
segments = [
|
||||
{"name": "Slack", "metadata": {"ai_sort": True, "ai_sort_level": 1}},
|
||||
{"name": "2025-03-15", "metadata": {"ai_sort": True, "ai_sort_level": 2}},
|
||||
{"name": "Slack", "metadata": {"source": "slack"}},
|
||||
{"name": "2025-03-15", "metadata": {"source": "slack"}},
|
||||
]
|
||||
|
||||
result = await ensure_folder_hierarchy_with_depth_validation(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
"use client";
|
||||
|
||||
import { IconBinaryTree, IconBinaryTreeFilled } from "@tabler/icons-react";
|
||||
import { FolderPlus, ListFilter, Search, Upload, X } from "lucide-react";
|
||||
import { useTranslations } from "next-intl";
|
||||
import React, { useCallback, useMemo, useRef, useState } from "react";
|
||||
|
|
@ -9,12 +8,10 @@ import { Button } from "@/components/ui/button";
|
|||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
|
||||
import { Spinner } from "@/components/ui/spinner";
|
||||
import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import type { DocumentTypeEnum } from "@/contracts/types/document.types";
|
||||
import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { getDocumentTypeIcon } from "./DocumentTypeIcon";
|
||||
|
||||
export function DocumentsFilters({
|
||||
|
|
@ -24,9 +21,6 @@ export function DocumentsFilters({
|
|||
onToggleType,
|
||||
activeTypes,
|
||||
onCreateFolder,
|
||||
aiSortEnabled = false,
|
||||
aiSortBusy = false,
|
||||
onToggleAiSort,
|
||||
onUploadClick,
|
||||
}: {
|
||||
typeCounts: Partial<Record<DocumentTypeEnum, number>>;
|
||||
|
|
@ -35,9 +29,6 @@ export function DocumentsFilters({
|
|||
onToggleType: (type: DocumentTypeEnum, checked: boolean) => void;
|
||||
activeTypes: DocumentTypeEnum[];
|
||||
onCreateFolder?: () => void;
|
||||
aiSortEnabled?: boolean;
|
||||
aiSortBusy?: boolean;
|
||||
onToggleAiSort?: () => void;
|
||||
onUploadClick?: () => void;
|
||||
}) {
|
||||
const t = useTranslations("documents");
|
||||
|
|
@ -77,7 +68,7 @@ export function DocumentsFilters({
|
|||
return (
|
||||
<div className="flex select-none">
|
||||
<div className="flex items-center gap-2 w-full">
|
||||
{/* New Folder + AI Sort + Filter Toggle Group */}
|
||||
{/* New Folder + Filter Toggle Group */}
|
||||
<ToggleGroup type="multiple" value={[]} className="overflow-visible">
|
||||
{onCreateFolder && (
|
||||
<Tooltip>
|
||||
|
|
@ -97,46 +88,6 @@ export function DocumentsFilters({
|
|||
</Tooltip>
|
||||
)}
|
||||
|
||||
{onToggleAiSort && (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<ToggleGroupItem
|
||||
value="ai-sort"
|
||||
disabled={aiSortBusy}
|
||||
className={cn(
|
||||
"h-8 w-8 shrink-0 border-0 bg-muted transition-colors",
|
||||
"relative before:absolute before:left-0 before:top-1/2 before:h-4 before:w-px before:-translate-y-1/2 before:bg-border/60 before:content-[''] dark:before:bg-white/10",
|
||||
"disabled:pointer-events-none disabled:opacity-50",
|
||||
aiSortEnabled
|
||||
? "bg-accent text-accent-foreground hover:bg-accent"
|
||||
: "text-muted-foreground hover:bg-accent hover:text-accent-foreground"
|
||||
)}
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
onToggleAiSort();
|
||||
}}
|
||||
aria-label={aiSortEnabled ? "Disable AI sort" : "Enable AI sort"}
|
||||
aria-pressed={aiSortEnabled}
|
||||
>
|
||||
{aiSortBusy ? (
|
||||
<Spinner size="xs" />
|
||||
) : aiSortEnabled ? (
|
||||
<IconBinaryTreeFilled size={14} />
|
||||
) : (
|
||||
<IconBinaryTree size={14} />
|
||||
)}
|
||||
</ToggleGroupItem>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
{aiSortBusy
|
||||
? "AI sort in progress..."
|
||||
: aiSortEnabled
|
||||
? "AI sort active — click to disable"
|
||||
: "Enable AI sort"}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
|
||||
<Popover>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
|
|
|
|||
|
|
@ -93,8 +93,6 @@ export function FolderTreeView({
|
|||
|
||||
const [openContextMenuId, setOpenContextMenuId] = useState<string | null>(null);
|
||||
|
||||
const [manuallyCollapsedAiIds, setManuallyCollapsedAiIds] = useState<Set<number>>(new Set());
|
||||
|
||||
// Single subscription for rename state — derived boolean passed to each FolderNode
|
||||
const [renamingFolderId, setRenamingFolderId] = useAtom(renamingFolderIdAtom);
|
||||
const handleStartRename = useCallback(
|
||||
|
|
@ -103,38 +101,6 @@ export function FolderTreeView({
|
|||
);
|
||||
const handleCancelRename = useCallback(() => setRenamingFolderId(null), [setRenamingFolderId]);
|
||||
|
||||
const aiSortFolderLevels = useMemo(() => {
|
||||
const map = new Map<number, number>();
|
||||
for (const f of folders) {
|
||||
if (f.metadata?.ai_sort === true && typeof f.metadata?.ai_sort_level === "number") {
|
||||
map.set(f.id, f.metadata.ai_sort_level as number);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}, [folders]);
|
||||
|
||||
const handleToggleExpand = useCallback(
|
||||
(folderId: number) => {
|
||||
const aiLevel = aiSortFolderLevels.get(folderId);
|
||||
if (aiLevel !== undefined && aiLevel < 4) {
|
||||
// AI-auto-expanded folder: only toggle the manual-collapse set.
|
||||
// Calling onToggleExpand would add it to expandedIds and fight auto-expand.
|
||||
setManuallyCollapsedAiIds((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(folderId)) {
|
||||
next.delete(folderId);
|
||||
} else {
|
||||
next.add(folderId);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
return;
|
||||
}
|
||||
onToggleExpand(folderId);
|
||||
},
|
||||
[onToggleExpand, aiSortFolderLevels]
|
||||
);
|
||||
|
||||
const effectiveActiveTypes = useMemo(() => {
|
||||
if (
|
||||
activeTypes.includes("FILE" as DocumentTypeEnum) &&
|
||||
|
|
@ -280,14 +246,9 @@ export function FolderTreeView({
|
|||
|
||||
function renderLevel(parentId: number | null, depth: number): React.ReactNode[] {
|
||||
const key = parentId ?? "root";
|
||||
const childFolders = (foldersByParent[key] ?? []).slice().sort((a, b) => {
|
||||
const aIsDate = a.metadata?.ai_sort === true && a.metadata?.ai_sort_level === 2;
|
||||
const bIsDate = b.metadata?.ai_sort === true && b.metadata?.ai_sort_level === 2;
|
||||
if (aIsDate && bIsDate) {
|
||||
return b.name.localeCompare(a.name);
|
||||
}
|
||||
return a.position.localeCompare(b.position);
|
||||
});
|
||||
const childFolders = (foldersByParent[key] ?? [])
|
||||
.slice()
|
||||
.sort((a, b) => a.position.localeCompare(b.position));
|
||||
const visibleFolders = hasDescendantMatch
|
||||
? childFolders.filter((f) => hasDescendantMatch[f.id])
|
||||
: childFolders;
|
||||
|
|
@ -317,14 +278,7 @@ export function FolderTreeView({
|
|||
};
|
||||
|
||||
const isSearchAutoExpanded = !!searchQuery && !!hasDescendantMatch?.[f.id];
|
||||
const isAiAutoExpandCandidate =
|
||||
f.metadata?.ai_sort === true &&
|
||||
typeof f.metadata?.ai_sort_level === "number" &&
|
||||
(f.metadata.ai_sort_level as number) < 4;
|
||||
const isManuallyCollapsed = manuallyCollapsedAiIds.has(f.id);
|
||||
const isExpanded = isManuallyCollapsed
|
||||
? isSearchAutoExpanded
|
||||
: expandedIds.has(f.id) || isSearchAutoExpanded || isAiAutoExpandCandidate;
|
||||
const isExpanded = expandedIds.has(f.id) || isSearchAutoExpanded;
|
||||
|
||||
nodes.push(
|
||||
<FolderNode
|
||||
|
|
@ -336,7 +290,7 @@ export function FolderTreeView({
|
|||
selectionState={folderSelectionStates[f.id] ?? "none"}
|
||||
processingState={folderProcessingStates[f.id] ?? "idle"}
|
||||
onToggleSelect={onToggleFolderSelect}
|
||||
onToggleExpand={handleToggleExpand}
|
||||
onToggleExpand={onToggleExpand}
|
||||
onRename={onRenameFolder}
|
||||
onStartRename={handleStartRename}
|
||||
onCancelRename={handleCancelRename}
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ import {
|
|||
folderWatchDialogOpenAtom,
|
||||
folderWatchInitialFolderAtom,
|
||||
} from "@/atoms/folder-sync/folder-sync.atoms";
|
||||
import { searchSpacesAtom } from "@/atoms/search-spaces/search-space-query.atoms";
|
||||
import { CreateFolderDialog } from "@/components/documents/CreateFolderDialog";
|
||||
import type { DocumentNodeDoc } from "@/components/documents/DocumentNode";
|
||||
import { DocumentsFilters } from "@/components/documents/DocumentsFilters";
|
||||
|
|
@ -76,7 +75,6 @@ import { useElectronAPI, usePlatform } from "@/hooks/use-platform";
|
|||
import { anonymousChatApiService } from "@/lib/apis/anonymous-chat-api.service";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
import { foldersApiService } from "@/lib/apis/folders-api.service";
|
||||
import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service";
|
||||
import { authenticatedFetch } from "@/lib/auth-fetch";
|
||||
import { getMentionDocKey } from "@/lib/chat/mention-doc-key";
|
||||
import { buildBackendUrl } from "@/lib/env-config";
|
||||
|
|
@ -362,47 +360,6 @@ function AuthenticatedDocumentsSidebarBase({
|
|||
[electronAPI, searchSpaceId]
|
||||
);
|
||||
|
||||
// AI File Sort state
|
||||
const { data: searchSpaces, refetch: refetchSearchSpaces } = useAtomValue(searchSpacesAtom);
|
||||
const activeSearchSpace = useMemo(
|
||||
() => searchSpaces?.find((s) => s.id === searchSpaceId),
|
||||
[searchSpaces, searchSpaceId]
|
||||
);
|
||||
const aiSortEnabled = activeSearchSpace?.ai_file_sort_enabled ?? false;
|
||||
const [aiSortBusy, setAiSortBusy] = useState(false);
|
||||
const [aiSortConfirmOpen, setAiSortConfirmOpen] = useState(false);
|
||||
|
||||
const handleToggleAiSort = useCallback(() => {
|
||||
if (aiSortEnabled) {
|
||||
// Disable: just update the setting, no confirmation needed
|
||||
setAiSortBusy(true);
|
||||
searchSpacesApiService
|
||||
.updateSearchSpace({ id: searchSpaceId, data: { ai_file_sort_enabled: false } })
|
||||
.then(() => {
|
||||
refetchSearchSpaces();
|
||||
toast.success("AI file sorting disabled");
|
||||
})
|
||||
.catch(() => toast.error("Failed to disable AI file sorting"))
|
||||
.finally(() => setAiSortBusy(false));
|
||||
} else {
|
||||
setAiSortConfirmOpen(true);
|
||||
}
|
||||
}, [aiSortEnabled, searchSpaceId, refetchSearchSpaces]);
|
||||
|
||||
const handleConfirmEnableAiSort = useCallback(() => {
|
||||
setAiSortConfirmOpen(false);
|
||||
setAiSortBusy(true);
|
||||
searchSpacesApiService
|
||||
.updateSearchSpace({ id: searchSpaceId, data: { ai_file_sort_enabled: true } })
|
||||
.then(() => searchSpacesApiService.triggerAiSort(searchSpaceId))
|
||||
.then(() => {
|
||||
refetchSearchSpaces();
|
||||
toast.success("AI file sorting enabled — organizing your documents in the background");
|
||||
})
|
||||
.catch(() => toast.error("Failed to enable AI file sorting"))
|
||||
.finally(() => setAiSortBusy(false));
|
||||
}, [searchSpaceId, refetchSearchSpaces]);
|
||||
|
||||
const handleWatchLocalFolder = useCallback(async () => {
|
||||
const api = window.electronAPI;
|
||||
if (!api?.selectFolder) return;
|
||||
|
|
@ -1251,9 +1208,6 @@ function AuthenticatedDocumentsSidebarBase({
|
|||
onToggleType={onToggleType}
|
||||
activeTypes={activeTypes}
|
||||
onCreateFolder={() => handleCreateFolder(null)}
|
||||
aiSortEnabled={aiSortEnabled}
|
||||
aiSortBusy={aiSortBusy}
|
||||
onToggleAiSort={handleToggleAiSort}
|
||||
/>
|
||||
</div>
|
||||
|
||||
|
|
@ -1588,22 +1542,6 @@ function AuthenticatedDocumentsSidebarBase({
|
|||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
|
||||
<AlertDialog open={aiSortConfirmOpen} onOpenChange={setAiSortConfirmOpen}>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>Enable AI File Sorting?</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
All documents in this search space will be organized into folders by connector type,
|
||||
date, and AI-generated categories. New documents will also be sorted automatically.
|
||||
You can disable this at any time.
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel>Cancel</AlertDialogCancel>
|
||||
<AlertDialogAction onClick={handleConfirmEnableAiSort}>Enable</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
</>
|
||||
);
|
||||
|
||||
|
|
@ -1935,7 +1873,6 @@ function AnonymousDocumentsSidebar({
|
|||
onToggleType={() => {}}
|
||||
activeTypes={[]}
|
||||
onCreateFolder={() => gate("create folders")}
|
||||
aiSortEnabled={false}
|
||||
onUploadClick={handleAnonUploadClick}
|
||||
/>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ export const searchSpace = z.object({
|
|||
api_access_enabled: z.boolean().optional().default(false),
|
||||
qna_custom_instructions: z.string().nullable(),
|
||||
shared_memory_md: z.string().nullable().optional(),
|
||||
ai_file_sort_enabled: z.boolean().optional().default(false),
|
||||
member_count: z.number(),
|
||||
is_owner: z.boolean(),
|
||||
});
|
||||
|
|
@ -58,7 +57,6 @@ export const updateSearchSpaceRequest = z.object({
|
|||
citations_enabled: true,
|
||||
api_access_enabled: true,
|
||||
qna_custom_instructions: true,
|
||||
ai_file_sort_enabled: true,
|
||||
})
|
||||
.partial(),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -139,17 +139,6 @@ class SearchSpacesApiService {
|
|||
return baseApiService.delete(`/api/v1/workspaces/${request.id}`, deleteSearchSpaceResponse);
|
||||
};
|
||||
|
||||
/**
|
||||
* Trigger AI file sorting for all documents in a search space
|
||||
*/
|
||||
triggerAiSort = async (searchSpaceId: number) => {
|
||||
return baseApiService.post(
|
||||
`/api/v1/workspaces/${searchSpaceId}/ai-sort`,
|
||||
z.object({ message: z.string() }),
|
||||
{}
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Leave a search space (remove own membership)
|
||||
* This is used by non-owners to leave a shared search space
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue