mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-04 21:32:39 +02:00
add kb sync for notion updates
This commit is contained in:
parent
ce110faa5a
commit
b240bc3e1a
3 changed files with 123 additions and 0 deletions
|
|
@ -218,6 +218,38 @@ def create_update_notion_page_tool(
|
||||||
logger.info(
|
logger.info(
|
||||||
f"update_page result: {result.get('status')} - {result.get('message', '')}"
|
f"update_page result: {result.get('status')} - {result.get('message', '')}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if result.get("status") == "success":
|
||||||
|
from app.services.notion import NotionKBSyncService
|
||||||
|
|
||||||
|
logger.info(f"Updating knowledge base for page {final_page_id}...")
|
||||||
|
kb_service = NotionKBSyncService(db_session)
|
||||||
|
kb_result = await kb_service.sync_after_update(
|
||||||
|
page_id=final_page_id,
|
||||||
|
search_space_id=search_space_id,
|
||||||
|
appended_content=final_content,
|
||||||
|
user_id=user_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
if kb_result["status"] == "success":
|
||||||
|
result["message"] = (
|
||||||
|
f"{result['message']}. Knowledge base updated - your search results now reflect the latest content."
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
f"Knowledge base successfully updated for page {final_page_id}"
|
||||||
|
)
|
||||||
|
elif kb_result["status"] == "not_indexed":
|
||||||
|
result["message"] = (
|
||||||
|
f"{result['message']}. (Note: This page hasn't been indexed yet, so it won't appear in search until the next scheduled indexing.)"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
result["message"] = (
|
||||||
|
f"{result['message']}. However, knowledge base update failed: {kb_result['message']}. The page will sync on the next scheduled indexing."
|
||||||
|
)
|
||||||
|
logger.warning(
|
||||||
|
f"KB update failed for page {final_page_id}: {kb_result['message']}"
|
||||||
|
)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
from app.services.notion.kb_sync_service import NotionKBSyncService
|
||||||
from app.services.notion.tool_metadata_service import (
|
from app.services.notion.tool_metadata_service import (
|
||||||
NotionAccount,
|
NotionAccount,
|
||||||
NotionPage,
|
NotionPage,
|
||||||
|
|
@ -6,6 +7,7 @@ from app.services.notion.tool_metadata_service import (
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"NotionAccount",
|
"NotionAccount",
|
||||||
|
"NotionKBSyncService",
|
||||||
"NotionPage",
|
"NotionPage",
|
||||||
"NotionToolMetadataService",
|
"NotionToolMetadataService",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
89
surfsense_backend/app/services/notion/kb_sync_service.py
Normal file
89
surfsense_backend/app/services/notion/kb_sync_service.py
Normal file
|
|
@ -0,0 +1,89 @@
|
||||||
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import delete
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
from sqlalchemy.future import select
|
||||||
|
|
||||||
|
from app.config import config
|
||||||
|
from app.db import Chunk, Document
|
||||||
|
from app.services.llm_service import get_user_long_context_llm
|
||||||
|
from app.tasks.connector_indexers.base import get_current_timestamp, safe_set_chunks
|
||||||
|
from app.utils.document_converters import (
|
||||||
|
create_document_chunks,
|
||||||
|
generate_document_summary,
|
||||||
|
)
|
||||||
|
from app.utils.hash_utils import generate_content_hash
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class NotionKBSyncService:
|
||||||
|
def __init__(self, db_session: AsyncSession):
|
||||||
|
self.db_session = db_session
|
||||||
|
|
||||||
|
async def sync_after_update(
|
||||||
|
self,
|
||||||
|
page_id: str,
|
||||||
|
search_space_id: int,
|
||||||
|
appended_content: str,
|
||||||
|
user_id: str,
|
||||||
|
) -> dict:
|
||||||
|
try:
|
||||||
|
result = await self.db_session.execute(
|
||||||
|
select(Document).filter(
|
||||||
|
Document.search_space_id == search_space_id,
|
||||||
|
Document.document_metadata["page_id"].astext == page_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
document = result.scalars().first()
|
||||||
|
|
||||||
|
if not document:
|
||||||
|
return {"status": "not_indexed"}
|
||||||
|
|
||||||
|
new_content = document.content + "\n\n" + appended_content
|
||||||
|
|
||||||
|
user_llm = await get_user_long_context_llm(
|
||||||
|
self.db_session, user_id, search_space_id
|
||||||
|
)
|
||||||
|
|
||||||
|
if user_llm:
|
||||||
|
document_metadata_for_summary = {
|
||||||
|
"page_title": document.document_metadata.get("page_title"),
|
||||||
|
"page_id": page_id,
|
||||||
|
"document_type": "Notion Page",
|
||||||
|
"connector_type": "Notion",
|
||||||
|
}
|
||||||
|
summary_content, summary_embedding = await generate_document_summary(
|
||||||
|
new_content, user_llm, document_metadata_for_summary
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
summary_content = f"Notion Page: {document.document_metadata.get('page_title')}\n\n{new_content[:500]}..."
|
||||||
|
summary_embedding = config.embedding_model_instance.embed(
|
||||||
|
summary_content
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.db_session.execute(
|
||||||
|
delete(Chunk).where(Chunk.document_id == document.id)
|
||||||
|
)
|
||||||
|
|
||||||
|
chunks = await create_document_chunks(new_content)
|
||||||
|
|
||||||
|
document.content = summary_content
|
||||||
|
document.content_hash = generate_content_hash(new_content)
|
||||||
|
document.embedding = summary_embedding
|
||||||
|
document.document_metadata = {
|
||||||
|
**document.document_metadata,
|
||||||
|
"indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
|
}
|
||||||
|
safe_set_chunks(document, chunks)
|
||||||
|
document.updated_at = get_current_timestamp()
|
||||||
|
|
||||||
|
await self.db_session.commit()
|
||||||
|
|
||||||
|
logger.info(f"Successfully synced KB for Notion page {page_id}")
|
||||||
|
return {"status": "success"}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to sync KB for page {page_id}: {e}", exc_info=True)
|
||||||
|
return {"status": "error", "message": str(e)}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue