apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-25 19:15:18 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1414 from AnishSarkar22/feature/memory-support-document-panel

Browse source 
feat: improve memory extraction & add document-panel memory editing
This commit is contained in:
Rohan Verma 2026-05-20 12:12:27 -07:00 • committed by GitHub
commit 55cce4ea59
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
56 changed files with 1898 additions and 2185 deletions
Download patch file Download diff file Expand all files Collapse all files

6
surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/private.md
View file

@ -6,4 +6,10 @@ standing instructions?
If yes, call `update_memory` **alongside** your normal response — don't
defer it to a later turn. Skip ephemeral chat noise (one-off Q/A, greetings,
session logistics). Stay within the budget shown in `<user_memory>`.
Memory is heading-based markdown. New entries should be under `##` headings
such as `## Facts`, `## Preferences`, or `## Instructions`, with bullets like
`- YYYY-MM-DD: text`. If existing memory contains legacy
`(YYYY-MM-DD) [fact|pref|instr]` markers, preserve the information but write
new saves in the heading-based format.
</memory_protocol>

8
surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/team.md
View file

@ -6,4 +6,12 @@ key facts?
If yes, call `update_memory` **alongside** your normal response — don't
defer it to a later turn. Skip ephemeral chat noise (one-off Q/A, greetings,
session logistics). Stay within the budget shown in `<team_memory>`.
Team memory is heading-based markdown. New entries should be under `##`
headings such as `## Product Decisions`, `## Engineering Conventions`,
`## Project Facts`, or `## Open Questions`, with bullets like
`- YYYY-MM-DD: text`. If existing memory contains legacy `(YYYY-MM-DD) [fact]`
markers, preserve the information but write new saves in the heading-based
format. Do not create personal headings such as `## Preferences` or
`## Instructions`.
</memory_protocol>

10
surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/description.md
View file

@ -9,7 +9,9 @@
- Skip ephemeral chat noise (one-off Q/A, greetings, session logistics).
- Args: `updated_memory` — FULL replacement markdown (merge and curate,
don't only append).
- Formatting: bullets `- (YYYY-MM-DD) [marker] text` with markers `[fact]`,
`[pref]`, `[instr]` (priority when trimming: `instr > pref > fact`).
Group bullets under short `##` headings; stay under the limit shown in
`<user_memory>`.
- Formatting: heading-based markdown with entries under `##` headings.
Recommended headings are `## Facts`, `## Preferences`, `## Instructions`,
though clearer natural headings are allowed. New bullets should look like
`- YYYY-MM-DD: text`; stay under the limit shown in `<user_memory>`.
- If existing memory uses legacy `(YYYY-MM-DD) [fact|pref|instr]` markers,
preserve the information but write the updated document in the new format.

10
surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/example.md
View file

@ -1,28 +1,28 @@
<example>
<user_name>Alex</user_name>, <user_memory> is empty.
user: "I'm a space enthusiast, explain astrophage to me"
→ update_memory(updated_memory="## Interests & background\n- (2025-03-15) [fact] Alex is a space enthusiast\n")
→ update_memory(updated_memory="## Facts\n- 2025-03-15: Alex is a space enthusiast\n")
(Casual durable fact; use first name, neutral heading.)
</example>
<example>
user: "Remember that I prefer concise answers over detailed explanations"
→ update_memory(updated_memory="## Interests & background\n- (2025-03-15) [fact] Alex is a space enthusiast\n\n## Response style\n- (2025-03-15) [pref] Alex prefers concise answers over detailed explanations\n")
→ update_memory(updated_memory="## Facts\n- 2025-03-15: Alex is a space enthusiast\n\n## Preferences\n- 2025-03-15: Alex prefers concise answers over detailed explanations\n")
(Durable preference; merge with existing memory.)
</example>
<example>
user: "I actually moved to Tokyo last month"
→ update_memory(updated_memory="...\n\n## Personal context\n- (2025-03-15) [fact] Alex lives in Tokyo (previously London)\n...")
→ update_memory(updated_memory="...\n\n## Facts\n- 2025-03-15: Alex lives in Tokyo (previously London)\n...")
(Updated fact; date reflects when recorded.)
</example>
<example>
user: "I'm a freelance photographer working on a nature documentary"
→ update_memory(updated_memory="...\n\n## Current focus\n- (2025-03-15) [fact] Alex is a freelance photographer\n- (2025-03-15) [fact] Alex is working on a nature documentary\n")
→ update_memory(updated_memory="...\n\n## Current Focus\n- 2025-03-15: Alex is a freelance photographer\n- 2025-03-15: Alex is working on a nature documentary\n")
</example>
<example>
user: "Always respond in bullet points"
→ update_memory(updated_memory="...\n\n## Response style\n- (2025-03-15) [instr] Always respond to Alex in bullet points\n")
→ update_memory(updated_memory="...\n\n## Instructions\n- 2025-03-15: Always respond to Alex in bullet points\n")
</example>

16
surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/description.md
View file

@ -9,8 +9,14 @@
- Skip ephemeral chat noise (one-off Q/A, greetings, session logistics).
- Args: `updated_memory` — FULL replacement markdown (merge and curate,
don't only append).
- Formatting: bullets `- (YYYY-MM-DD) [fact] text`. Team memory uses ONLY
the `[fact]` marker (never `[pref]` or `[instr]`). Group bullets under
short `##` headings (2-3 words each); stay under the limit shown in
`<team_memory>`. When trimming, prioritise: decisions/conventions > key
facts > current priorities.
- Formatting: heading-based markdown with entries under `##` headings.
Recommended headings are `## Product Decisions`,
`## Engineering Conventions`, `## Project Facts`, and `## Open Questions`.
New bullets should look like `- YYYY-MM-DD: text`; stay under the limit
shown in `<team_memory>`.
- If existing memory uses legacy `(YYYY-MM-DD) [fact]` markers, preserve the
information but write the updated document in the new format.
- Do not create personal headings such as `## Preferences`,
`## Instructions`, `## Personal Notes`, or `## Personal Instructions`.
When trimming, prioritise: decisions/conventions > key facts > current
priorities.

4
surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/example.md
View file

@ -1,9 +1,9 @@
<example>
user: "Let's remember that we decided to do weekly standup meetings on Mondays"
→ update_memory(updated_memory="...\n\n## Team rituals\n- (2025-03-15) [fact] Weekly standup meetings on Mondays\n...")
→ update_memory(updated_memory="...\n\n## Product Decisions\n- 2025-03-15: Weekly standup meetings happen on Mondays\n...")
</example>
<example>
user: "Our office is in downtown Seattle, 5th floor"
→ update_memory(updated_memory="...\n\n## Workspace\n- (2025-03-15) [fact] Office location: downtown Seattle, 5th floor\n...")
→ update_memory(updated_memory="...\n\n## Project Facts\n- 2025-03-15: Office location is downtown Seattle, 5th floor\n...")
</example>

7
surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/system_prompt.md
View file

@ -18,6 +18,10 @@ Persist durable preferences/facts/instructions with `update_memory` while avoidi
- Do not store transient chatter.
- Do not store secrets unless explicitly instructed.
- If memory intent is unclear, return `status=blocked` with the missing intent signal.
- Persisted memory is heading-based markdown. New saved bullets should look like
`- YYYY-MM-DD: text` under `##` headings. If existing memory has legacy
`(YYYY-MM-DD) [fact|pref|instr]` markers, preserve the information but write
the updated document in the heading-based format.
</tool_policy>
<out_of_scope>
@ -53,4 +57,7 @@ Rules:
- `status=success` -> `next_step=null`, `missing_fields=null`.
- `status=partial|blocked|error` -> `next_step` must be non-null.
- `status=blocked` due to missing required inputs -> `missing_fields` must be non-null.
- `evidence.memory_category` is a semantic classification for supervisor logs
only. It is not the persisted storage format and must not force inline
`[fact|preference|instruction]` markers into saved memory.
</output_contract>

351
surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/tools/update_memory.py
View file

@ -1,280 +1,23 @@
"""Overwrite one markdown memory document per user or team, with size and shrink guards."""
"""Memory update tools backed by the canonical memory service."""
from __future__ import annotations
import logging
import re
from typing import Any, Literal
from typing import Any
from uuid import UUID
from langchain_core.messages import HumanMessage
from langchain_core.tools import tool
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import SearchSpace, User
from app.services.memory import (
MEMORY_HARD_LIMIT,
MEMORY_SOFT_LIMIT,
MemoryScope,
save_memory,
)
logger = logging.getLogger(__name__)
MEMORY_SOFT_LIMIT = 18_000
MEMORY_HARD_LIMIT = 25_000
_SECTION_HEADING_RE = re.compile(r"^##\s+(.+)$", re.MULTILINE)
_HEADING_NORMALIZE_RE = re.compile(r"\s+")
_MARKER_RE = re.compile(r"\[(fact|pref|instr)\]")
_BULLET_FORMAT_RE = re.compile(r"^- \(\d{4}-\d{2}-\d{2}\) \[(fact|pref|instr)\] .+$")
_PERSONAL_ONLY_MARKERS = {"pref", "instr"}
# ---------------------------------------------------------------------------
# Diff validation
# ---------------------------------------------------------------------------
def _extract_headings(memory: str) -> set[str]:
"""Return all ``## …`` heading texts (without the ``## `` prefix)."""
return set(_SECTION_HEADING_RE.findall(memory))
def _normalize_heading(heading: str) -> str:
"""Normalize heading text for robust scope checks."""
return _HEADING_NORMALIZE_RE.sub(" ", heading.strip().lower())
def _validate_memory_scope(
content: str, scope: Literal["user", "team"]
) -> dict[str, Any] | None:
"""Reject personal-only markers ([pref], [instr]) in team memory."""
if scope != "team":
return None
markers = set(_MARKER_RE.findall(content))
leaked = sorted(markers & _PERSONAL_ONLY_MARKERS)
if leaked:
tags = ", ".join(f"[{m}]" for m in leaked)
return {
"status": "error",
"message": (
f"Team memory cannot include personal markers: {tags}. "
"Use [fact] only in team memory."
),
}
return None
def _validate_bullet_format(content: str) -> list[str]:
"""Return warnings for bullet lines that don't match the required format.
Expected: ``- (YYYY-MM-DD) [fact|pref|instr] text``
"""
warnings: list[str] = []
for line in content.splitlines():
stripped = line.strip()
if not stripped.startswith("- "):
continue
if not _BULLET_FORMAT_RE.match(stripped):
short = stripped[:80] + ("..." if len(stripped) > 80 else "")
warnings.append(f"Malformed bullet: {short}")
return warnings
def _validate_diff(old_memory: str | None, new_memory: str) -> list[str]:
"""Return a list of warning strings about suspicious changes."""
if not old_memory:
return []
warnings: list[str] = []
old_headings = _extract_headings(old_memory)
new_headings = _extract_headings(new_memory)
dropped = old_headings - new_headings
if dropped:
names = ", ".join(sorted(dropped))
warnings.append(
f"Sections removed: {names}. "
"If unintentional, the user can restore from the settings page."
)
old_len = len(old_memory)
new_len = len(new_memory)
if old_len > 0 and new_len < old_len * 0.4:
warnings.append(
f"Memory shrank significantly ({old_len:,} -> {new_len:,} chars). "
"Possible data loss."
)
return warnings
# ---------------------------------------------------------------------------
# Size validation & soft warning
# ---------------------------------------------------------------------------
def _validate_memory_size(content: str) -> dict[str, Any] | None:
"""Return an error/warning dict if *content* is too large, else None."""
length = len(content)
if length > MEMORY_HARD_LIMIT:
return {
"status": "error",
"message": (
f"Memory exceeds {MEMORY_HARD_LIMIT:,} character limit "
f"({length:,} chars). Consolidate by merging related items, "
"removing outdated entries, and shortening descriptions. "
"Then call update_memory again."
),
}
return None
def _soft_warning(content: str) -> str | None:
"""Return a warning string if content exceeds the soft limit."""
length = len(content)
if length > MEMORY_SOFT_LIMIT:
return (
f"Memory is at {length:,}/{MEMORY_HARD_LIMIT:,} characters. "
"Consolidate by merging related items and removing less important "
"entries on your next update."
)
return None
# ---------------------------------------------------------------------------
# Forced rewrite when memory exceeds the hard limit
# ---------------------------------------------------------------------------
_FORCED_REWRITE_PROMPT = """\
You are a memory curator. The following memory document exceeds the character \
limit and must be shortened.
RULES:
1. Rewrite the document to be under {target} characters.
2. Preserve existing ## headings. Every entry must remain under a heading. You may merge
or rename headings to consolidate, but keep names personal and descriptive.
3. Priority for keeping content: [instr] > [pref] > [fact].
4. Merge duplicate entries, remove outdated entries, shorten verbose descriptions.
5. Every bullet MUST have format: - (YYYY-MM-DD) [fact|pref|instr] text
6. Preserve the user's first name in entries — do not replace it with "the user".
7. Output ONLY the consolidated markdown — no explanations, no wrapping.
<memory_document>
{content}
</memory_document>"""
async def _forced_rewrite(content: str, llm: Any) -> str | None:
"""Use a focused LLM call to compress *content* under the hard limit.
Returns the rewritten string, or ``None`` if the call fails.
"""
try:
prompt = _FORCED_REWRITE_PROMPT.format(
target=MEMORY_HARD_LIMIT, content=content
)
response = await llm.ainvoke(
[HumanMessage(content=prompt)],
config={"tags": ["surfsense:internal"]},
)
text = (
response.content
if isinstance(response.content, str)
else str(response.content)
)
return text.strip()
except Exception:
logger.exception("Forced rewrite LLM call failed")
return None
# ---------------------------------------------------------------------------
# Shared save-and-respond logic
# ---------------------------------------------------------------------------
async def _save_memory(
*,
updated_memory: str,
old_memory: str | None,
llm: Any | None,
apply_fn,
commit_fn,
rollback_fn,
label: str,
scope: Literal["user", "team"],
) -> dict[str, Any]:
"""Validate, optionally force-rewrite if over the hard limit, save, and
return a response dict.
Parameters
----------
updated_memory : str
The new document the agent submitted.
old_memory : str | None
The previously persisted document (for diff checks).
llm : Any | None
LLM instance for forced rewrite (may be ``None``).
apply_fn : callable(str) -> None
Callback that sets the new memory on the ORM object.
commit_fn : coroutine
``session.commit``.
rollback_fn : coroutine
``session.rollback``.
label : str
Human label for log messages (e.g. "user memory", "team memory").
"""
content = updated_memory
# --- forced rewrite if over the hard limit ---
if len(content) > MEMORY_HARD_LIMIT and llm is not None:
rewritten = await _forced_rewrite(content, llm)
if rewritten is not None and len(rewritten) < len(content):
content = rewritten
# --- hard-limit gate (reject if still too large after rewrite) ---
size_err = _validate_memory_size(content)
if size_err:
return size_err
scope_err = _validate_memory_scope(content, scope)
if scope_err:
return scope_err
# --- persist ---
try:
apply_fn(content)
await commit_fn()
except Exception as e:
logger.exception("Failed to update %s: %s", label, e)
await rollback_fn()
return {"status": "error", "message": f"Failed to update {label}: {e}"}
# --- build response ---
resp: dict[str, Any] = {
"status": "saved",
"message": f"{label.capitalize()} updated.",
}
if content is not updated_memory:
resp["notice"] = "Memory was automatically rewritten to fit within limits."
diff_warnings = _validate_diff(old_memory, content)
if diff_warnings:
resp["diff_warnings"] = diff_warnings
format_warnings = _validate_bullet_format(content)
if format_warnings:
resp["format_warnings"] = format_warnings
warning = _soft_warning(content)
if warning:
resp["warning"] = warning
return resp
# ---------------------------------------------------------------------------
# Tool factories
# ---------------------------------------------------------------------------
def create_update_memory_tool(
user_id: str | UUID,
@ -287,40 +30,22 @@ def create_update_memory_tool(
async def update_memory(updated_memory: str) -> dict[str, Any]:
"""Update the user's personal memory document.
Your current memory is shown in <user_memory> in the system prompt.
When the user shares important long-term information (preferences,
facts, instructions, context), rewrite the memory document to include
the new information. Merge new facts with existing ones, update
contradictions, remove outdated entries, and keep it concise.
Args:
updated_memory: The FULL updated markdown document (not a diff).
The current memory is shown in <user_memory>. Pass the FULL updated
markdown document, not a diff.
"""
try:
result = await db_session.execute(select(User).where(User.id == uid))
user = result.scalars().first()
if not user:
return {"status": "error", "message": "User not found."}
old_memory = user.memory_md
return await _save_memory(
updated_memory=updated_memory,
old_memory=old_memory,
result = await save_memory(
scope=MemoryScope.USER,
target_id=uid,
content=updated_memory,
session=db_session,
llm=llm,
apply_fn=lambda content: setattr(user, "memory_md", content),
commit_fn=db_session.commit,
rollback_fn=db_session.rollback,
label="memory",
scope="user",
)
return result.to_dict()
except Exception as e:
logger.exception("Failed to update user memory: %s", e)
await db_session.rollback()
return {
"status": "error",
"message": f"Failed to update memory: {e}",
}
return {"status": "error", "message": f"Failed to update memory: {e}"}
return update_memory
@ -334,36 +59,18 @@ def create_update_team_memory_tool(
async def update_memory(updated_memory: str) -> dict[str, Any]:
"""Update the team's shared memory document for this search space.
Your current team memory is shown in <team_memory> in the system
prompt. When the team shares important long-term information
(decisions, conventions, key facts, priorities), rewrite the memory
document to include the new information. Merge new facts with
existing ones, update contradictions, remove outdated entries, and
keep it concise.
Args:
updated_memory: The FULL updated markdown document (not a diff).
The current team memory is shown in <team_memory>. Pass the FULL updated
markdown document, not a diff.
"""
try:
result = await db_session.execute(
select(SearchSpace).where(SearchSpace.id == search_space_id)
)
space = result.scalars().first()
if not space:
return {"status": "error", "message": "Search space not found."}
old_memory = space.shared_memory_md
return await _save_memory(
updated_memory=updated_memory,
old_memory=old_memory,
result = await save_memory(
scope=MemoryScope.TEAM,
target_id=search_space_id,
content=updated_memory,
session=db_session,
llm=llm,
apply_fn=lambda content: setattr(space, "shared_memory_md", content),
commit_fn=db_session.commit,
rollback_fn=db_session.rollback,
label="team memory",
scope="team",
)
return result.to_dict()
except Exception as e:
logger.exception("Failed to update team memory: %s", e)
await db_session.rollback()
@ -373,3 +80,11 @@ def create_update_team_memory_tool(
}
return update_memory
__all__ = [
"MEMORY_HARD_LIMIT",
"MEMORY_SOFT_LIMIT",
"create_update_memory_tool",
"create_update_team_memory_tool",
]

232
surfsense_backend/app/agents/new_chat/memory_extraction.py
View file

@ -1,232 +0,0 @@
"""Background memory extraction for the SurfSense agent.
After each agent response, if the agent did not call ``update_memory`` during
the turn, this module can run a lightweight LLM call to decide whether the
latest message contains long-term information worth persisting.
"""
from __future__ import annotations
import logging
from typing import Any
from uuid import UUID
from langchain_core.messages import HumanMessage
from sqlalchemy import select
from app.agents.new_chat.tools.update_memory import _save_memory
from app.db import SearchSpace, User, shielded_async_session
from app.utils.content_utils import extract_text_content
logger = logging.getLogger(__name__)
_MEMORY_EXTRACT_PROMPT = """\
You are a memory extraction assistant. Analyze the user's message and decide \
if it contains any long-term information worth persisting to memory.
Worth remembering: preferences, background/identity, goals, projects, \
instructions, tools/languages they use, decisions, expertise, workplace — \
durable facts that will matter in future conversations.
NOT worth remembering: greetings, one-off factual questions, session \
logistics, ephemeral requests, follow-up clarifications with no new personal \
info, things that only matter for the current task.
If the message contains memorizable information, output the FULL updated \
memory document with the new facts merged into the existing content. Follow \
these rules:
- Every entry MUST be under a ## heading. Preserve existing headings; create new ones
freely. Keep heading names short (2-3 words) and natural. Do NOT include the user's
name in headings.
- Keep entries as single bullet points. Be descriptive but concise — include relevant
details and context rather than just a few words.
- Every bullet MUST use format: - (YYYY-MM-DD) [fact|pref|instr] text
[fact] = durable facts, [pref] = preferences, [instr] = standing instructions.
- Use the user's first name (from <user_name>) in entry text, not "the user".
- If a new fact contradicts an existing entry, update the existing entry.
- Do not duplicate information that is already present.
If nothing is worth remembering, output exactly: NO_UPDATE
<user_name>{user_name}</user_name>
<current_memory>
{current_memory}
</current_memory>
<user_message>
{user_message}
</user_message>"""
_TEAM_MEMORY_EXTRACT_PROMPT = """\
You are a team-memory extraction assistant. Analyze the latest message and \
decide if it contains durable TEAM-level information worth persisting.
Decision policy:
- Prioritize recall for durable team context, while avoiding personal-only facts.
- Do NOT require explicit consensus language. A direct team-level statement can
be stored if it is stable and broadly useful for future team chats.
- If evidence is weak or clearly tentative, output NO_UPDATE.
Worth remembering (team-level only):
- Decisions and defaults that guide future team work
- Team conventions/standards (naming, review policy, coding norms)
- Stable org/project facts (locations, ownership, constraints)
- Long-lived architecture/process facts
- Ongoing priorities that are likely relevant beyond this turn
NOT worth remembering:
- Personal preferences or biography of one person
- Questions, brainstorming, tentative ideas, or speculation
- One-off requests, status updates, TODOs, logistics for this session
- Information scoped only to a single ephemeral task
If the message contains memorizable team information, output the FULL updated \
team memory document with new facts merged into existing content. Follow rules:
- Every entry MUST be under a ## heading. Preserve existing headings; create new ones
freely. Keep heading names short (2-3 words) and natural.
- Keep entries as single bullet points. Be descriptive but concise — include relevant
details and context rather than just a few words.
- Every bullet MUST use format: - (YYYY-MM-DD) [fact] text
Team memory uses ONLY the [fact] marker. Never use [pref] or [instr].
- If a new fact contradicts an existing entry, update the existing entry.
- Do not duplicate existing information.
- Preserve neutral team phrasing; avoid person-specific memory unless role-anchored.
If nothing is worth remembering, output exactly: NO_UPDATE
<current_team_memory>
{current_memory}
</current_team_memory>
<latest_message_author>
{author}
</latest_message_author>
<latest_message>
{user_message}
</latest_message>"""
async def extract_and_save_memory(
*,
user_message: str,
user_id: str | None,
llm: Any,
) -> None:
"""Background task: extract memorizable info and persist it.
Designed to be fire-and-forget — catches all exceptions internally.
"""
if not user_id:
return
try:
uid = UUID(user_id) if isinstance(user_id, str) else user_id
async with shielded_async_session() as session:
result = await session.execute(select(User).where(User.id == uid))
user = result.scalars().first()
if not user:
return
old_memory = user.memory_md
first_name = (
user.display_name.strip().split()[0]
if user.display_name and user.display_name.strip()
else "The user"
)
prompt = _MEMORY_EXTRACT_PROMPT.format(
current_memory=old_memory or "(empty)",
user_message=user_message,
user_name=first_name,
)
response = await llm.ainvoke(
[HumanMessage(content=prompt)],
config={"tags": ["surfsense:internal", "memory-extraction"]},
)
text = extract_text_content(response.content).strip()
if text == "NO_UPDATE" or not text:
logger.debug("Memory extraction: no update needed (user %s)", uid)
return
save_result = await _save_memory(
updated_memory=text,
old_memory=old_memory,
llm=llm,
apply_fn=lambda content: setattr(user, "memory_md", content),
commit_fn=session.commit,
rollback_fn=session.rollback,
label="memory",
scope="user",
)
logger.info(
"Background memory extraction for user %s: %s",
uid,
save_result.get("status"),
)
except Exception:
logger.exception("Background user memory extraction failed")
async def extract_and_save_team_memory(
*,
user_message: str,
search_space_id: int | None,
llm: Any,
author_display_name: str | None = None,
) -> None:
"""Background task: extract team-level memory and persist it.
Runs only for shared threads. Designed to be fire-and-forget and catches
exceptions internally.
"""
if not search_space_id:
return
try:
async with shielded_async_session() as session:
result = await session.execute(
select(SearchSpace).where(SearchSpace.id == search_space_id)
)
space = result.scalars().first()
if not space:
return
old_memory = space.shared_memory_md
prompt = _TEAM_MEMORY_EXTRACT_PROMPT.format(
current_memory=old_memory or "(empty)",
author=author_display_name or "Unknown team member",
user_message=user_message,
)
response = await llm.ainvoke(
[HumanMessage(content=prompt)],
config={"tags": ["surfsense:internal", "team-memory-extraction"]},
)
text = extract_text_content(response.content).strip()
if text == "NO_UPDATE" or not text:
logger.debug(
"Team memory extraction: no update needed (space %s)",
search_space_id,
)
return
save_result = await _save_memory(
updated_memory=text,
old_memory=old_memory,
llm=llm,
apply_fn=lambda content: setattr(space, "shared_memory_md", content),
commit_fn=session.commit,
rollback_fn=session.rollback,
label="team memory",
scope="team",
)
logger.info(
"Background team memory extraction for space %s: %s",
search_space_id,
save_result.get("status"),
)
except Exception:
logger.exception("Background team memory extraction failed")

2
surfsense_backend/app/agents/new_chat/middleware/memory_injection.py
View file

@ -17,8 +17,8 @@ from langgraph.runtime import Runtime
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.agents.new_chat.tools.update_memory import MEMORY_HARD_LIMIT, MEMORY_SOFT_LIMIT
from app.db import ChatVisibility, SearchSpace, User, shielded_async_session
from app.services.memory import MEMORY_HARD_LIMIT, MEMORY_SOFT_LIMIT
logger = logging.getLogger(__name__)

6
surfsense_backend/app/agents/new_chat/prompts/base/memory_protocol_private.md
View file

@ -3,4 +3,10 @@ IMPORTANT — After understanding each user message, ALWAYS check: does this mes
reveal durable facts about the user (role, interests, preferences, projects,
background, or standing instructions)? If yes, you MUST call update_memory
alongside your normal response — do not defer this to a later turn.
Memory is stored as a heading-based markdown document. New entries should be
under `##` headings such as `## Facts`, `## Preferences`, or `## Instructions`
with bullets like `- YYYY-MM-DD: text`. If existing memory contains legacy
`(YYYY-MM-DD) [fact|pref|instr]` markers, preserve the information but write
new saves in the heading-based format.
</memory_protocol>

8
surfsense_backend/app/agents/new_chat/prompts/base/memory_protocol_team.md
View file

@ -3,4 +3,12 @@ IMPORTANT — After understanding each user message, ALWAYS check: does this mes
reveal durable facts about the team (decisions, conventions, architecture, processes,
or key facts)? If yes, you MUST call update_memory alongside your normal response —
do not defer this to a later turn.
Team memory is stored as a heading-based markdown document. New entries should
be under `##` headings such as `## Product Decisions`,
`## Engineering Conventions`, `## Project Facts`, or `## Open Questions` with
bullets like `- YYYY-MM-DD: text`. If existing memory contains legacy
`(YYYY-MM-DD) [fact]` markers, preserve the information but write new saves in
the heading-based format. Do not create personal headings such as
`## Preferences` or `## Instructions`.
</memory_protocol>

14
surfsense_backend/app/agents/new_chat/prompts/examples/update_memory_private.md
View file

@ -1,16 +1,16 @@
- <user_name>Alex</user_name>, <user_memory> is empty. User: "I'm a space enthusiast, explain astrophage to me"
- The user casually shared a durable fact. Use their first name in the entry, short neutral heading:
update_memory(updated_memory="## Interests & background\n- (2025-03-15) [fact] Alex is a space enthusiast\n")
- The user casually shared a durable fact:
update_memory(updated_memory="## Facts\n- 2025-03-15: Alex is a space enthusiast\n")
- User: "Remember that I prefer concise answers over detailed explanations"
- Durable preference. Merge with existing memory, add a new heading:
update_memory(updated_memory="## Interests & background\n- (2025-03-15) [fact] Alex is a space enthusiast\n\n## Response style\n- (2025-03-15) [pref] Alex prefers concise answers over detailed explanations\n")
- Durable preference. Merge with existing memory:
update_memory(updated_memory="## Facts\n- 2025-03-15: Alex is a space enthusiast\n\n## Preferences\n- 2025-03-15: Alex prefers concise answers over detailed explanations\n")
- User: "I actually moved to Tokyo last month"
- Updated fact, date prefix reflects when recorded:
update_memory(updated_memory="## Interests & background\n...\n\n## Personal context\n- (2025-03-15) [fact] Alex lives in Tokyo (previously London)\n...")
update_memory(updated_memory="## Facts\n- 2025-03-15: Alex lives in Tokyo (previously London)\n...")
- User: "I'm a freelance photographer working on a nature documentary"
- Durable background info under a fitting heading:
update_memory(updated_memory="...\n\n## Current focus\n- (2025-03-15) [fact] Alex is a freelance photographer\n- (2025-03-15) [fact] Alex is working on a nature documentary\n")
update_memory(updated_memory="...\n\n## Current Focus\n- 2025-03-15: Alex is a freelance photographer\n- 2025-03-15: Alex is working on a nature documentary\n")
- User: "Always respond in bullet points"
- Standing instruction:
update_memory(updated_memory="...\n\n## Response style\n- (2025-03-15) [instr] Always respond to Alex in bullet points\n")
update_memory(updated_memory="...\n\n## Instructions\n- 2025-03-15: Always respond to Alex in bullet points\n")

4
surfsense_backend/app/agents/new_chat/prompts/examples/update_memory_team.md
View file

@ -1,7 +1,7 @@
- User: "Let's remember that we decided to do weekly standup meetings on Mondays"
- Durable team decision:
update_memory(updated_memory="- (2025-03-15) [fact] Weekly standup meetings on Mondays\n...")
update_memory(updated_memory="## Product Decisions\n- 2025-03-15: Weekly standup meetings happen on Mondays\n...")
- User: "Our office is in downtown Seattle, 5th floor"
- Durable team fact:
update_memory(updated_memory="- (2025-03-15) [fact] Office location: downtown Seattle, 5th floor\n...")
update_memory(updated_memory="## Project Facts\n- 2025-03-15: Office location is downtown Seattle, 5th floor\n...")

51
surfsense_backend/app/agents/new_chat/prompts/tools/update_memory_private.md
View file

@ -1,31 +1,26 @@
- update_memory: Update your personal memory document about the user.
- Your current memory is already in <user_memory> in your context. The `chars` and
`limit` attributes show your current usage and the maximum allowed size.
- This is your curated long-term memory — the distilled essence of what you know about
the user, not raw conversation logs.
- Call update_memory when:
* The user explicitly asks to remember or forget something
* The user shares durable facts or preferences that will matter in future conversations
- The user's first name is provided in <user_name>. Use it in memory entries
instead of "the user" (e.g. "{name} works at..." not "The user works at...").
Do not store the name itself as a separate memory entry.
- Do not store short-lived or ephemeral info: one-off questions, greetings,
session logistics, or things that only matter for the current task.
- Your current memory is already in <user_memory> in your context. The `chars`
and `limit` attributes show current usage and the maximum allowed size.
- This is curated long-term memory, not raw conversation logs.
- Call update_memory when the user explicitly asks to remember/forget
something or shares durable facts, preferences, or standing instructions.
- The user's first name is provided in <user_name>. Use it in entries instead
of "the user" when helpful. Do not store the name alone as a memory entry.
- Do not store short-lived info: one-off questions, greetings, session
logistics, or things that only matter for the current task.
- Args:
- updated_memory: The FULL updated markdown document (not a diff).
Merge new facts with existing ones, update contradictions, remove outdated entries.
Treat every update as a curation pass — consolidate, don't just append.
- Every bullet MUST use this format: - (YYYY-MM-DD) [marker] text
Markers:
[fact] — durable facts (role, background, projects, tools, expertise)
[pref] — preferences (response style, languages, formats, tools)
[instr] — standing instructions (always/never do, response rules)
- Keep it concise and well under the character limit shown in <user_memory>.
- Every entry MUST be under a `##` heading. Keep heading names short (2-3 words) and
natural. Do NOT include the user's name in headings. Organize by context — e.g.
who they are, what they're focused on, how they prefer things. Create, split, or
merge headings freely as the memory grows.
- Each entry MUST be a single bullet point. Be descriptive but concise — include relevant
details and context rather than just a few words.
- During consolidation, prioritize keeping: [instr] > [pref] > [fact].
- updated_memory: The FULL updated markdown document, not a diff. Merge new
facts with existing ones, update contradictions, remove outdated entries,
and consolidate instead of only appending.
- Use heading-based Markdown:
* Every entry must be under a `##` heading.
* Recommended headings: `## Facts`, `## Preferences`, `## Instructions`.
Specific natural headings are allowed when clearer.
* New bullets should use `- YYYY-MM-DD: text`.
* Each entry should be one concise but descriptive bullet.
- If existing memory uses legacy `(YYYY-MM-DD) [fact|pref|instr]` markers,
preserve the information but write the updated document in the new
heading-based format.
- During consolidation, prioritize durable instructions and preferences before
generic facts.

48
surfsense_backend/app/agents/new_chat/prompts/tools/update_memory_team.md
View file

@ -1,26 +1,28 @@
- update_memory: Update the team's shared memory document for this search space.
- Your current team memory is already in <team_memory> in your context. The `chars`
and `limit` attributes show current usage and the maximum allowed size.
- This is the team's curated long-term memory — decisions, conventions, key facts.
- NEVER store personal memory in team memory (e.g. personal bio, individual
preferences, or user-only standing instructions).
- Call update_memory when:
* A team member explicitly asks to remember or forget something
* The conversation surfaces durable team decisions, conventions, or facts
that will matter in future conversations
- Do not store short-lived or ephemeral info: one-off questions, greetings,
session logistics, or things that only matter for the current task.
- Your current team memory is already in <team_memory> in your context. The
`chars` and `limit` attributes show current usage and the maximum allowed size.
- This is curated long-term team memory: decisions, conventions, architecture,
processes, and key shared facts.
- NEVER store personal memory in team memory: individual bios, personal
preferences, or user-only standing instructions.
- Call update_memory when a team member asks to remember/forget something, or
when the conversation surfaces durable team context that matters later.
- Do not store short-lived info: one-off questions, greetings, session
logistics, or things that only matter for the current task.
- Args:
- updated_memory: The FULL updated markdown document (not a diff).
Merge new facts with existing ones, update contradictions, remove outdated entries.
Treat every update as a curation pass — consolidate, don't just append.
- Every bullet MUST use this format: - (YYYY-MM-DD) [fact] text
Team memory uses ONLY the [fact] marker. Never use [pref] or [instr] in team memory.
- Keep it concise and well under the character limit shown in <team_memory>.
- Every entry MUST be under a `##` heading. Keep heading names short (2-3 words) and
natural. Organize by context — e.g. what the team decided, current architecture,
active processes. Create, split, or merge headings freely as the memory grows.
- Each entry MUST be a single bullet point. Be descriptive but concise — include relevant
details and context rather than just a few words.
- During consolidation, prioritize keeping: decisions/conventions > key facts > current priorities.
- updated_memory: The FULL updated markdown document, not a diff. Merge new
facts with existing ones, update contradictions, remove outdated entries,
and consolidate instead of only appending.
- Use heading-based Markdown:
* Every entry must be under a `##` heading.
* Recommended headings: `## Product Decisions`, `## Engineering Conventions`,
`## Project Facts`, `## Open Questions`.
* New bullets should use `- YYYY-MM-DD: text`.
* Each entry should be one concise but descriptive bullet.
- If existing memory uses legacy `(YYYY-MM-DD) [fact]` markers, preserve the
information but write the updated document in the new heading-based format.
- Do not create personal headings such as `## Preferences`, `## Instructions`,
`## Personal Notes`, or `## Personal Instructions`.
- During consolidation, prioritize decisions/conventions, then key facts, then
current priorities.

414
surfsense_backend/app/agents/new_chat/tools/update_memory.py
View file

@ -1,369 +1,53 @@
"""Markdown-document memory tool for the SurfSense agent.
Replaces the old row-per-fact save_memory / recall_memory tools with a single
update_memory tool that overwrites a freeform markdown TEXT column. The LLM
always sees the current memory in <user_memory> / <team_memory> tags injected
by MemoryInjectionMiddleware, so it passes the FULL updated document each time.
Overflow handling:
- Soft limit (18K chars): a warning is returned telling the agent to
consolidate on the next update.
- Hard limit (25K chars): a forced LLM-driven rewrite compresses the document.
If it still exceeds the limit after rewriting, the save is rejected.
- Diff validation: warns when entire ``##`` sections are dropped or when the
document shrinks by more than 60%.
"""
"""Memory update tools backed by the canonical memory service."""
from __future__ import annotations
import logging
import re
from typing import Any, Literal
from typing import Any
from uuid import UUID
from langchain_core.messages import HumanMessage
from langchain_core.tools import tool
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import SearchSpace, User, async_session_maker
from app.utils.content_utils import extract_text_content
from app.db import async_session_maker
from app.services.memory import MemoryScope, save_memory
logger = logging.getLogger(__name__)
MEMORY_SOFT_LIMIT = 18_000
MEMORY_HARD_LIMIT = 25_000
_SECTION_HEADING_RE = re.compile(r"^##\s+(.+)$", re.MULTILINE)
_HEADING_NORMALIZE_RE = re.compile(r"\s+")
_MARKER_RE = re.compile(r"\[(fact|pref|instr)\]")
_BULLET_FORMAT_RE = re.compile(r"^- \(\d{4}-\d{2}-\d{2}\) \[(fact|pref|instr)\] .+$")
_PERSONAL_ONLY_MARKERS = {"pref", "instr"}
# ---------------------------------------------------------------------------
# Diff validation
# ---------------------------------------------------------------------------
def _extract_headings(memory: str) -> set[str]:
"""Return all ``## …`` heading texts (without the ``## `` prefix)."""
return set(_SECTION_HEADING_RE.findall(memory))
def _normalize_heading(heading: str) -> str:
"""Normalize heading text for robust scope checks."""
return _HEADING_NORMALIZE_RE.sub(" ", heading.strip().lower())
def _validate_memory_scope(
content: str, scope: Literal["user", "team"]
) -> dict[str, Any] | None:
"""Reject personal-only markers ([pref], [instr]) in team memory."""
if scope != "team":
return None
markers = set(_MARKER_RE.findall(content))
leaked = sorted(markers & _PERSONAL_ONLY_MARKERS)
if leaked:
tags = ", ".join(f"[{m}]" for m in leaked)
return {
"status": "error",
"message": (
f"Team memory cannot include personal markers: {tags}. "
"Use [fact] only in team memory."
),
}
return None
def _validate_bullet_format(content: str) -> list[str]:
"""Return warnings for bullet lines that don't match the required format.
Expected: ``- (YYYY-MM-DD) [fact|pref|instr] text``
"""
warnings: list[str] = []
for line in content.splitlines():
stripped = line.strip()
if not stripped.startswith("- "):
continue
if not _BULLET_FORMAT_RE.match(stripped):
short = stripped[:80] + ("..." if len(stripped) > 80 else "")
warnings.append(f"Malformed bullet: {short}")
return warnings
def _validate_diff(old_memory: str | None, new_memory: str) -> list[str]:
"""Return a list of warning strings about suspicious changes."""
if not old_memory:
return []
warnings: list[str] = []
old_headings = _extract_headings(old_memory)
new_headings = _extract_headings(new_memory)
dropped = old_headings - new_headings
if dropped:
names = ", ".join(sorted(dropped))
warnings.append(
f"Sections removed: {names}. "
"If unintentional, the user can restore from the settings page."
)
old_len = len(old_memory)
new_len = len(new_memory)
if old_len > 0 and new_len < old_len * 0.4:
warnings.append(
f"Memory shrank significantly ({old_len:,} -> {new_len:,} chars). "
"Possible data loss."
)
return warnings
# ---------------------------------------------------------------------------
# Size validation & soft warning
# ---------------------------------------------------------------------------
def _validate_memory_size(content: str) -> dict[str, Any] | None:
"""Return an error/warning dict if *content* is too large, else None."""
length = len(content)
if length > MEMORY_HARD_LIMIT:
return {
"status": "error",
"message": (
f"Memory exceeds {MEMORY_HARD_LIMIT:,} character limit "
f"({length:,} chars). Consolidate by merging related items, "
"removing outdated entries, and shortening descriptions. "
"Then call update_memory again."
),
}
return None
def _soft_warning(content: str) -> str | None:
"""Return a warning string if content exceeds the soft limit."""
length = len(content)
if length > MEMORY_SOFT_LIMIT:
return (
f"Memory is at {length:,}/{MEMORY_HARD_LIMIT:,} characters. "
"Consolidate by merging related items and removing less important "
"entries on your next update."
)
return None
# ---------------------------------------------------------------------------
# Forced rewrite when memory exceeds the hard limit
# ---------------------------------------------------------------------------
_FORCED_REWRITE_PROMPT = """\
You are a memory curator. The following memory document exceeds the character \
limit and must be shortened.
RULES:
1. Rewrite the document to be under {target} characters.
2. Preserve existing ## headings. Every entry must remain under a heading. You may merge
or rename headings to consolidate, but keep names personal and descriptive.
3. Priority for keeping content: [instr] > [pref] > [fact].
4. Merge duplicate entries, remove outdated entries, shorten verbose descriptions.
5. Every bullet MUST have format: - (YYYY-MM-DD) [fact|pref|instr] text
6. Preserve the user's first name in entries — do not replace it with "the user".
7. Output ONLY the consolidated markdown — no explanations, no wrapping.
<memory_document>
{content}
</memory_document>"""
async def _forced_rewrite(content: str, llm: Any) -> str | None:
"""Use a focused LLM call to compress *content* under the hard limit.
Returns the rewritten string, or ``None`` if the call fails.
"""
try:
prompt = _FORCED_REWRITE_PROMPT.format(
target=MEMORY_HARD_LIMIT, content=content
)
response = await llm.ainvoke(
[HumanMessage(content=prompt)],
config={"tags": ["surfsense:internal"]},
)
text = extract_text_content(response.content).strip()
if not text:
logger.warning("Forced rewrite returned empty text; aborting rewrite")
return None
return text
except Exception:
logger.exception("Forced rewrite LLM call failed")
return None
# ---------------------------------------------------------------------------
# Shared save-and-respond logic
# ---------------------------------------------------------------------------
async def _save_memory(
*,
updated_memory: str,
old_memory: str | None,
llm: Any | None,
apply_fn,
commit_fn,
rollback_fn,
label: str,
scope: Literal["user", "team"],
) -> dict[str, Any]:
"""Validate, optionally force-rewrite if over the hard limit, save, and
return a response dict.
Parameters
----------
updated_memory : str
The new document the agent submitted.
old_memory : str | None
The previously persisted document (for diff checks).
llm : Any | None
LLM instance for forced rewrite (may be ``None``).
apply_fn : callable(str) -> None
Callback that sets the new memory on the ORM object.
commit_fn : coroutine
``session.commit``.
rollback_fn : coroutine
``session.rollback``.
label : str
Human label for log messages (e.g. "user memory", "team memory").
"""
if not isinstance(updated_memory, str):
logger.warning(
"Refusing non-string memory payload (type=%s)",
type(updated_memory).__name__,
)
return {
"status": "error",
"message": "Internal error: memory payload must be a string.",
}
content = updated_memory
# --- forced rewrite if over the hard limit ---
if len(content) > MEMORY_HARD_LIMIT and llm is not None:
rewritten = await _forced_rewrite(content, llm)
if rewritten is not None and len(rewritten) < len(content):
content = rewritten
# --- hard-limit gate (reject if still too large after rewrite) ---
size_err = _validate_memory_size(content)
if size_err:
return size_err
scope_err = _validate_memory_scope(content, scope)
if scope_err:
return scope_err
# --- persist ---
try:
apply_fn(content)
await commit_fn()
except Exception as e:
logger.exception("Failed to update %s: %s", label, e)
await rollback_fn()
return {"status": "error", "message": f"Failed to update {label}: {e}"}
# --- build response ---
resp: dict[str, Any] = {
"status": "saved",
"message": f"{label.capitalize()} updated.",
}
if content is not updated_memory:
resp["notice"] = "Memory was automatically rewritten to fit within limits."
diff_warnings = _validate_diff(old_memory, content)
if diff_warnings:
resp["diff_warnings"] = diff_warnings
format_warnings = _validate_bullet_format(content)
if format_warnings:
resp["format_warnings"] = format_warnings
warning = _soft_warning(content)
if warning:
resp["warning"] = warning
return resp
# ---------------------------------------------------------------------------
# Tool factories
# ---------------------------------------------------------------------------
def create_update_memory_tool(
user_id: str | UUID,
db_session: AsyncSession,
llm: Any | None = None,
):
"""Factory function to create the user-memory update tool.
"""Factory for the user-memory update tool.
The tool acquires its own short-lived ``AsyncSession`` per call via
:data:`async_session_maker` so the closure is safe to share across
HTTP requests by the compiled-agent cache. Capturing a per-request
session here would surface stale/closed sessions on cache hits.
The session's bound ``commit``/``rollback`` methods are captured at
call time, after ``async with`` has bound ``db_session`` locally.
Args:
user_id: ID of the user whose memory document is being updated.
db_session: Reserved for registry compatibility. Per-call sessions
are opened via :data:`async_session_maker` inside the tool body.
llm: Optional LLM for the forced-rewrite path.
Returns:
Configured update_memory tool for the user-memory scope.
Uses a fresh short-lived session per call so compiled-agent caches never
retain a stale request-scoped session.
"""
del db_session # per-call session — see docstring
del db_session
uid = UUID(user_id) if isinstance(user_id, str) else user_id
@tool
async def update_memory(updated_memory: str) -> dict[str, Any]:
"""Update the user's personal memory document.
Your current memory is shown in <user_memory> in the system prompt.
When the user shares important long-term information (preferences,
facts, instructions, context), rewrite the memory document to include
the new information. Merge new facts with existing ones, update
contradictions, remove outdated entries, and keep it concise.
Args:
updated_memory: The FULL updated markdown document (not a diff).
The current memory is shown in <user_memory>. Pass the FULL updated
markdown document, not a diff.
"""
try:
async with async_session_maker() as db_session:
result = await db_session.execute(select(User).where(User.id == uid))
user = result.scalars().first()
if not user:
return {"status": "error", "message": "User not found."}
old_memory = user.memory_md
return await _save_memory(
updated_memory=updated_memory,
old_memory=old_memory,
result = await save_memory(
scope=MemoryScope.USER,
target_id=uid,
content=updated_memory,
session=db_session,
llm=llm,
apply_fn=lambda content: setattr(user, "memory_md", content),
commit_fn=db_session.commit,
rollback_fn=db_session.rollback,
label="memory",
scope="user",
)
return result.to_dict()
except Exception as e:
logger.exception("Failed to update user memory: %s", e)
return {
"status": "error",
"message": f"Failed to update memory: {e}",
}
return {"status": "error", "message": f"Failed to update memory: {e}"}
return update_memory
@ -373,64 +57,26 @@ def create_update_team_memory_tool(
db_session: AsyncSession,
llm: Any | None = None,
):
"""Factory function to create the team-memory update tool.
The tool acquires its own short-lived ``AsyncSession`` per call via
:data:`async_session_maker` so the closure is safe to share across
HTTP requests by the compiled-agent cache. Capturing a per-request
session here would surface stale/closed sessions on cache hits.
The session's bound ``commit``/``rollback`` methods are captured at
call time, after ``async with`` has bound ``db_session`` locally.
Args:
search_space_id: ID of the search space whose team memory is being
updated.
db_session: Reserved for registry compatibility. Per-call sessions
are opened via :data:`async_session_maker` inside the tool body.
llm: Optional LLM for the forced-rewrite path.
Returns:
Configured update_memory tool for the team-memory scope.
"""
del db_session # per-call session — see docstring
"""Factory for the team-memory update tool."""
del db_session
@tool
async def update_memory(updated_memory: str) -> dict[str, Any]:
"""Update the team's shared memory document for this search space.
Your current team memory is shown in <team_memory> in the system
prompt. When the team shares important long-term information
(decisions, conventions, key facts, priorities), rewrite the memory
document to include the new information. Merge new facts with
existing ones, update contradictions, remove outdated entries, and
keep it concise.
Args:
updated_memory: The FULL updated markdown document (not a diff).
The current team memory is shown in <team_memory>. Pass the FULL updated
markdown document, not a diff.
"""
try:
async with async_session_maker() as db_session:
result = await db_session.execute(
select(SearchSpace).where(SearchSpace.id == search_space_id)
)
space = result.scalars().first()
if not space:
return {"status": "error", "message": "Search space not found."}
old_memory = space.shared_memory_md
return await _save_memory(
updated_memory=updated_memory,
old_memory=old_memory,
result = await save_memory(
scope=MemoryScope.TEAM,
target_id=search_space_id,
content=updated_memory,
session=db_session,
llm=llm,
apply_fn=lambda content: setattr(
space, "shared_memory_md", content
),
commit_fn=db_session.commit,
rollback_fn=db_session.rollback,
label="team memory",
scope="team",
)
return result.to_dict()
except Exception as e:
logger.exception("Failed to update team memory: %s", e)
return {
@ -439,3 +85,9 @@ def create_update_team_memory_tool(
}
return update_memory
__all__ = [
"create_update_memory_tool",
"create_update_team_memory_tool",
]

2
surfsense_backend/app/routes/__init__.py
View file

@ -54,6 +54,7 @@ from .search_spaces_routes import router as search_spaces_router
from .slack_add_connector_route import router as slack_add_connector_router
from .stripe_routes import router as stripe_router
from .surfsense_docs_routes import router as surfsense_docs_router
from .team_memory_routes import router as team_memory_router
from .teams_add_connector_route import router as teams_add_connector_router
from .video_presentations_routes import router as video_presentations_router
from .vision_llm_routes import router as vision_llm_router
@ -117,3 +118,4 @@ router.include_router(stripe_router) # Stripe checkout for additional page pack
router.include_router(youtube_router) # YouTube playlist resolution
router.include_router(prompts_router)
router.include_router(memory_router) # User personal memory (memory.md style)
router.include_router(team_memory_router) # Search-space team memory

147
surfsense_backend/app/routes/memory_routes.py
View file

@ -1,75 +1,40 @@
"""Routes for user memory management (personal memory.md)."""
"""Routes for user memory management."""
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, HTTPException
from langchain_core.messages import HumanMessage
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
from app.agents.new_chat.llm_config import (
create_chat_litellm_from_agent_config,
load_agent_llm_config_for_search_space,
)
from app.agents.new_chat.tools.update_memory import MEMORY_HARD_LIMIT, _save_memory
from app.db import User, get_async_session
from app.services.memory import (
MemoryRead,
MemoryScope,
memory_limits,
read_memory,
reset_memory,
save_memory,
)
from app.users import current_active_user
from app.utils.content_utils import extract_text_content
logger = logging.getLogger(__name__)
router = APIRouter()
class MemoryRead(BaseModel):
memory_md: str
class MemoryUpdate(BaseModel):
memory_md: str
class MemoryEditRequest(BaseModel):
query: str
search_space_id: int
_MEMORY_EDIT_PROMPT = """\
You are a memory editor. The user wants to modify their memory document. \
Apply the user's instruction to the existing memory document and output the \
FULL updated document.
RULES:
1. If the instruction asks to add something, add it with format: \
- (YYYY-MM-DD) [fact|pref|instr] text, under an existing or new ## heading. \
Heading names should be personal and descriptive, not generic categories.
2. If the instruction asks to remove something, remove the matching entry.
3. If the instruction asks to change something, update the matching entry.
4. Preserve existing ## headings and all other entries.
5. Every bullet must include a marker: [fact], [pref], or [instr].
6. Use the user's first name (from <user_name>) in entries instead of "the user".
7. Output ONLY the updated markdown — no explanations, no wrapping.
<user_name>{user_name}</user_name>
<current_memory>
{current_memory}
</current_memory>
<user_instruction>
{instruction}
</user_instruction>"""
@router.get("/users/me/memory", response_model=MemoryRead)
async def get_user_memory(
user: User = Depends(current_active_user),
session: AsyncSession = Depends(get_async_session),
):
await session.refresh(user, ["memory_md"])
return MemoryRead(memory_md=user.memory_md or "")
memory_md = await read_memory(
scope=MemoryScope.USER,
target_id=user.id,
session=session,
)
return MemoryRead(memory_md=memory_md, limits=memory_limits())
@router.put("/users/me/memory", response_model=MemoryRead)
@ -78,73 +43,27 @@ async def update_user_memory(
user: User = Depends(current_active_user),
session: AsyncSession = Depends(get_async_session),
):
if len(body.memory_md) > MEMORY_HARD_LIMIT:
raise HTTPException(
status_code=400,
detail=f"Memory exceeds {MEMORY_HARD_LIMIT:,} character limit ({len(body.memory_md):,} chars).",
)
user.memory_md = body.memory_md
session.add(user)
await session.commit()
await session.refresh(user, ["memory_md"])
return MemoryRead(memory_md=user.memory_md or "")
result = await save_memory(
scope=MemoryScope.USER,
target_id=user.id,
content=body.memory_md,
session=session,
)
if result.status == "error":
raise HTTPException(status_code=400, detail=result.message)
return MemoryRead(memory_md=result.memory_md, limits=memory_limits())
@router.post("/users/me/memory/edit", response_model=MemoryRead)
async def edit_user_memory(
body: MemoryEditRequest,
@router.post("/users/me/memory/reset", response_model=MemoryRead)
async def reset_user_memory(
user: User = Depends(current_active_user),
session: AsyncSession = Depends(get_async_session),
):
"""Apply a natural language edit to the user's personal memory via LLM."""
agent_config = await load_agent_llm_config_for_search_space(
session, body.search_space_id
result = await reset_memory(
scope=MemoryScope.USER,
target_id=user.id,
session=session,
)
if not agent_config:
raise HTTPException(status_code=500, detail="No LLM configuration available.")
llm = create_chat_litellm_from_agent_config(agent_config)
if not llm:
raise HTTPException(status_code=500, detail="Failed to create LLM instance.")
await session.refresh(user, ["memory_md", "display_name"])
current_memory = user.memory_md or ""
first_name = (
user.display_name.strip().split()[0]
if user.display_name and user.display_name.strip()
else "The user"
)
prompt = _MEMORY_EDIT_PROMPT.format(
current_memory=current_memory or "(empty)",
instruction=body.query,
user_name=first_name,
)
try:
response = await llm.ainvoke(
[HumanMessage(content=prompt)],
config={"tags": ["surfsense:internal", "memory-edit"]},
)
updated = extract_text_content(response.content).strip()
except Exception as e:
logger.exception("Memory edit LLM call failed: %s", e)
raise HTTPException(status_code=500, detail="Memory edit failed.") from e
if not updated:
raise HTTPException(status_code=400, detail="LLM returned empty result.")
result = await _save_memory(
updated_memory=updated,
old_memory=current_memory,
llm=llm,
apply_fn=lambda content: setattr(user, "memory_md", content),
commit_fn=session.commit,
rollback_fn=session.rollback,
label="memory",
scope="user",
)
if result.get("status") == "error":
raise HTTPException(status_code=400, detail=result["message"])
await session.refresh(user, ["memory_md"])
return MemoryRead(memory_md=user.memory_md or "")
if result.status == "error":
raise HTTPException(status_code=400, detail=result.message)
return MemoryRead(memory_md=result.memory_md, limits=memory_limits())

111
surfsense_backend/app/routes/search_spaces_routes.py
View file

@ -1,17 +1,10 @@
import logging
from fastapi import APIRouter, Depends, HTTPException
from langchain_core.messages import HumanMessage
from pydantic import BaseModel as PydanticBaseModel
from sqlalchemy import func, update
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from app.agents.new_chat.llm_config import (
create_chat_litellm_from_agent_config,
load_agent_llm_config_for_search_space,
)
from app.agents.new_chat.tools.update_memory import MEMORY_HARD_LIMIT, _save_memory
from app.config import config
from app.db import (
ImageGenerationConfig,
@ -35,7 +28,6 @@ from app.schemas import (
SearchSpaceWithStats,
)
from app.users import current_active_user
from app.utils.content_utils import extract_text_content
from app.utils.rbac import check_permission, check_search_space_access
logger = logging.getLogger(__name__)
@ -43,34 +35,6 @@ logger = logging.getLogger(__name__)
router = APIRouter()
class _TeamMemoryEditRequest(PydanticBaseModel):
query: str
_TEAM_MEMORY_EDIT_PROMPT = """\
You are a memory editor for a team workspace. The user wants to modify the \
team's shared memory document. Apply the user's instruction to the existing \
memory document and output the FULL updated document.
RULES:
1. If the instruction asks to add something, add it with format: \
- (YYYY-MM-DD) [fact] text, under an existing or new ## heading. \
Heading names should be descriptive, not generic categories.
2. If the instruction asks to remove something, remove the matching entry.
3. If the instruction asks to change something, update the matching entry.
4. Preserve existing ## headings and all other entries.
5. NEVER use [pref] or [instr] markers. Team memory uses [fact] only.
6. Output ONLY the updated markdown — no explanations, no wrapping.
<current_memory>
{current_memory}
</current_memory>
<user_instruction>
{instruction}
</user_instruction>"""
async def create_default_roles_and_membership(
session: AsyncSession,
search_space_id: int,
@ -294,15 +258,6 @@ async def update_search_space(
update_data = search_space_update.model_dump(exclude_unset=True)
if (
"shared_memory_md" in update_data
and len(update_data["shared_memory_md"] or "") > MEMORY_HARD_LIMIT
):
raise HTTPException(
status_code=400,
detail=f"Team memory exceeds {MEMORY_HARD_LIMIT:,} character limit.",
)
for key, value in update_data.items():
setattr(db_search_space, key, value)
await session.commit()
@ -317,72 +272,6 @@ async def update_search_space(
) from e
@router.post(
"/searchspaces/{search_space_id}/memory/edit",
response_model=SearchSpaceRead,
)
async def edit_team_memory(
search_space_id: int,
body: _TeamMemoryEditRequest,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""Apply a natural language edit to the team memory via LLM."""
await check_search_space_access(session, user, search_space_id)
agent_config = await load_agent_llm_config_for_search_space(
session, search_space_id
)
if not agent_config:
raise HTTPException(status_code=500, detail="No LLM configuration available.")
llm = create_chat_litellm_from_agent_config(agent_config)
if not llm:
raise HTTPException(status_code=500, detail="Failed to create LLM instance.")
result = await session.execute(
select(SearchSpace).filter(SearchSpace.id == search_space_id)
)
db_search_space = result.scalars().first()
if not db_search_space:
raise HTTPException(status_code=404, detail="Search space not found")
current_memory = db_search_space.shared_memory_md or ""
prompt = _TEAM_MEMORY_EDIT_PROMPT.format(
current_memory=current_memory or "(empty)",
instruction=body.query,
)
try:
response = await llm.ainvoke(
[HumanMessage(content=prompt)],
config={"tags": ["surfsense:internal", "memory-edit"]},
)
updated = extract_text_content(response.content).strip()
except Exception as e:
logger.exception("Team memory edit LLM call failed: %s", e)
raise HTTPException(status_code=500, detail="Team memory edit failed.") from e
if not updated:
raise HTTPException(status_code=400, detail="LLM returned empty result.")
save_result = await _save_memory(
updated_memory=updated,
old_memory=current_memory,
llm=llm,
apply_fn=lambda content: setattr(db_search_space, "shared_memory_md", content),
commit_fn=session.commit,
rollback_fn=session.rollback,
label="team memory",
scope="team",
)
if save_result.get("status") == "error":
raise HTTPException(status_code=400, detail=save_result["message"])
await session.refresh(db_search_space)
return db_search_space
@router.post("/searchspaces/{search_space_id}/ai-sort")
async def trigger_ai_sort(
search_space_id: int,

76
surfsense_backend/app/routes/team_memory_routes.py Normal file
View file

@ -0,0 +1,76 @@
"""Routes for search-space team memory."""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import User, get_async_session
from app.services.memory import (
MemoryRead,
MemoryScope,
memory_limits,
read_memory,
reset_memory,
save_memory,
)
from app.users import current_active_user
from app.utils.rbac import check_search_space_access
router = APIRouter()
class TeamMemoryUpdate(BaseModel):
memory_md: str
@router.get("/searchspaces/{search_space_id}/memory", response_model=MemoryRead)
async def get_team_memory(
search_space_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
await check_search_space_access(session, user, search_space_id)
memory_md = await read_memory(
scope=MemoryScope.TEAM,
target_id=search_space_id,
session=session,
)
return MemoryRead(memory_md=memory_md, limits=memory_limits())
@router.put("/searchspaces/{search_space_id}/memory", response_model=MemoryRead)
async def update_team_memory(
search_space_id: int,
body: TeamMemoryUpdate,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
await check_search_space_access(session, user, search_space_id)
result = await save_memory(
scope=MemoryScope.TEAM,
target_id=search_space_id,
content=body.memory_md,
session=session,
)
if result.status == "error":
raise HTTPException(status_code=400, detail=result.message)
return MemoryRead(memory_md=result.memory_md, limits=memory_limits())
@router.post("/searchspaces/{search_space_id}/memory/reset", response_model=MemoryRead)
async def reset_team_memory(
search_space_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
await check_search_space_access(session, user, search_space_id)
result = await reset_memory(
scope=MemoryScope.TEAM,
target_id=search_space_id,
session=session,
)
if result.status == "error":
raise HTTPException(status_code=400, detail=result.message)
return MemoryRead(memory_md=result.memory_md, limits=memory_limits())

1
surfsense_backend/app/schemas/search_space.py
View file

@ -21,7 +21,6 @@ class SearchSpaceUpdate(BaseModel):
description: str | None = None
citations_enabled: bool | None = None
qna_custom_instructions: str | None = None
shared_memory_md: str | None = None
ai_file_sort_enabled: bool | None = None

32
surfsense_backend/app/services/memory/__init__.py Normal file
View file

@ -0,0 +1,32 @@
"""First-class memory service for user and team markdown memory."""
from .schemas import MemoryLimits, MemoryRead
from .service import (
MemoryScope,
SaveResult,
memory_limits,
read_memory,
reset_memory,
save_memory,
)
from .validation import (
MEMORY_HARD_LIMIT,
MEMORY_SOFT_LIMIT,
validate_bullet_format,
validate_memory_scope,
)
__all__ = [
"MEMORY_HARD_LIMIT",
"MEMORY_SOFT_LIMIT",
"MemoryLimits",
"MemoryRead",
"MemoryScope",
"SaveResult",
"memory_limits",
"read_memory",
"reset_memory",
"save_memory",
"validate_bullet_format",
"validate_memory_scope",
]

200
surfsense_backend/app/services/memory/document.py Normal file
View file

@ -0,0 +1,200 @@
"""Memory-specific markdown document model and canonical renderer.
This intentionally parses only SurfSense memory's small markdown contract:
``##`` sections with dated bullet items. Unknown lines are preserved so user
edits are not lost, while legacy marker bullets are normalized on render.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import date
DEFAULT_LEGACY_SECTION = "Memory"
LEGACY_MARKERS = frozenset({"fact", "pref", "instr"})
@dataclass(frozen=True)
class MemoryBullet:
entry_date: date
text: str
@dataclass(frozen=True)
class MemoryRawLine:
text: str
MemoryLine = MemoryBullet | MemoryRawLine
@dataclass(frozen=True)
class MemorySection:
heading: str
lines: list[MemoryLine] = field(default_factory=list)
explicit_heading: bool = True
@dataclass(frozen=True)
class MemoryDocument:
sections: list[MemorySection] = field(default_factory=list)
@property
def has_explicit_heading(self) -> bool:
return any(section.explicit_heading for section in self.sections)
def is_section_heading(line: str) -> bool:
return line.startswith("## ") and bool(line[3:].strip())
def heading_text(line: str) -> str:
return line[3:].strip()
def normalize_heading(heading: str) -> str:
chars: list[str] = []
previous_was_space = True
for char in heading.strip().lower():
if char.isalnum():
chars.append(char)
previous_was_space = False
elif not previous_was_space:
chars.append(" ")
previous_was_space = True
return "".join(chars).strip()
def parse_bullet_line(line: str) -> MemoryBullet | None:
stripped = line.strip()
if not stripped.startswith("- "):
return None
body = stripped[2:]
parsed = _parse_canonical_bullet(body)
if parsed is not None:
return parsed
return _parse_legacy_bullet(body)
def _parse_canonical_bullet(body: str) -> MemoryBullet | None:
if len(body) < 13 or body[10:12] != ": ":
return None
try:
entry_date = date.fromisoformat(body[:10])
except ValueError:
return None
text = body[12:].strip()
if not text:
return None
return MemoryBullet(entry_date=entry_date, text=text)
def _parse_legacy_bullet(body: str) -> MemoryBullet | None:
if len(body) < 20 or not body.startswith("("):
return None
if len(body) < 14 or body[11:14] != ") [":
return None
try:
entry_date = date.fromisoformat(body[1:11])
except ValueError:
return None
marker_end = body.find("] ", 14)
if marker_end == -1:
return None
marker = body[14:marker_end]
if marker not in LEGACY_MARKERS:
return None
text = body[marker_end + 2 :].strip()
if not text:
return None
return MemoryBullet(entry_date=entry_date, text=text)
def parse_memory_document(content: str | None) -> MemoryDocument:
if not content:
return MemoryDocument()
sections: list[MemorySection] = []
current_heading: str | None = None
current_explicit = True
current_lines: list[MemoryLine] = []
def flush_current() -> None:
nonlocal current_heading, current_explicit, current_lines
if current_heading is None:
return
sections.append(
MemorySection(
heading=current_heading,
lines=current_lines,
explicit_heading=current_explicit,
)
)
current_heading = None
current_explicit = True
current_lines = []
for raw_line in content.strip().splitlines():
line = raw_line.rstrip()
if is_section_heading(line):
flush_current()
current_heading = heading_text(line)
current_explicit = True
current_lines = []
continue
bullet = parse_bullet_line(line)
if current_heading is None:
if bullet is None:
continue
current_heading = DEFAULT_LEGACY_SECTION
current_explicit = False
current_lines = [bullet]
continue
current_lines.append(bullet if bullet is not None else MemoryRawLine(text=line))
flush_current()
return MemoryDocument(sections=sections)
def render_memory_document(document: MemoryDocument) -> str:
rendered_sections: list[str] = []
for section in document.sections:
section_lines = [f"## {section.heading}"]
for line in section.lines:
if isinstance(line, MemoryBullet):
section_lines.append(f"- {line.entry_date.isoformat()}: {line.text}")
else:
section_lines.append(line.text)
rendered_sections.append("\n".join(section_lines).strip())
return "\n\n".join(section for section in rendered_sections if section).strip()
def extract_headings(memory: str | None) -> set[str]:
document = parse_memory_document(memory)
return {
normalize_heading(section.heading)
for section in document.sections
if section.explicit_heading
}
def has_explicit_heading(content: str) -> bool:
return parse_memory_document(content).has_explicit_heading
def nonstandard_bullets(content: str) -> list[str]:
warnings: list[str] = []
for line in content.splitlines():
stripped = line.strip()
if not stripped.startswith("- "):
continue
if parse_bullet_line(stripped) is not None:
continue
short = stripped[:80] + ("..." if len(stripped) > 80 else "")
warnings.append(f"Non-standard memory bullet: {short}")
return warnings

20
surfsense_backend/app/services/memory/prompts.py Normal file
View file

@ -0,0 +1,20 @@
"""Prompts used by the memory service."""
FORCED_REWRITE_PROMPT = """\
You are a memory curator. The following memory document exceeds the character \
limit and must be shortened.
RULES:
1. Rewrite the document to be under {target} characters.
2. Output Markdown only. Use clear `##` headings and concise bullet points.
3. New-format bullets should look like: `- YYYY-MM-DD: memory text`.
4. If the input contains legacy markers like `(YYYY-MM-DD) [fact]`, preserve the
information but remove the inline marker in the output.
5. Preserve durable instructions and preferences before generic facts when
compressing personal memory.
6. Preserve existing headings when useful; merge duplicate headings and bullets.
7. Output ONLY the consolidated markdown — no explanations, no wrapping.
<memory_document>
{content}
</memory_document>"""

35
surfsense_backend/app/services/memory/rewrite.py Normal file
View file

@ -0,0 +1,35 @@
"""LLM-backed memory rewrite helpers."""
from __future__ import annotations
import logging
from typing import Any
from langchain_core.messages import HumanMessage
from app.services.memory.prompts import FORCED_REWRITE_PROMPT
from app.services.memory.validation import MEMORY_HARD_LIMIT
from app.utils.content_utils import extract_text_content
logger = logging.getLogger(__name__)
async def forced_rewrite(content: str, llm: Any) -> str | None:
"""Use a focused LLM call to compress memory under the hard limit."""
try:
prompt = FORCED_REWRITE_PROMPT.format(
target=MEMORY_HARD_LIMIT,
content=content,
)
response = await llm.ainvoke(
[HumanMessage(content=prompt)],
config={"tags": ["surfsense:internal", "memory-rewrite"]},
)
text = extract_text_content(response.content).strip()
if not text:
logger.warning("Forced memory rewrite returned empty text")
return None
return text
except Exception:
logger.exception("Forced memory rewrite LLM call failed")
return None

19
surfsense_backend/app/services/memory/schemas.py Normal file
View file

@ -0,0 +1,19 @@
"""Schemas for memory API responses and structured extraction."""
from __future__ import annotations
from pydantic import BaseModel
class MemoryLimits(BaseModel):
"""Canonical memory size limits exposed to clients."""
soft: int
hard: int
class MemoryRead(BaseModel):
"""Memory document payload returned by user and team memory APIs."""
memory_md: str
limits: MemoryLimits

247
surfsense_backend/app/services/memory/service.py Normal file
View file

@ -0,0 +1,247 @@
"""Canonical read/write/reset/extract service for markdown memory."""
from __future__ import annotations
import logging
from dataclasses import dataclass, field
from enum import StrEnum
from typing import Any, Literal
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import SearchSpace, User
from app.services.memory.document import parse_memory_document, render_memory_document
from app.services.memory.rewrite import forced_rewrite
from app.services.memory.schemas import MemoryLimits
from app.services.memory.validation import (
MEMORY_HARD_LIMIT,
MEMORY_SOFT_LIMIT,
soft_limit_warning,
strip_preamble_to_first_heading,
validate_bullet_format,
validate_diff,
validate_heading_sanity,
validate_memory_scope,
validate_memory_size,
)
logger = logging.getLogger(__name__)
_NO_UPDATE_SENTINELS = frozenset(
{
"NO_UPDATE",
"NO UPDATE",
"NO_CHANGE",
"NO CHANGE",
}
)
class MemoryScope(StrEnum):
USER = "user"
TEAM = "team"
@dataclass(frozen=True)
class SaveResult:
status: Literal["saved", "error", "no_op"]
message: str
memory_md: str = ""
warnings: list[str] = field(default_factory=list)
diff_warnings: list[str] = field(default_factory=list)
format_warnings: list[str] = field(default_factory=list)
notice: str | None = None
def to_dict(self) -> dict[str, Any]:
data: dict[str, Any] = {
"status": self.status,
"message": self.message,
"memory_md": self.memory_md,
}
if self.notice:
data["notice"] = self.notice
if self.warnings:
data["warnings"] = self.warnings
if len(self.warnings) == 1:
data["warning"] = self.warnings[0]
if self.diff_warnings:
data["diff_warnings"] = self.diff_warnings
if self.format_warnings:
data["format_warnings"] = self.format_warnings
return data
def memory_limits() -> MemoryLimits:
return MemoryLimits(soft=MEMORY_SOFT_LIMIT, hard=MEMORY_HARD_LIMIT)
def _normalize_scope(scope: MemoryScope | str) -> MemoryScope:
return scope if isinstance(scope, MemoryScope) else MemoryScope(scope)
def _normalize_user_id(target_id: str | UUID) -> UUID:
return UUID(target_id) if isinstance(target_id, str) else target_id
async def _load_target(
*,
scope: MemoryScope | str,
target_id: str | int | UUID,
session: AsyncSession,
) -> User | SearchSpace | None:
normalized = _normalize_scope(scope)
if normalized is MemoryScope.USER:
result = await session.execute(
select(User).where(User.id == _normalize_user_id(target_id)) # type: ignore[arg-type]
)
return result.scalars().first()
result = await session.execute(
select(SearchSpace).where(SearchSpace.id == int(target_id))
)
return result.scalars().first()
def _get_memory(target: User | SearchSpace, scope: MemoryScope) -> str:
if scope is MemoryScope.USER:
return getattr(target, "memory_md", None) or ""
return getattr(target, "shared_memory_md", None) or ""
def _set_memory(target: User | SearchSpace, scope: MemoryScope, content: str) -> None:
if scope is MemoryScope.USER:
target.memory_md = content
else:
target.shared_memory_md = content
async def read_memory(
*,
scope: MemoryScope | str,
target_id: str | int | UUID,
session: AsyncSession,
) -> str:
normalized = _normalize_scope(scope)
target = await _load_target(scope=normalized, target_id=target_id, session=session)
if target is None:
return ""
return _get_memory(target, normalized)
async def save_memory(
*,
scope: MemoryScope | str,
target_id: str | int | UUID,
content: str,
session: AsyncSession,
llm: Any | None = None,
) -> SaveResult:
normalized = _normalize_scope(scope)
if not isinstance(content, str):
return SaveResult(
status="error",
message="Internal error: memory payload must be a string.",
)
target = await _load_target(scope=normalized, target_id=target_id, session=session)
if target is None:
return SaveResult(
status="error",
message="User not found."
if normalized is MemoryScope.USER
else "Search space not found.",
)
old_memory = _get_memory(target, normalized)
next_content = strip_preamble_to_first_heading(content.strip())
notice: str | None = None
warnings: list[str] = []
if next_content.upper() in _NO_UPDATE_SENTINELS:
return SaveResult(
status="no_op",
message="No memory update requested.",
memory_md=old_memory,
)
if len(next_content) > MEMORY_HARD_LIMIT and llm is not None:
rewritten = await forced_rewrite(next_content, llm)
if rewritten is not None and len(rewritten) < len(next_content):
next_content = strip_preamble_to_first_heading(rewritten)
notice = "Memory was automatically rewritten to fit within limits."
for validation in (
validate_memory_size(next_content),
validate_heading_sanity(next_content),
):
if validation:
return SaveResult(
status="error",
message=validation["message"],
memory_md=old_memory,
)
scope_error, scope_warnings = validate_memory_scope(
next_content,
normalized.value,
old_memory=old_memory,
)
warnings.extend(scope_warnings)
if scope_error:
return SaveResult(
status="error",
message=scope_error["message"],
memory_md=old_memory,
warnings=warnings,
)
next_content = render_memory_document(parse_memory_document(next_content))
try:
_set_memory(target, normalized, next_content)
session.add(target)
await session.commit()
except Exception as e:
logger.exception("Failed to update %s memory: %s", normalized.value, e)
await session.rollback()
return SaveResult(
status="error",
message=f"Failed to update {normalized.value} memory: {e}",
memory_md=old_memory,
)
diff_warnings = validate_diff(old_memory, next_content)
format_warnings = validate_bullet_format(next_content)
warning = soft_limit_warning(next_content)
if warning:
warnings.append(warning)
return SaveResult(
status="saved",
message=(
"Memory updated."
if normalized is MemoryScope.USER
else "Team memory updated."
),
memory_md=next_content,
warnings=warnings,
diff_warnings=diff_warnings,
format_warnings=format_warnings,
notice=notice,
)
async def reset_memory(
*,
scope: MemoryScope | str,
target_id: str | int | UUID,
session: AsyncSession,
) -> SaveResult:
return await save_memory(
scope=scope,
target_id=target_id,
content="",
session=session,
llm=None,
)

140
surfsense_backend/app/services/memory/validation.py Normal file
View file

@ -0,0 +1,140 @@
"""Validation helpers for markdown-backed memory."""
from __future__ import annotations
from typing import Literal
from app.services.memory.document import (
extract_headings,
has_explicit_heading,
nonstandard_bullets,
parse_memory_document,
)
MEMORY_SOFT_LIMIT = 18_000
MEMORY_HARD_LIMIT = 25_000
_FORBIDDEN_TEAM_HEADINGS = {
"preferences",
"instructions",
"personal notes",
"personal instructions",
}
def has_markdown_heading(content: str) -> bool:
return has_explicit_heading(content)
def strip_preamble_to_first_heading(content: str) -> str:
"""Drop model preamble before the first ``##`` heading, if one exists."""
lines = content.splitlines()
for index, line in enumerate(lines):
if line.startswith("## ") and line[3:].strip():
return "\n".join(lines[index:]).strip()
return content.strip()
def validate_memory_size(content: str) -> dict[str, str] | None:
length = len(content)
if length > MEMORY_HARD_LIMIT:
return {
"status": "error",
"message": (
f"Memory exceeds {MEMORY_HARD_LIMIT:,} character limit "
f"({length:,} chars). Consolidate by merging related items, "
"removing outdated entries, and shortening descriptions."
),
}
return None
def validate_heading_sanity(content: str) -> dict[str, str] | None:
"""Block long prose blobs without headings unless they are legacy bullets."""
stripped = content.strip()
if not stripped:
return None
if has_markdown_heading(stripped):
return None
if len(stripped) <= 40:
return None
if parse_memory_document(stripped).sections:
return None
return {
"status": "error",
"message": "Memory must be markdown with at least one ## heading.",
}
def validate_memory_scope(
content: str,
scope: Literal["user", "team"],
*,
old_memory: str | None = None,
) -> tuple[dict[str, str] | None, list[str]]:
"""Reject new personal headings in team memory, grandfather existing ones."""
if scope != "team":
return None, []
old_forbidden = extract_headings(old_memory) & _FORBIDDEN_TEAM_HEADINGS
new_forbidden = extract_headings(content) & _FORBIDDEN_TEAM_HEADINGS
introduced = sorted(new_forbidden - old_forbidden)
grandfathered = sorted(new_forbidden & old_forbidden)
warnings: list[str] = []
if grandfathered:
warnings.append(
"Team memory contains legacy personal headings: "
+ ", ".join(grandfathered)
+ ". Please consolidate them into team-safe headings."
)
if introduced:
return (
{
"status": "error",
"message": (
"Team memory cannot introduce personal headings: "
+ ", ".join(introduced)
+ ". Use team-safe headings instead."
),
},
warnings,
)
return None, warnings
def validate_bullet_format(content: str) -> list[str]:
return nonstandard_bullets(content)
def validate_diff(old_memory: str | None, new_memory: str) -> list[str]:
if not old_memory:
return []
warnings: list[str] = []
old_headings = extract_headings(old_memory)
new_headings = extract_headings(new_memory)
dropped = old_headings - new_headings
if dropped:
names = ", ".join(sorted(dropped))
warnings.append(
f"Sections removed: {names}. If unintentional, restore them from the memory document."
)
old_len = len(old_memory)
new_len = len(new_memory)
if old_len > 0 and new_len < old_len * 0.4:
warnings.append(
f"Memory shrank significantly ({old_len:,} -> {new_len:,} chars). Possible data loss."
)
return warnings
def soft_limit_warning(content: str) -> str | None:
length = len(content)
if length > MEMORY_SOFT_LIMIT:
return (
f"Memory is at {length:,}/{MEMORY_HARD_LIMIT:,} characters. "
"Consolidate by merging related items and removing less important entries."
)
return None

35
surfsense_backend/app/tasks/chat/stream_new_chat.py
View file

@ -39,10 +39,6 @@ from app.agents.new_chat.llm_config import (
load_agent_config,
load_global_llm_config_by_id,
)
from app.agents.new_chat.memory_extraction import (
extract_and_save_memory,
extract_and_save_team_memory,
)
from app.agents.new_chat.mention_resolver import resolve_mentions, substitute_in_text
from app.agents.new_chat.middleware.busy_mutex import (
end_turn,
@ -283,7 +279,6 @@ class StreamResult:
accumulated_text: str = ""
is_interrupted: bool = False
sandbox_files: list[str] = field(default_factory=list)
agent_called_update_memory: bool = False
request_id: str | None = None
turn_id: str = ""
filesystem_mode: str = "cloud"
@ -2208,36 +2203,6 @@ async def stream_new_chat(
},
)
# Fire background memory extraction if the agent didn't handle it.
# Shared threads write to team memory; private threads write to user memory.
if not stream_result.agent_called_update_memory:
memory_seed = user_query.strip() or (
f"[{len(user_image_data_urls or [])} image(s)]"
if user_image_data_urls
else "(message)"
)
if visibility == ChatVisibility.SEARCH_SPACE:
task = asyncio.create_task(
extract_and_save_team_memory(
user_message=memory_seed,
search_space_id=search_space_id,
llm=llm,
author_display_name=current_user_display_name,
)
)
_background_tasks.add(task)
task.add_done_callback(_background_tasks.discard)
elif user_id:
task = asyncio.create_task(
extract_and_save_memory(
user_message=memory_seed,
user_id=user_id,
llm=llm,
)
)
_background_tasks.add(task)
task.add_done_callback(_background_tasks.discard)
# Finish the step and message
yield streaming_service.format_data("turn-status", {"status": "idle"})
yield streaming_service.format_finish_step()

1
surfsense_backend/app/tasks/chat/streaming/graph_stream/event_stream.py
View file

@ -48,4 +48,3 @@ async def stream_output(
yield frame
result.accumulated_text = state.accumulated_text
result.agent_called_update_memory = state.called_update_memory

1
surfsense_backend/app/tasks/chat/streaming/graph_stream/result.py
View file

@ -11,7 +11,6 @@ class StreamingResult:
accumulated_text: str = ""
is_interrupted: bool = False
sandbox_files: list[str] = field(default_factory=list)
agent_called_update_memory: bool = False
request_id: str | None = None
turn_id: str = ""
filesystem_mode: str = "cloud"

3
surfsense_backend/app/tasks/chat/streaming/handlers/tool_end.py
View file

@ -36,9 +36,6 @@ def iter_tool_end_frames(
raw_output = event.get("data", {}).get("output", "")
staged_file_path = state.file_path_by_run.pop(run_id, None) if run_id else None
if tool_name == "update_memory":
state.called_update_memory = True
if hasattr(raw_output, "content"):
content = raw_output.content
if isinstance(content, str):

1
surfsense_backend/app/tasks/chat/streaming/relay/state.py
View file

@ -32,7 +32,6 @@ class AgentEventRelayState:
last_active_step_items: list[str] = field(default_factory=list)
just_finished_tool: bool = False
active_tool_depth: int = 0
called_update_memory: bool = False
current_reasoning_id: str | None = None
pending_tool_call_chunks: list[dict[str, Any]] = field(default_factory=list)
lc_tool_call_id_by_run: dict[str, str] = field(default_factory=dict)

41
surfsense_backend/tests/unit/agents/new_chat/test_memory_response_content.py
View file

@ -2,28 +2,12 @@
import pytest
from app.agents.new_chat.tools.update_memory import _save_memory
from app.services.memory import MemoryScope, save_memory
from app.utils.content_utils import extract_text_content
pytestmark = pytest.mark.unit
class _Recorder:
def __init__(self) -> None:
self.applied_content: str | None = None
self.commit_calls = 0
self.rollback_calls = 0
def apply(self, content: str) -> None:
self.applied_content = content
async def commit(self) -> None:
self.commit_calls += 1
async def rollback(self) -> None:
self.rollback_calls += 1
def test_extract_text_content_keeps_no_update_bare_string_from_content_blocks() -> None:
content = [
{"type": "thinking", "thinking": "No"},
@ -69,21 +53,12 @@ def test_extract_text_content_preserves_plain_string_responses() -> None:
@pytest.mark.asyncio
async def test_save_memory_rejects_non_string_payload_before_commit() -> None:
recorder = _Recorder()
result = await _save_memory(
updated_memory=["NO_UPDATE"], # type: ignore[arg-type]
old_memory=None,
llm=None,
apply_fn=recorder.apply,
commit_fn=recorder.commit,
rollback_fn=recorder.rollback,
label="memory",
scope="user",
result = await save_memory(
scope=MemoryScope.USER,
target_id="00000000-0000-0000-0000-000000000000",
content=["NO_UPDATE"], # type: ignore[arg-type]
session=None, # type: ignore[arg-type]
)
assert result["status"] == "error"
assert "must be a string" in result["message"]
assert recorder.applied_content is None
assert recorder.commit_calls == 0
assert recorder.rollback_calls == 0
assert result.status == "error"
assert "must be a string" in result.message

284
surfsense_backend/tests/unit/agents/new_chat/tools/test_update_memory_scope.py
View file

@ -1,24 +1,24 @@
"""Unit tests for memory scope validation and bullet format validation."""
"""Unit tests for heading-based memory validation."""
import pytest
from app.agents.new_chat.tools.update_memory import (
_save_memory,
_validate_bullet_format,
_validate_memory_scope,
from app.services.memory import MemoryScope, save_memory
from app.services.memory.validation import (
validate_bullet_format,
validate_memory_scope,
)
pytestmark = pytest.mark.unit
class _Recorder:
class _FakeSession:
def __init__(self) -> None:
self.applied_content: str | None = None
self.added = []
self.commit_calls = 0
self.rollback_calls = 0
def apply(self, content: str) -> None:
self.applied_content = content
def add(self, obj) -> None:
self.added.append(obj)
async def commit(self) -> None:
self.commit_calls += 1
@ -27,172 +27,148 @@ class _Recorder:
self.rollback_calls += 1
# ---------------------------------------------------------------------------
# _validate_memory_scope — marker-based
# ---------------------------------------------------------------------------
def test_validate_memory_scope_rejects_pref_marker_in_team_scope() -> None:
content = "- (2026-04-10) [pref] Prefers dark mode\n"
result = _validate_memory_scope(content, "team")
def test_validate_memory_scope_rejects_new_personal_heading_in_team() -> None:
content = "## Preferences\n- 2026-04-10: Prefers dark mode\n"
result, _warnings = validate_memory_scope(content, "team")
assert result is not None
assert result["status"] == "error"
assert "[pref]" in result["message"]
assert "preferences" in result["message"]
def test_validate_memory_scope_rejects_instr_marker_in_team_scope() -> None:
content = "- (2026-04-10) [instr] Always respond in Spanish\n"
result = _validate_memory_scope(content, "team")
assert result is not None
assert result["status"] == "error"
assert "[instr]" in result["message"]
def test_validate_memory_scope_allows_old_marker_payload_in_team_scope() -> None:
content = "- (2026-04-10) [pref] Legacy personal marker remains readable\n"
result, _warnings = validate_memory_scope(content, "team")
assert result is None
def test_validate_memory_scope_rejects_both_personal_markers_in_team() -> None:
def test_validate_memory_scope_allows_team_headings() -> None:
content = "## Engineering Conventions\n- 2026-04-10: Uses PostgreSQL\n"
result, _warnings = validate_memory_scope(content, "team")
assert result is None
def test_validate_bullet_format_accepts_new_and_legacy_bullets() -> None:
content = (
"- (2026-04-10) [pref] Prefers dark mode\n"
"- (2026-04-10) [instr] Always respond in Spanish\n"
"## Facts\n"
"- 2026-04-10: Senior Python developer\n"
"- (2026-04-10) [fact] Legacy fact is preserved\n"
)
result = _validate_memory_scope(content, "team")
assert result is not None
assert result["status"] == "error"
assert "[instr]" in result["message"]
assert "[pref]" in result["message"]
def test_validate_memory_scope_allows_fact_in_team_scope() -> None:
content = "- (2026-04-10) [fact] Office is in downtown Seattle\n"
result = _validate_memory_scope(content, "team")
assert result is None
def test_validate_memory_scope_allows_all_markers_in_user_scope() -> None:
content = (
"- (2026-04-10) [fact] Python developer\n"
"- (2026-04-10) [pref] Prefers concise answers\n"
"- (2026-04-10) [instr] Always use bullet points\n"
)
result = _validate_memory_scope(content, "user")
assert result is None
def test_validate_memory_scope_allows_any_heading_in_team() -> None:
content = "## Architecture\n- (2026-04-10) [fact] Uses PostgreSQL for persistence\n"
result = _validate_memory_scope(content, "team")
assert result is None
def test_validate_memory_scope_allows_any_heading_in_user() -> None:
content = "## My Projects\n- (2026-04-10) [fact] Working on SurfSense\n"
result = _validate_memory_scope(content, "user")
assert result is None
# ---------------------------------------------------------------------------
# _validate_bullet_format
# ---------------------------------------------------------------------------
def test_validate_bullet_format_passes_valid_bullets() -> None:
content = (
"## Work\n"
"- (2026-04-10) [fact] Senior Python developer\n"
"- (2026-04-10) [pref] Prefers dark mode\n"
"- (2026-04-10) [instr] Always respond in bullet points\n"
)
warnings = _validate_bullet_format(content)
warnings = validate_bullet_format(content)
assert warnings == []
def test_validate_bullet_format_warns_on_missing_marker() -> None:
content = "- (2026-04-10) Senior Python developer\n"
warnings = _validate_bullet_format(content)
def test_validate_bullet_format_warns_on_nonstandard_bullet() -> None:
content = "## Facts\n- Senior Python developer\n"
warnings = validate_bullet_format(content)
assert len(warnings) == 1
assert "Malformed bullet" in warnings[0]
def test_validate_bullet_format_warns_on_missing_date() -> None:
content = "- [fact] Senior Python developer\n"
warnings = _validate_bullet_format(content)
assert len(warnings) == 1
assert "Malformed bullet" in warnings[0]
def test_validate_bullet_format_warns_on_unknown_marker() -> None:
content = "- (2026-04-10) [context] Working on project X\n"
warnings = _validate_bullet_format(content)
assert len(warnings) == 1
assert "Malformed bullet" in warnings[0]
def test_validate_bullet_format_ignores_non_bullet_lines() -> None:
content = "## Some Heading\nSome paragraph text\n"
warnings = _validate_bullet_format(content)
assert warnings == []
def test_validate_bullet_format_warns_on_old_format_without_marker() -> None:
content = "## About the user\n- (2026-04-10) Likes cats\n"
warnings = _validate_bullet_format(content)
assert len(warnings) == 1
# ---------------------------------------------------------------------------
# _save_memory — end-to-end with marker scope check
# ---------------------------------------------------------------------------
assert "Non-standard memory bullet" in warnings[0]
@pytest.mark.asyncio
async def test_save_memory_blocks_pref_in_team_before_commit() -> None:
recorder = _Recorder()
result = await _save_memory(
updated_memory="- (2026-04-10) [pref] Prefers dark mode\n",
old_memory=None,
llm=None,
apply_fn=recorder.apply,
commit_fn=recorder.commit,
rollback_fn=recorder.rollback,
label="team memory",
scope="team",
async def test_save_memory_normalizes_legacy_marker_bullets(monkeypatch) -> None:
target = type("Target", (), {"memory_md": ""})()
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.USER,
target_id="00000000-0000-0000-0000-000000000000",
content="- (2026-04-10) [fact] Legacy fact is preserved\n",
session=session,
)
assert result["status"] == "error"
assert recorder.commit_calls == 0
assert recorder.applied_content is None
assert result.status == "saved"
assert target.memory_md == "## Memory\n- 2026-04-10: Legacy fact is preserved"
@pytest.mark.asyncio
async def test_save_memory_allows_fact_in_team_and_commits() -> None:
recorder = _Recorder()
content = "- (2026-04-10) [fact] Weekly standup on Mondays\n"
result = await _save_memory(
updated_memory=content,
old_memory=None,
llm=None,
apply_fn=recorder.apply,
commit_fn=recorder.commit,
rollback_fn=recorder.rollback,
label="team memory",
scope="team",
async def test_save_memory_blocks_new_personal_heading_in_team_before_commit(
monkeypatch,
) -> None:
target = type("Target", (), {"shared_memory_md": ""})()
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.TEAM,
target_id=1,
content="## Preferences\n- 2026-04-10: Prefers dark mode\n",
session=session,
)
assert result["status"] == "saved"
assert recorder.commit_calls == 1
assert recorder.applied_content == content
assert result.status == "error"
assert session.commit_calls == 0
assert target.shared_memory_md == ""
@pytest.mark.asyncio
async def test_save_memory_includes_format_warnings() -> None:
recorder = _Recorder()
content = "- (2026-04-10) Missing marker text\n"
result = await _save_memory(
updated_memory=content,
old_memory=None,
llm=None,
apply_fn=recorder.apply,
commit_fn=recorder.commit,
rollback_fn=recorder.rollback,
label="memory",
scope="user",
async def test_save_memory_allows_grandfathered_personal_heading_in_team(
monkeypatch,
) -> None:
content = "## Preferences\n- 2026-04-10: Prefers dark mode\n"
target = type("Target", (), {"shared_memory_md": content})()
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.TEAM,
target_id=1,
content=content,
session=session,
)
assert result["status"] == "saved"
assert "format_warnings" in result
assert len(result["format_warnings"]) == 1
assert result.status == "saved"
assert session.commit_calls == 1
assert target.shared_memory_md == content.strip()
assert result.warnings
@pytest.mark.asyncio
async def test_save_memory_strips_preamble_before_heading(monkeypatch) -> None:
target = type("Target", (), {"memory_md": ""})()
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.USER,
target_id="00000000-0000-0000-0000-000000000000",
content="Sure, here is the update:\n\n## Facts\n- 2026-04-10: Likes cats\n",
session=session,
)
assert result.status == "saved"
assert target.memory_md == "## Facts\n- 2026-04-10: Likes cats"
@pytest.mark.asyncio
async def test_save_memory_rejects_long_no_heading_payload(monkeypatch) -> None:
target = type("Target", (), {"memory_md": ""})()
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.USER,
target_id="00000000-0000-0000-0000-000000000000",
content="NO_UPDATE because there is nothing durable to remember.",
session=session,
)
assert result.status == "error"
assert "## heading" in result.message
assert session.commit_calls == 0

187
surfsense_backend/tests/unit/services/test_memory_service.py Normal file
View file

@ -0,0 +1,187 @@
"""Unit tests for the first-class memory service."""
from types import SimpleNamespace
import pytest
from app.services.memory import (
MemoryScope,
reset_memory,
save_memory,
)
pytestmark = pytest.mark.unit
class _FakeSession:
def __init__(self) -> None:
self.commit_calls = 0
self.rollback_calls = 0
self.added = []
def add(self, obj) -> None:
self.added.append(obj)
async def commit(self) -> None:
self.commit_calls += 1
async def rollback(self) -> None:
self.rollback_calls += 1
@pytest.mark.asyncio
async def test_save_memory_saves_heading_based_memory(monkeypatch) -> None:
target = SimpleNamespace(memory_md="")
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.USER,
target_id="00000000-0000-0000-0000-000000000000",
content="## Facts\n- 2026-05-19: Anish works on SurfSense\n",
session=session,
)
assert result.status == "saved"
assert target.memory_md.startswith("## Facts")
assert session.commit_calls == 1
@pytest.mark.asyncio
async def test_save_memory_accepts_legacy_marker_payload(monkeypatch) -> None:
target = SimpleNamespace(memory_md="")
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.USER,
target_id="00000000-0000-0000-0000-000000000000",
content="- (2026-05-19) [fact] Legacy marker memory\n",
session=session,
)
assert result.status == "saved"
assert target.memory_md == "## Memory\n- 2026-05-19: Legacy marker memory"
@pytest.mark.asyncio
async def test_save_memory_rejects_long_no_heading_payload(monkeypatch) -> None:
target = SimpleNamespace(memory_md="## Facts\n- 2026-05-19: Existing\n")
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.USER,
target_id="00000000-0000-0000-0000-000000000000",
content="reasoning text before NO_UPDATE should not become saved memory",
session=session,
)
assert result.status == "error"
assert session.commit_calls == 0
assert target.memory_md.startswith("## Facts")
@pytest.mark.asyncio
async def test_save_memory_no_update_sentinel_is_no_op(monkeypatch) -> None:
existing = "## Preferences\n- 2026-05-20: Existing preference\n"
target = SimpleNamespace(memory_md=existing)
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.USER,
target_id="00000000-0000-0000-0000-000000000000",
content="NO_UPDATE",
session=session,
)
assert result.status == "no_op"
assert result.memory_md == existing
assert target.memory_md == existing
assert session.commit_calls == 0
@pytest.mark.asyncio
async def test_save_memory_no_update_sentinel_is_case_insensitive(monkeypatch) -> None:
existing = "## Preferences\n- 2026-05-20: Existing preference\n"
target = SimpleNamespace(memory_md=existing)
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.USER,
target_id="00000000-0000-0000-0000-000000000000",
content=" no update ",
session=session,
)
assert result.status == "no_op"
assert result.memory_md == existing
assert target.memory_md == existing
assert session.commit_calls == 0
@pytest.mark.asyncio
async def test_save_memory_grandfathers_existing_team_personal_heading(
monkeypatch,
) -> None:
content = "## Preferences\n- 2026-05-19: Existing legacy heading\n"
target = SimpleNamespace(shared_memory_md=content)
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await save_memory(
scope=MemoryScope.TEAM,
target_id=1,
content=content,
session=session,
)
assert result.status == "saved"
assert result.warnings
assert session.commit_calls == 1
@pytest.mark.asyncio
async def test_reset_memory_clears_memory(monkeypatch) -> None:
target = SimpleNamespace(memory_md="## Facts\n- 2026-05-19: Existing\n")
session = _FakeSession()
async def fake_load_target(**_kwargs):
return target
monkeypatch.setattr("app.services.memory.service._load_target", fake_load_target)
result = await reset_memory(
scope=MemoryScope.USER,
target_id="00000000-0000-0000-0000-000000000000",
session=session,
)
assert result.status == "saved"
assert target.memory_md == ""

1
surfsense_backend/tests/unit/tasks/chat/streaming/test_stream_output.py
View file

@ -89,7 +89,6 @@ async def test_stream_output_emits_text_lifecycle_and_updates_result() -> None:
"text_end:text-1",
]
assert result.accumulated_text == "Hello world"
assert result.agent_called_update_memory is False
async def test_stream_output_passes_runtime_context_to_agent() -> None:

7
surfsense_web/app/dashboard/[search_space_id]/search-space-settings/layout-shell.tsx
View file

@ -3,7 +3,6 @@
import {
BookText,
Bot,
Brain,
CircleUser,
Earth,
ImageIcon,
@ -27,7 +26,6 @@ export type SearchSpaceSettingsTab =
| "vision-models"
| "team-roles"
| "prompts"
| "team-memory"
| "public-links";
const DEFAULT_TAB: SearchSpaceSettingsTab = "general";
@ -89,11 +87,6 @@ export function SearchSpaceSettingsLayoutShell({
label: t("nav_system_instructions"),
icon: <BookText className="h-4 w-4" />,
},
{
value: "team-memory" as const,
label: "Team Memory",
icon: <Brain className="h-4 w-4" />,
},
{
value: "public-links" as const,
label: t("nav_public_links"),

6
surfsense_web/app/dashboard/[search_space_id]/search-space-settings/team-memory/page.tsx
View file

@ -1,6 +0,0 @@
import { TeamMemoryManager } from "@/components/settings/team-memory-manager";
export default async function Page({ params }: { params: Promise<{ search_space_id: string }> }) {
const { search_space_id } = await params;
return <TeamMemoryManager searchSpaceId={Number(search_space_id)} />;
}

293
surfsense_web/app/dashboard/[search_space_id]/user-settings/components/MemoryContent.tsx
View file

@ -1,293 +0,0 @@
"use client";
import { useAtomValue } from "jotai";
import { ArrowUp, ChevronDown, ClipboardCopy, Download, Info, Pencil } from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import { toast } from "sonner";
import { z } from "zod";
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
import { PlateEditor } from "@/components/editor/plate-editor";
import { Alert, AlertDescription } from "@/components/ui/alert";
import { Button } from "@/components/ui/button";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { Spinner } from "@/components/ui/spinner";
import { baseApiService } from "@/lib/apis/base-api.service";
const MEMORY_HARD_LIMIT = 25_000;
const MemoryReadSchema = z.object({
memory_md: z.string(),
});
export function MemoryContent() {
const activeSearchSpaceId = useAtomValue(activeSearchSpaceIdAtom);
const [memory, setMemory] = useState("");
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [editQuery, setEditQuery] = useState("");
const [editing, setEditing] = useState(false);
const [showInput, setShowInput] = useState(false);
const textareaRef = useRef<HTMLInputElement>(null);
const inputContainerRef = useRef<HTMLDivElement>(null);
const fetchMemory = useCallback(async () => {
try {
setLoading(true);
const data = await baseApiService.get("/api/v1/users/me/memory", MemoryReadSchema);
setMemory(data.memory_md);
} catch {
toast.error("Failed to load memory");
} finally {
setLoading(false);
}
}, []);
useEffect(() => {
fetchMemory();
}, [fetchMemory]);
useEffect(() => {
if (!showInput) return;
const handlePointerDownOutside = (event: MouseEvent | TouchEvent) => {
const target = event.target;
if (!(target instanceof Node)) return;
if (inputContainerRef.current?.contains(target)) return;
setShowInput(false);
};
document.addEventListener("mousedown", handlePointerDownOutside);
document.addEventListener("touchstart", handlePointerDownOutside, { passive: true });
return () => {
document.removeEventListener("mousedown", handlePointerDownOutside);
document.removeEventListener("touchstart", handlePointerDownOutside);
};
}, [showInput]);
const handleClear = async () => {
try {
setSaving(true);
const data = await baseApiService.put("/api/v1/users/me/memory", MemoryReadSchema, {
body: { memory_md: "" },
});
setMemory(data.memory_md);
toast.success("Memory cleared");
} catch {
toast.error("Failed to clear memory");
} finally {
setSaving(false);
}
};
const handleEdit = async () => {
const query = editQuery.trim();
if (!query) return;
try {
setEditing(true);
const data = await baseApiService.post("/api/v1/users/me/memory/edit", MemoryReadSchema, {
body: { query, search_space_id: Number(activeSearchSpaceId) },
});
setMemory(data.memory_md);
setEditQuery("");
setShowInput(false);
toast.success("Memory updated");
} catch {
toast.error("Failed to edit memory");
} finally {
setEditing(false);
}
};
const openInput = () => {
setShowInput(true);
requestAnimationFrame(() => textareaRef.current?.focus());
};
const handleDownload = () => {
if (!memory) return;
try {
const blob = new Blob([memory], { type: "text/markdown;charset=utf-8" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = "personal-memory.md";
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(url);
} catch {
toast.error("Failed to download memory");
}
};
const handleCopyMarkdown = async () => {
if (!memory) return;
try {
await navigator.clipboard.writeText(memory);
toast.success("Copied to clipboard");
} catch {
toast.error("Failed to copy memory");
}
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
handleEdit();
}
};
const displayMemory = memory.replace(/\(\d{4}-\d{2}-\d{2}\)\s*\[(fact|pref|instr)\]\s*/g, "");
const charCount = memory.length;
const getCounterColor = () => {
if (charCount > MEMORY_HARD_LIMIT) return "text-red-500";
if (charCount > 15_000) return "text-orange-500";
if (charCount > 10_000) return "text-yellow-500";
return "text-muted-foreground";
};
if (loading) {
return (
<div className="flex items-center justify-center py-12">
<Spinner size="md" className="text-muted-foreground" />
</div>
);
}
if (!memory) {
return (
<div className="flex flex-col items-center justify-center py-16 text-center">
<h3 className="text-base font-medium text-foreground">What does SurfSense remember?</h3>
<p className="mt-2 max-w-sm text-sm text-muted-foreground">
Nothing yet. SurfSense picks up on your preferences and context as you chat.
</p>
</div>
);
}
return (
<div className="space-y-4">
<Alert>
<Info />
<AlertDescription>
<p>
SurfSense uses this personal memory to personalize your responses across all
conversations.
</p>
</AlertDescription>
</Alert>
<div className="relative h-[380px] rounded-lg border bg-background">
<div className="h-full overflow-y-auto scrollbar-thin">
<PlateEditor
markdown={displayMemory}
readOnly
preset="readonly"
variant="default"
editorVariant="none"
className="px-5 py-4 text-sm min-h-full"
/>
</div>
{showInput ? (
<div className="absolute bottom-3 inset-x-3 z-10">
<div
ref={inputContainerRef}
className="relative flex h-[54px] items-center gap-2 rounded-[9999px] border bg-muted/60 backdrop-blur-sm pl-4 pr-1 shadow-sm"
>
<input
ref={textareaRef}
type="text"
value={editQuery}
onChange={(e) => setEditQuery(e.target.value)}
onKeyDown={handleKeyDown}
placeholder="Tell SurfSense what to remember or forget"
disabled={editing}
className="flex-1 bg-transparent text-sm outline-none placeholder:text-muted-foreground/70"
/>
<Button
type="button"
size="icon"
variant="ghost"
onClick={handleEdit}
disabled={editing || !editQuery.trim()}
className={`h-11 w-11 shrink-0 rounded-full ${
editing
? ""
: "bg-muted-foreground/15 hover:bg-accent hover:text-accent-foreground"
}`}
>
{editing ? (
<Spinner size="sm" />
) : (
<ArrowUp className="!h-5 !w-5 text-foreground" strokeWidth={2.25} />
)}
</Button>
</div>
</div>
) : (
<Button
type="button"
size="icon"
variant="secondary"
onClick={openInput}
className="absolute bottom-3 right-3 z-10 h-[54px] w-[54px] rounded-full border bg-muted/60 backdrop-blur-sm shadow-sm"
>
<Pencil className="!h-5 !w-5" />
</Button>
)}
</div>
<div className="flex items-center justify-between gap-2">
<span className={`text-xs shrink-0 ${getCounterColor()}`}>
{charCount.toLocaleString()} / {MEMORY_HARD_LIMIT.toLocaleString()}
<span className="hidden sm:inline"> characters</span>
<span className="sm:hidden"> chars</span>
{charCount > 15_000 && charCount <= MEMORY_HARD_LIMIT && " - Approaching limit"}
{charCount > MEMORY_HARD_LIMIT && " - Exceeds limit"}
</span>
<div className="flex items-center gap-1.5 sm:gap-2">
<Button
type="button"
variant="destructive"
size="sm"
className="text-xs sm:text-sm"
onClick={handleClear}
disabled={saving || editing || !memory}
>
<span className="hidden sm:inline">Reset Memory</span>
<span className="sm:hidden">Reset</span>
</Button>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button type="button" variant="secondary" size="sm" disabled={!memory}>
Export
<ChevronDown className="h-3 w-3 opacity-60" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuItem onClick={handleCopyMarkdown}>
<ClipboardCopy className="h-4 w-4 mr-2" />
Copy as Markdown
</DropdownMenuItem>
<DropdownMenuItem onClick={handleDownload}>
<Download className="h-4 w-4 mr-2" />
Download as Markdown
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
</div>
</div>
);
}

7
surfsense_web/app/dashboard/[search_space_id]/user-settings/layout-shell.tsx
View file

@ -1,7 +1,6 @@
"use client";
import {
Brain,
CircleUser,
Keyboard,
KeyRound,
@ -26,7 +25,6 @@ export type UserSettingsTab =
| "api-key"
| "prompts"
| "community-prompts"
| "memory"
| "agent-permissions"
| "agent-status"
| "purchases"
@ -75,11 +73,6 @@ export function UserSettingsLayoutShell({ searchSpaceId, children }: UserSetting
label: "Community Prompts",
icon: <Library className="h-4 w-4" />,
},
{
value: "memory" as const,
label: "Memory",
icon: <Brain className="h-4 w-4" />,
},
{
value: "agent-permissions" as const,
label: "Agent Permissions",

5
surfsense_web/app/dashboard/[search_space_id]/user-settings/memory/page.tsx
View file

@ -1,5 +0,0 @@
import { MemoryContent } from "../components/MemoryContent";
export default function Page() {
return <MemoryContent />;
}

26
surfsense_web/atoms/editor/editor-panel.atom.ts
View file

@ -3,10 +3,11 @@ import { rightPanelCollapsedAtom, rightPanelTabAtom } from "@/atoms/layout/right
interface EditorPanelState {
isOpen: boolean;
kind: "document" | "local_file";
kind: "document" | "local_file" | "memory";
documentId: number | null;
localFilePath: string | null;
searchSpaceId: number | null;
memoryScope: "user" | "team" | null;
title: string | null;
}
@ -16,6 +17,7 @@ const initialState: EditorPanelState = {
documentId: null,
localFilePath: null,
searchSpaceId: null,
memoryScope: null,
title: null,
};
@ -38,6 +40,12 @@ export const openEditorPanelAtom = atom(
title?: string;
searchSpaceId?: number;
}
| {
kind: "memory";
memoryScope: "user" | "team";
title?: string;
searchSpaceId?: number;
}
) => {
if (!get(editorPanelAtom).isOpen) {
set(preEditorCollapsedAtom, get(rightPanelCollapsedAtom));
@ -49,6 +57,21 @@ export const openEditorPanelAtom = atom(
documentId: null,
localFilePath: payload.localFilePath,
searchSpaceId: payload.searchSpaceId ?? null,
memoryScope: null,
title: payload.title ?? null,
});
set(rightPanelTabAtom, "editor");
set(rightPanelCollapsedAtom, false);
return;
}
if (payload.kind === "memory") {
set(editorPanelAtom, {
isOpen: true,
kind: "memory",
documentId: null,
localFilePath: null,
searchSpaceId: payload.searchSpaceId ?? null,
memoryScope: payload.memoryScope,
title: payload.title ?? null,
});
set(rightPanelTabAtom, "editor");
@ -61,6 +84,7 @@ export const openEditorPanelAtom = atom(
documentId: payload.documentId,
localFilePath: null,
searchSpaceId: payload.searchSpaceId,
memoryScope: null,
title: payload.title ?? null,
});
set(rightPanelTabAtom, "editor");

25
surfsense_web/components/assistant-ui/assistant-message.tsx
View file

@ -14,6 +14,7 @@ import {
ClipboardPaste,
CopyIcon,
DownloadIcon,
Dot,
ExternalLink,
Globe,
MessageCircleReply,
@ -330,9 +331,14 @@ const MessageInfoDropdown: FC<{ chatTurnId: string | null | undefined }> = ({ ch
{icon}
{name}
</span>
<span className="text-xs text-muted-foreground">
{counts.total_tokens.toLocaleString()} tokens
{costMicros && costMicros > 0 ? ` · ${formatTurnCost(costMicros)}` : ""}
<span className="flex items-center text-xs text-muted-foreground">
<span>{counts.total_tokens.toLocaleString()} tokens</span>
{costMicros && costMicros > 0 ? (
<>
<Dot className="size-4 shrink-0" aria-hidden="true" />
<span>{formatTurnCost(costMicros)}</span>
</>
) : null}
</span>
</ActionBarMorePrimitive.Item>
);
@ -342,11 +348,14 @@ const MessageInfoDropdown: FC<{ chatTurnId: string | null | undefined }> = ({ ch
className="focus:bg-accent focus:text-accent-foreground relative flex cursor-default flex-col items-start gap-0.5 rounded-sm px-2 py-1.5 text-sm outline-hidden select-none"
onSelect={(e) => e.preventDefault()}
>
<span className="text-xs text-muted-foreground">
{usage.total_tokens.toLocaleString()} tokens
{usage.cost_micros && usage.cost_micros > 0
? ` · ${formatTurnCost(usage.cost_micros)}`
: ""}
<span className="flex items-center text-xs text-muted-foreground">
<span>{usage.total_tokens.toLocaleString()} tokens</span>
{usage.cost_micros && usage.cost_micros > 0 ? (
<>
<Dot className="size-4 shrink-0" aria-hidden="true" />
<span>{formatTurnCost(usage.cost_micros)}</span>
</>
) : null}
</span>
</ActionBarMorePrimitive.Item>
)}

143
surfsense_web/components/documents/DocumentNode.tsx
View file

@ -9,6 +9,7 @@ import {
MoreHorizontal,
Move,
Pencil,
RotateCcw,
Trash2,
} from "lucide-react";
import React, { useCallback, useRef, useState } from "react";
@ -61,8 +62,13 @@ interface DocumentNodeProps {
onEdit: (doc: DocumentNodeDoc) => void;
onDelete: (doc: DocumentNodeDoc) => void;
onMove: (doc: DocumentNodeDoc) => void;
onReset?: (doc: DocumentNodeDoc) => void;
onExport?: (doc: DocumentNodeDoc, format: string) => void;
onVersionHistory?: (doc: DocumentNodeDoc) => void;
canDelete?: boolean;
canMove?: boolean;
canMention?: boolean;
canEdit?: boolean;
contextMenuOpen?: boolean;
onContextMenuOpenChange?: (open: boolean) => void;
}
@ -76,8 +82,13 @@ export const DocumentNode = React.memo(function DocumentNode({
onEdit,
onDelete,
onMove,
onReset,
onExport,
onVersionHistory,
canDelete = true,
canMove = true,
canMention = true,
canEdit = true,
contextMenuOpen,
onContextMenuOpenChange,
}: DocumentNodeProps) {
@ -85,8 +96,13 @@ export const DocumentNode = React.memo(function DocumentNode({
const isFailed = statusState === "failed";
const isProcessing = statusState === "pending" || statusState === "processing";
const isUnavailable = isProcessing || isFailed;
const isSelectable = !isUnavailable;
const isEditable = EDITABLE_DOCUMENT_TYPES.has(doc.document_type) && !isUnavailable;
const isMemoryDocument =
doc.document_type === "USER_MEMORY" || doc.document_type === "TEAM_MEMORY";
const isSelectable = canMention && !isUnavailable;
const isEditable =
canEdit &&
(isMemoryDocument || EDITABLE_DOCUMENT_TYPES.has(doc.document_type)) &&
!isUnavailable;
const handleCheckChange = useCallback(() => {
if (isSelectable) {
@ -94,13 +110,22 @@ export const DocumentNode = React.memo(function DocumentNode({
}
}, [doc, isMentioned, isSelectable, onToggleChatMention]);
const handlePrimaryClick = useCallback(() => {
if (canMention) {
handleCheckChange();
return;
}
onPreview(doc);
}, [canMention, doc, handleCheckChange, onPreview]);
const [{ isDragging }, drag] = useDrag(
() => ({
type: DND_TYPES.DOCUMENT,
item: { id: doc.id },
canDrag: canMove,
collect: (monitor) => ({ isDragging: monitor.isDragging() }),
}),
[doc.id]
[canMove, doc.id]
);
const [dropdownOpen, setDropdownOpen] = useState(false);
@ -130,9 +155,11 @@ export const DocumentNode = React.memo(function DocumentNode({
const attachRef = useCallback(
(node: HTMLDivElement | null) => {
(rowRef as React.MutableRefObject<HTMLDivElement | null>).current = node;
drag(node);
if (canMove) {
drag(node);
}
},
[drag]
[canMove, drag]
);
return (
@ -187,12 +214,32 @@ export const DocumentNode = React.memo(function DocumentNode({
);
}
return (
<Checkbox
checked={isMentioned}
onCheckedChange={handleCheckChange}
onClick={(e) => e.stopPropagation()}
className="h-3.5 w-3.5 shrink-0"
/>
<>
{isMemoryDocument ? (
<span aria-disabled="true" className="h-3.5 w-3.5 shrink-0 cursor-default">
<Checkbox
checked={false}
disabled
aria-disabled
className="h-3.5 w-3.5 pointer-events-none"
/>
</span>
) : canMention ? (
<Checkbox
checked={isMentioned}
onCheckedChange={handleCheckChange}
onClick={(e) => e.stopPropagation()}
className="h-3.5 w-3.5 shrink-0"
/>
) : (
<span className="flex h-3.5 w-3.5 shrink-0 items-center justify-center">
{getDocumentTypeIcon(
doc.document_type as DocumentTypeEnum,
"h-3.5 w-3.5 text-muted-foreground"
)}
</span>
)}
</>
);
})()}
@ -205,8 +252,8 @@ export const DocumentNode = React.memo(function DocumentNode({
<Button
type="button"
variant="ghost"
aria-disabled={!isSelectable}
onClick={handleCheckChange}
aria-disabled={canMention ? !isSelectable : false}
onClick={handlePrimaryClick}
className="h-full min-w-0 flex-1 justify-start bg-transparent px-0 py-0 text-left font-normal text-inherit hover:bg-transparent hover:text-inherit"
>
<span ref={titleRef} className="min-w-0 flex-1 truncate">
@ -268,11 +315,18 @@ export const DocumentNode = React.memo(function DocumentNode({
Edit
</DropdownMenuItem>
)}
<DropdownMenuItem onClick={() => onMove(doc)}>
<Move className="mr-2 h-4 w-4" />
Move to...
</DropdownMenuItem>
{onExport && (
{canMove && (
<DropdownMenuItem onClick={() => onMove(doc)}>
<Move className="mr-2 h-4 w-4" />
Move to...
</DropdownMenuItem>
)}
{onExport && isMemoryDocument ? (
<DropdownMenuItem disabled={isUnavailable} onClick={() => handleExport("md")}>
<Download className="mr-2 h-4 w-4" />
Export as MD
</DropdownMenuItem>
) : onExport ? (
<DropdownMenuSub>
<DropdownMenuSubTrigger disabled={isUnavailable}>
<Download className="mr-2 h-4 w-4" />
@ -282,17 +336,25 @@ export const DocumentNode = React.memo(function DocumentNode({
<ExportDropdownItems onExport={handleExport} exporting={exporting} />
</DropdownMenuSubContent>
</DropdownMenuSub>
)}
) : null}
{onVersionHistory && isVersionableType(doc.document_type) && (
<DropdownMenuItem disabled={isUnavailable} onClick={() => onVersionHistory(doc)}>
<History className="mr-2 h-4 w-4" />
Versions
</DropdownMenuItem>
)}
<DropdownMenuItem disabled={isProcessing} onClick={() => onDelete(doc)}>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</DropdownMenuItem>
{isMemoryDocument && onReset && (
<DropdownMenuItem onClick={() => onReset(doc)}>
<RotateCcw className="mr-2 h-4 w-4" />
Reset
</DropdownMenuItem>
)}
{canDelete && (
<DropdownMenuItem disabled={isProcessing} onClick={() => onDelete(doc)}>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</DropdownMenuItem>
)}
</DropdownMenuContent>
</DropdownMenu>
</span>
@ -311,11 +373,18 @@ export const DocumentNode = React.memo(function DocumentNode({
Edit
</ContextMenuItem>
)}
<ContextMenuItem onClick={() => onMove(doc)}>
<Move className="mr-2 h-4 w-4" />
Move to...
</ContextMenuItem>
{onExport && (
{canMove && (
<ContextMenuItem onClick={() => onMove(doc)}>
<Move className="mr-2 h-4 w-4" />
Move to...
</ContextMenuItem>
)}
{onExport && isMemoryDocument ? (
<ContextMenuItem disabled={isUnavailable} onClick={() => handleExport("md")}>
<Download className="mr-2 h-4 w-4" />
Export as MD
</ContextMenuItem>
) : onExport ? (
<ContextMenuSub>
<ContextMenuSubTrigger disabled={isUnavailable}>
<Download className="mr-2 h-4 w-4" />
@ -325,17 +394,25 @@ export const DocumentNode = React.memo(function DocumentNode({
<ExportContextItems onExport={handleExport} exporting={exporting} />
</ContextMenuSubContent>
</ContextMenuSub>
)}
) : null}
{onVersionHistory && isVersionableType(doc.document_type) && (
<ContextMenuItem disabled={isUnavailable} onClick={() => onVersionHistory(doc)}>
<History className="mr-2 h-4 w-4" />
Versions
</ContextMenuItem>
)}
<ContextMenuItem disabled={isProcessing} onClick={() => onDelete(doc)}>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</ContextMenuItem>
{isMemoryDocument && onReset && (
<ContextMenuItem onClick={() => onReset(doc)}>
<RotateCcw className="mr-2 h-4 w-4" />
Reset
</ContextMenuItem>
)}
{canDelete && (
<ContextMenuItem disabled={isProcessing} onClick={() => onDelete(doc)}>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</ContextMenuItem>
)}
</ContextMenuContent>
)}
</ContextMenu>

79
surfsense_web/components/documents/FolderTreeView.tsx
View file

@ -32,6 +32,7 @@ interface FolderTreeViewProps {
onEditDocument: (doc: DocumentNodeDoc) => void;
onDeleteDocument: (doc: DocumentNodeDoc) => void;
onMoveDocument: (doc: DocumentNodeDoc) => void;
onResetDocument?: (doc: DocumentNodeDoc) => void;
onExportDocument?: (doc: DocumentNodeDoc, format: string) => void;
onVersionHistory?: (doc: DocumentNodeDoc) => void;
activeTypes: DocumentTypeEnum[];
@ -74,6 +75,7 @@ export function FolderTreeView({
onEditDocument,
onDeleteDocument,
onMoveDocument,
onResetDocument,
onExportDocument,
onVersionHistory,
activeTypes,
@ -236,6 +238,47 @@ export function FolderTreeView({
return states;
}, [folders, docsByFolder, foldersByParent, folderMap]);
const renderDocumentNode = useCallback(
(d: DocumentNodeDoc, depth: number) => {
const isMemoryDocument =
d.document_type === "USER_MEMORY" || d.document_type === "TEAM_MEMORY";
return (
<DocumentNode
key={`doc-${d.id}`}
doc={d}
depth={depth}
isMentioned={!isMemoryDocument && mentionedDocKeys.has(getMentionDocKey(d))}
onToggleChatMention={onToggleChatMention}
onPreview={onPreviewDocument}
onEdit={onEditDocument}
onDelete={onDeleteDocument}
onMove={onMoveDocument}
onReset={onResetDocument}
onExport={onExportDocument}
onVersionHistory={isMemoryDocument ? undefined : onVersionHistory}
canDelete={!isMemoryDocument}
canMove={!isMemoryDocument}
canMention={!isMemoryDocument}
canEdit
contextMenuOpen={openContextMenuId === `doc-${d.id}`}
onContextMenuOpenChange={(open) => setOpenContextMenuId(open ? `doc-${d.id}` : null)}
/>
);
},
[
mentionedDocKeys,
onDeleteDocument,
onEditDocument,
onExportDocument,
onMoveDocument,
onPreviewDocument,
onResetDocument,
onToggleChatMention,
onVersionHistory,
openContextMenuId,
]
);
function renderLevel(parentId: number | null, depth: number): React.ReactNode[] {
const key = parentId ?? "root";
const childFolders = (foldersByParent[key] ?? []).slice().sort((a, b) => {
@ -263,23 +306,7 @@ export function FolderTreeView({
return state === "pending" || state === "processing";
});
for (const d of processingDocs) {
nodes.push(
<DocumentNode
key={`doc-${d.id}`}
doc={d}
depth={depth}
isMentioned={mentionedDocKeys.has(getMentionDocKey(d))}
onToggleChatMention={onToggleChatMention}
onPreview={onPreviewDocument}
onEdit={onEditDocument}
onDelete={onDeleteDocument}
onMove={onMoveDocument}
onExport={onExportDocument}
onVersionHistory={onVersionHistory}
contextMenuOpen={openContextMenuId === `doc-${d.id}`}
onContextMenuOpenChange={(open) => setOpenContextMenuId(open ? `doc-${d.id}` : null)}
/>
);
nodes.push(renderDocumentNode(d, depth));
}
}
@ -343,23 +370,7 @@ export function FolderTreeView({
: childDocs;
for (const d of remainingDocs) {
nodes.push(
<DocumentNode
key={`doc-${d.id}`}
doc={d}
depth={depth}
isMentioned={mentionedDocKeys.has(getMentionDocKey(d))}
onToggleChatMention={onToggleChatMention}
onPreview={onPreviewDocument}
onEdit={onEditDocument}
onDelete={onDeleteDocument}
onMove={onMoveDocument}
onExport={onExportDocument}
onVersionHistory={onVersionHistory}
contextMenuOpen={openContextMenuId === `doc-${d.id}`}
onContextMenuOpenChange={(open) => setOpenContextMenuId(open ? `doc-${d.id}` : null)}
/>
);
nodes.push(renderDocumentNode(d, depth));
}
return nodes;

147
surfsense_web/components/editor-panel/editor-panel.tsx
View file

@ -17,10 +17,17 @@ import { toast } from "sonner";
import { closeEditorPanelAtom, editorPanelAtom } from "@/atoms/editor/editor-panel.atom";
import { VersionHistoryButton } from "@/components/documents/version-history";
import { SourceCodeEditor } from "@/components/editor/source-code-editor";
import {
fetchMemoryEditorDocument,
getMemoryLimitState,
type MemoryLimits,
saveMemoryMarkdown,
} from "@/components/editor-panel/memory";
import { MarkdownViewer } from "@/components/markdown-viewer";
import { Alert, AlertDescription } from "@/components/ui/alert";
import { Button } from "@/components/ui/button";
import { Drawer, DrawerContent, DrawerHandle, DrawerTitle } from "@/components/ui/drawer";
import { Separator } from "@/components/ui/separator";
import { Spinner } from "@/components/ui/spinner";
import { useMediaQuery } from "@/hooks/use-media-query";
import { useElectronAPI } from "@/hooks/use-platform";
@ -107,13 +114,15 @@ export function EditorPanelContent({
kind = "document",
documentId,
localFilePath,
memoryScope,
searchSpaceId,
title,
onClose,
}: {
kind?: "document" | "local_file";
kind?: "document" | "local_file" | "memory";
documentId?: number;
localFilePath?: string;
memoryScope?: "user" | "team";
searchSpaceId?: number;
title: string | null;
onClose?: () => void;
@ -125,6 +134,7 @@ export function EditorPanelContent({
const [saving, setSaving] = useState(false);
const [downloading, setDownloading] = useState(false);
const [isEditing, setIsEditing] = useState(false);
const [memoryLimits, setMemoryLimits] = useState<MemoryLimits | null>(null);
const [editedMarkdown, setEditedMarkdown] = useState<string | null>(null);
const [localFileContent, setLocalFileContent] = useState("");
@ -135,6 +145,7 @@ export function EditorPanelContent({
const changeCountRef = useRef(0);
const [displayTitle, setDisplayTitle] = useState(title || "Untitled");
const isLocalFileMode = kind === "local_file";
const isMemoryMode = kind === "memory";
const editorRenderMode: EditorRenderMode = isLocalFileMode ? "source_code" : "rich_markdown";
const resolveLocalVirtualPath = useCallback(
@ -165,6 +176,7 @@ export function EditorPanelContent({
setLocalFileContent("");
setHasCopied(false);
setIsEditing(false);
setMemoryLimits(null);
initialLoadDone.current = false;
changeCountRef.current = 0;
@ -199,6 +211,24 @@ export function EditorPanelContent({
initialLoadDone.current = true;
return;
}
if (isMemoryMode) {
if (!memoryScope) throw new Error("Missing memory context");
const { document, limits } = await fetchMemoryEditorDocument({
scope: memoryScope,
searchSpaceId,
title,
signal: controller.signal,
});
if (controller.signal.aborted) return;
setMemoryLimits(limits);
const content: EditorContent = document;
markdownRef.current = content.source_markdown;
setDisplayTitle(content.title);
setEditorDoc(content);
initialLoadDone.current = true;
return;
}
if (!documentId || !searchSpaceId) {
throw new Error("Missing document context");
}
@ -253,7 +283,9 @@ export function EditorPanelContent({
documentId,
electronAPI,
isLocalFileMode,
isMemoryMode,
localFilePath,
memoryScope,
resolveLocalVirtualPath,
searchSpaceId,
title,
@ -267,13 +299,20 @@ export function EditorPanelContent({
};
}, []);
const handleMarkdownChange = useCallback((md: string) => {
markdownRef.current = md;
if (!initialLoadDone.current) return;
changeCountRef.current += 1;
if (changeCountRef.current <= 1) return;
setEditedMarkdown(md);
}, []);
const handleMarkdownChange = useCallback(
(md: string) => {
if (!isEditing) return;
markdownRef.current = md;
if (!initialLoadDone.current) return;
changeCountRef.current += 1;
if (changeCountRef.current <= 1) return;
const savedContent = editorDoc?.source_markdown ?? "";
setEditedMarkdown(md === savedContent ? null : md);
},
[editorDoc?.source_markdown, isEditing]
);
const handleCopy = useCallback(async () => {
try {
@ -316,6 +355,23 @@ export function EditorPanelContent({
setEditedMarkdown(markdownRef.current === contentToSave ? null : markdownRef.current);
return true;
}
if (isMemoryMode) {
if (!memoryScope) throw new Error("Missing memory context");
const { markdown: savedContent, limits } = await saveMemoryMarkdown({
scope: memoryScope,
searchSpaceId,
markdown: markdownRef.current,
});
markdownRef.current = savedContent;
setMemoryLimits(limits ?? memoryLimits);
setEditorDoc((prev) => (prev ? { ...prev, source_markdown: savedContent } : prev));
setEditedMarkdown(null);
if (!options?.silent) {
toast.success("Memory saved");
}
return true;
}
if (!searchSpaceId || !documentId) {
throw new Error("Missing document context");
}
@ -361,14 +417,18 @@ export function EditorPanelContent({
documentId,
electronAPI,
isLocalFileMode,
isMemoryMode,
localFilePath,
memoryLimits,
memoryScope,
resolveLocalVirtualPath,
searchSpaceId,
]
);
const isEditableType = editorDoc
? (editorRenderMode === "source_code" ||
? (isMemoryMode ||
editorRenderMode === "source_code" ||
EDITABLE_DOCUMENT_TYPES.has(editorDoc.document_type ?? "")) &&
!isLargeDocument
: false;
@ -381,6 +441,17 @@ export function EditorPanelContent({
const showDesktopHeader = !!onClose;
const showEditingActions = isEditableType && isEditing;
const localFileLanguage = inferMonacoLanguageFromPath(localFilePath);
const activeMarkdown = editedMarkdown ?? editorDoc?.source_markdown ?? "";
const memoryLimitState = isMemoryMode
? getMemoryLimitState(activeMarkdown.length, memoryLimits)
: null;
const memoryCounterClassName =
memoryLimitState?.level === "error"
? "text-red-500"
: memoryLimitState?.level === "warning"
? "text-orange-500"
: "text-muted-foreground";
const saveDisabled = saving || !hasUnsavedChanges || (memoryLimitState?.isOverLimit ?? false);
const handleCancelEditing = useCallback(() => {
const savedContent = editorDoc?.source_markdown ?? "";
@ -466,6 +537,17 @@ export function EditorPanelContent({
<div className="grid h-10 grid-cols-[minmax(0,1fr)_auto] items-center gap-3 border-b px-4">
<div className="min-w-0 flex flex-1 items-center gap-2">
<p className="truncate text-sm text-muted-foreground">{displayTitle}</p>
{memoryLimitState && (
<>
<Separator
orientation="vertical"
className="mx-1 bg-border data-[orientation=vertical]:h-4 data-[orientation=vertical]:w-px dark:bg-white/10"
/>
<span className={`shrink-0 text-xs ${memoryCounterClassName}`}>
{memoryLimitState.label}
</span>
</>
)}
</div>
<div className="flex items-center gap-1 shrink-0">
{showEditingActions ? (
@ -487,7 +569,7 @@ export function EditorPanelContent({
const saveSucceeded = await handleSave({ silent: true });
if (saveSucceeded) setIsEditing(false);
}}
disabled={saving || !hasUnsavedChanges}
disabled={saveDisabled}
>
<span className={saving ? "opacity-0" : ""}>Save</span>
{saving && <Spinner size="xs" className="absolute" />}
@ -495,7 +577,7 @@ export function EditorPanelContent({
</>
) : (
<>
{!isLocalFileMode && editorDoc?.document_type && documentId && (
{!isLocalFileMode && !isMemoryMode && editorDoc?.document_type && documentId && (
<VersionHistoryButton
documentId={documentId}
documentType={editorDoc.document_type}
@ -539,6 +621,17 @@ export function EditorPanelContent({
<div className="flex h-14 items-center justify-between border-b px-4 shrink-0">
<div className="flex flex-1 min-w-0 items-center gap-2">
<h2 className="text-sm font-semibold truncate">{displayTitle}</h2>
{memoryLimitState && (
<>
<Separator
orientation="vertical"
className="mx-1 bg-border data-[orientation=vertical]:h-4 data-[orientation=vertical]:w-px dark:bg-white/10"
/>
<span className={`shrink-0 text-xs ${memoryCounterClassName}`}>
{memoryLimitState.label}
</span>
</>
)}
</div>
<div className="flex items-center gap-1 shrink-0">
{showEditingActions ? (
@ -560,7 +653,7 @@ export function EditorPanelContent({
const saveSucceeded = await handleSave({ silent: true });
if (saveSucceeded) setIsEditing(false);
}}
disabled={saving || !hasUnsavedChanges}
disabled={saveDisabled}
>
<span className={saving ? "opacity-0" : ""}>Save</span>
{saving && <Spinner size="xs" className="absolute" />}
@ -568,7 +661,7 @@ export function EditorPanelContent({
</>
) : (
<>
{!isLocalFileMode && editorDoc?.document_type && documentId && (
{!isLocalFileMode && !isMemoryMode && editorDoc?.document_type && documentId && (
<VersionHistoryButton
documentId={documentId}
documentType={editorDoc.document_type}
@ -664,7 +757,13 @@ export function EditorPanelContent({
<div className="flex h-full min-h-0 flex-col">
<div className="flex-1 min-h-0 overflow-hidden">
<PlateEditor
key={`${isLocalFileMode ? (localFilePath ?? "local-file") : documentId}-${isEditing ? "editing" : "viewing"}`}
key={`${
isMemoryMode
? `memory-${memoryScope ?? "user"}`
: isLocalFileMode
? (localFilePath ?? "local-file")
: documentId
}-${isEditing ? "editing" : "viewing"}`}
preset="full"
markdown={editorDoc.source_markdown}
onMarkdownChange={handleMarkdownChange}
@ -672,14 +771,14 @@ export function EditorPanelContent({
placeholder="Start writing..."
editorVariant="default"
allowModeToggle={false}
reserveToolbarSpace={isEditing}
reserveToolbarSpace
defaultEditing={isEditing}
className="**:[[role=toolbar]]:bg-sidebar!"
// Render `[citation:N]` badges in view mode only.
// Edit mode keeps raw text so the user can edit/delete
// tokens directly. `local_file` never reaches this branch
// (handled by the source_code editor above).
enableCitations={!isEditing && !isLocalFileMode}
enableCitations={!isEditing && !isLocalFileMode && !isMemoryMode}
/>
</div>
</div>
@ -708,7 +807,9 @@ function DesktopEditorPanel() {
const hasTarget =
panelState.kind === "document"
? !!panelState.documentId && !!panelState.searchSpaceId
: !!panelState.localFilePath;
: panelState.kind === "local_file"
? !!panelState.localFilePath
: !!panelState.memoryScope;
if (!panelState.isOpen || !hasTarget) return null;
return (
@ -717,6 +818,7 @@ function DesktopEditorPanel() {
kind={panelState.kind}
documentId={panelState.documentId ?? undefined}
localFilePath={panelState.localFilePath ?? undefined}
memoryScope={panelState.memoryScope ?? undefined}
searchSpaceId={panelState.searchSpaceId ?? undefined}
title={panelState.title}
onClose={closePanel}
@ -734,7 +836,7 @@ function MobileEditorDrawer() {
const hasTarget =
panelState.kind === "document"
? !!panelState.documentId && !!panelState.searchSpaceId
: !!panelState.localFilePath;
: !!panelState.memoryScope;
if (!hasTarget) return null;
return (
@ -756,6 +858,7 @@ function MobileEditorDrawer() {
kind={panelState.kind}
documentId={panelState.documentId ?? undefined}
localFilePath={panelState.localFilePath ?? undefined}
memoryScope={panelState.memoryScope ?? undefined}
searchSpaceId={panelState.searchSpaceId ?? undefined}
title={panelState.title}
/>
@ -771,7 +874,9 @@ export function EditorPanel() {
const hasTarget =
panelState.kind === "document"
? !!panelState.documentId && !!panelState.searchSpaceId
: !!panelState.localFilePath;
: panelState.kind === "local_file"
? !!panelState.localFilePath
: !!panelState.memoryScope;
if (!panelState.isOpen || !hasTarget) return null;
if (!isDesktop && panelState.kind === "local_file") return null;
@ -789,7 +894,9 @@ export function MobileEditorPanel() {
const hasTarget =
panelState.kind === "document"
? !!panelState.documentId && !!panelState.searchSpaceId
: !!panelState.localFilePath;
: panelState.kind === "local_file"
? !!panelState.localFilePath
: !!panelState.memoryScope;
if (isDesktop || !panelState.isOpen || !hasTarget || panelState.kind === "local_file")
return null;

116
surfsense_web/components/editor-panel/memory.ts Normal file
View file

@ -0,0 +1,116 @@
"use client";
import { authenticatedFetch } from "@/lib/auth-utils";
export type MemoryScope = "user" | "team";
export interface MemoryLimits {
soft: number;
hard: number;
}
export type MemoryLimitLevel = "ok" | "warning" | "error";
export interface MemoryEditorDocument {
document_id: number;
title: string;
document_type: "USER_MEMORY" | "TEAM_MEMORY";
source_markdown: string;
}
interface MemoryReadResponse {
memory_md?: string;
limits?: MemoryLimits;
}
function getMemoryPath(scope: MemoryScope, searchSpaceId?: number | null) {
if (scope === "user") return "/api/v1/users/me/memory";
if (!searchSpaceId) throw new Error("Missing search space context");
return `/api/v1/searchspaces/${searchSpaceId}/memory`;
}
function getBackendUrl(path: string) {
return `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}${path}`;
}
export function getMemoryLimitState(length: number, limits?: MemoryLimits | null) {
if (!limits) {
return {
level: "ok" as MemoryLimitLevel,
label: `${length.toLocaleString()} chars`,
isOverLimit: false,
};
}
const isOverLimit = length > limits.hard;
const isNearLimit = length > limits.soft;
const level: MemoryLimitLevel = isOverLimit ? "error" : isNearLimit ? "warning" : "ok";
const suffix = isOverLimit ? " - Exceeds limit" : isNearLimit ? " - Approaching limit" : "";
return {
level,
label: `${length.toLocaleString()}/${limits.hard.toLocaleString()} chars${suffix}`,
isOverLimit,
};
}
export async function fetchMemoryEditorDocument({
scope,
searchSpaceId,
title,
signal,
}: {
scope: MemoryScope;
searchSpaceId?: number | null;
title?: string | null;
signal?: AbortSignal;
}) {
const response = await authenticatedFetch(getBackendUrl(getMemoryPath(scope, searchSpaceId)), {
method: "GET",
signal,
});
if (!response.ok) {
const errorData = await response.json().catch(() => ({ detail: "Failed to fetch memory" }));
throw new Error(errorData.detail || "Failed to fetch memory");
}
const data = (await response.json()) as MemoryReadResponse;
const isTeamMemory = scope === "team";
return {
limits: data.limits ?? null,
document: {
document_id: isTeamMemory ? -1002 : -1001,
title: title || (isTeamMemory ? "Team Memory" : "Personal Memory"),
document_type: isTeamMemory ? "TEAM_MEMORY" : "USER_MEMORY",
source_markdown: data.memory_md ?? "",
} satisfies MemoryEditorDocument,
};
}
export async function saveMemoryMarkdown({
scope,
searchSpaceId,
markdown,
}: {
scope: MemoryScope;
searchSpaceId?: number | null;
markdown: string;
}) {
const response = await authenticatedFetch(getBackendUrl(getMemoryPath(scope, searchSpaceId)), {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ memory_md: markdown }),
});
if (!response.ok) {
const errorData = await response.json().catch(() => ({ detail: "Failed to save memory" }));
throw new Error(errorData.detail || "Failed to save memory");
}
const data = (await response.json()) as MemoryReadResponse;
return {
markdown: data.memory_md ?? markdown,
limits: data.limits,
};
}

19
surfsense_web/components/layout/ui/right-panel/RightPanel.tsx
View file

@ -103,7 +103,11 @@ export function RightPanelToggleButton({
const reportOpen = reportState.isOpen && !!reportState.reportId;
const editorOpen =
editorState.isOpen &&
(editorState.kind === "document" ? !!editorState.documentId : !!editorState.localFilePath);
(editorState.kind === "document"
? !!editorState.documentId
: editorState.kind === "memory"
? !!editorState.memoryScope
: !!editorState.localFilePath);
const hitlEditOpen = hitlEditState.isOpen && !!hitlEditState.onSave;
const citationOpen = citationState.isOpen && citationState.chunkId != null;
const hasContent = documentsOpen || reportOpen || editorOpen || hitlEditOpen || citationOpen;
@ -151,7 +155,11 @@ export function RightPanelExpandButton() {
const reportOpen = reportState.isOpen && !!reportState.reportId;
const editorOpen =
editorState.isOpen &&
(editorState.kind === "document" ? !!editorState.documentId : !!editorState.localFilePath);
(editorState.kind === "document"
? !!editorState.documentId
: editorState.kind === "memory"
? !!editorState.memoryScope
: !!editorState.localFilePath);
const hitlEditOpen = hitlEditState.isOpen && !!hitlEditState.onSave;
const citationOpen = citationState.isOpen && citationState.chunkId != null;
const hasContent = documentsOpen || reportOpen || editorOpen || hitlEditOpen || citationOpen;
@ -193,7 +201,11 @@ export function RightPanel({
const reportOpen = reportState.isOpen && !!reportState.reportId;
const editorOpen =
editorState.isOpen &&
(editorState.kind === "document" ? !!editorState.documentId : !!editorState.localFilePath);
(editorState.kind === "document"
? !!editorState.documentId
: editorState.kind === "memory"
? !!editorState.memoryScope
: !!editorState.localFilePath);
const hitlEditOpen = hitlEditState.isOpen && !!hitlEditState.onSave;
const citationOpen = citationState.isOpen && citationState.chunkId != null;
@ -292,6 +304,7 @@ export function RightPanel({
kind={editorState.kind}
documentId={editorState.documentId ?? undefined}
localFilePath={editorState.localFilePath ?? undefined}
memoryScope={editorState.memoryScope ?? undefined}
searchSpaceId={editorState.searchSpaceId ?? undefined}
title={editorState.title}
onClose={closeEditor}

127
surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx
View file

@ -88,7 +88,43 @@ const DesktopLocalTabContent = dynamic(
{ ssr: false }
);
const NON_DELETABLE_DOCUMENT_TYPES: readonly string[] = ["SURFSENSE_DOCS"];
const NON_DELETABLE_DOCUMENT_TYPES: readonly string[] = [
"SURFSENSE_DOCS",
"USER_MEMORY",
"TEAM_MEMORY",
];
const MEMORY_DOCUMENTS: DocumentNodeDoc[] = [
{
id: -1001,
title: "MEMORY.md",
document_type: "USER_MEMORY",
folderId: null,
status: { state: "ready" },
},
{
id: -1002,
title: "TEAM_MEMORY.md",
document_type: "TEAM_MEMORY",
folderId: null,
status: { state: "ready" },
},
];
function isMemoryDocument(doc: { document_type: string }) {
return doc.document_type === "USER_MEMORY" || doc.document_type === "TEAM_MEMORY";
}
function downloadTextFile(content: string, fileName: string, type = "text/markdown;charset=utf-8") {
const blob = new Blob([content], { type });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = fileName;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(url);
}
const LOCAL_FILESYSTEM_TRUST_KEY = "surfsense.local-filesystem-trust.v1";
const MAX_LOCAL_FILESYSTEM_ROOTS = 10;
@ -784,6 +820,30 @@ function AuthenticatedDocumentsSidebarBase({
const handleExportDocument = useCallback(
async (doc: DocumentNodeDoc, format: string) => {
if (isMemoryDocument(doc)) {
try {
const endpoint =
doc.document_type === "USER_MEMORY"
? `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/users/me/memory`
: `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/searchspaces/${searchSpaceId}/memory`;
const response = await authenticatedFetch(endpoint, { method: "GET" });
if (!response.ok) {
const errorData = await response.json().catch(() => ({ detail: "Export failed" }));
throw new Error(errorData.detail || "Export failed");
}
const data = (await response.json()) as { memory_md?: string };
downloadTextFile(
data.memory_md ?? "",
doc.title.endsWith(".md") ? doc.title : `${doc.title}.md`
);
return;
} catch (err) {
console.error("Memory export failed:", err);
toast.error(err instanceof Error ? err.message : "Export failed");
return;
}
}
const safeTitle =
doc.title
.replace(/[^a-zA-Z0-9 _-]/g, "_")
@ -879,6 +939,7 @@ function AuthenticatedDocumentsSidebarBase({
const handleToggleChatMention = useCallback(
(doc: { id: number; title: string; document_type: string }, isMentioned: boolean) => {
if (isMemoryDocument(doc)) return;
const key = getMentionDocKey({ ...doc, kind: "doc" });
if (isMentioned) {
setSidebarDocs((prev) => prev.filter((d) => getMentionDocKey(d) !== key));
@ -927,11 +988,66 @@ function AuthenticatedDocumentsSidebarBase({
[treeFolders, setSidebarDocs]
);
const treeDocumentsWithMemory = useMemo(
() => [...MEMORY_DOCUMENTS, ...treeDocuments],
[treeDocuments]
);
const searchFilteredDocuments = useMemo(() => {
const query = debouncedSearch.trim().toLowerCase();
if (!query) return treeDocuments;
return treeDocuments.filter((d) => d.title.toLowerCase().includes(query));
}, [treeDocuments, debouncedSearch]);
if (!query) return treeDocumentsWithMemory;
return treeDocumentsWithMemory.filter((d) => d.title.toLowerCase().includes(query));
}, [treeDocumentsWithMemory, debouncedSearch]);
const openMemoryDocument = useCallback(
(doc: DocumentNodeDoc) => {
if (doc.document_type === "USER_MEMORY") {
openEditorPanel({
kind: "memory",
memoryScope: "user",
searchSpaceId,
title: doc.title,
});
return true;
}
if (doc.document_type === "TEAM_MEMORY") {
openEditorPanel({
kind: "memory",
memoryScope: "team",
searchSpaceId,
title: doc.title,
});
return true;
}
return false;
},
[openEditorPanel, searchSpaceId]
);
const handleResetMemoryDocument = useCallback(
async (doc: DocumentNodeDoc) => {
if (!isMemoryDocument(doc)) return;
if (!window.confirm(`Reset ${doc.title.toLowerCase()}? This clears the memory document.`)) {
return;
}
const endpoint =
doc.document_type === "USER_MEMORY"
? `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/users/me/memory/reset`
: `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/searchspaces/${searchSpaceId}/memory/reset`;
try {
const response = await authenticatedFetch(endpoint, { method: "POST" });
if (!response.ok) {
const errorData = await response.json().catch(() => ({ detail: "Reset failed" }));
throw new Error(errorData.detail || "Reset failed");
}
toast.success(`${doc.title} reset`);
openMemoryDocument(doc);
} catch (error) {
toast.error((error as Error)?.message || `Failed to reset ${doc.title.toLowerCase()}`);
}
},
[openMemoryDocument, searchSpaceId]
);
const typeCounts = useMemo(() => {
const counts: Partial<Record<string, number>> = {};
@ -1169,6 +1285,7 @@ function AuthenticatedDocumentsSidebarBase({
onCreateFolder={handleCreateFolder}
searchQuery={debouncedSearch.trim() || undefined}
onPreviewDocument={(doc) => {
if (openMemoryDocument(doc)) return;
openEditorPanel({
documentId: doc.id,
searchSpaceId,
@ -1176,6 +1293,7 @@ function AuthenticatedDocumentsSidebarBase({
});
}}
onEditDocument={(doc) => {
if (openMemoryDocument(doc)) return;
openEditorPanel({
documentId: doc.id,
searchSpaceId,
@ -1184,6 +1302,7 @@ function AuthenticatedDocumentsSidebarBase({
}}
onDeleteDocument={(doc) => handleDeleteDocument(doc.id)}
onMoveDocument={handleMoveDocument}
onResetDocument={handleResetMemoryDocument}
onExportDocument={handleExportDocument}
onVersionHistory={(doc) => setVersionDocId(doc.id)}
activeTypes={activeTypes}

299
surfsense_web/components/settings/team-memory-manager.tsx
View file

@ -1,299 +0,0 @@
"use client";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { useAtomValue } from "jotai";
import { ArrowUp, ChevronDown, ClipboardCopy, Download, Info, Pencil } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { toast } from "sonner";
import { z } from "zod";
import { updateSearchSpaceMutationAtom } from "@/atoms/search-spaces/search-space-mutation.atoms";
import { PlateEditor } from "@/components/editor/plate-editor";
import { Alert, AlertDescription } from "@/components/ui/alert";
import { Button } from "@/components/ui/button";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { Spinner } from "@/components/ui/spinner";
import { baseApiService } from "@/lib/apis/base-api.service";
import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service";
import { cacheKeys } from "@/lib/query-client/cache-keys";
const MEMORY_HARD_LIMIT = 25_000;
const SearchSpaceSchema = z
.object({
shared_memory_md: z.string().optional().default(""),
})
.passthrough();
interface TeamMemoryManagerProps {
searchSpaceId: number;
}
export function TeamMemoryManager({ searchSpaceId }: TeamMemoryManagerProps) {
const queryClient = useQueryClient();
const { data: searchSpace, isLoading: loading } = useQuery({
queryKey: cacheKeys.searchSpaces.detail(searchSpaceId.toString()),
queryFn: () => searchSpacesApiService.getSearchSpace({ id: searchSpaceId }),
enabled: !!searchSpaceId,
});
const { mutateAsync: updateSearchSpace } = useAtomValue(updateSearchSpaceMutationAtom);
const [saving, setSaving] = useState(false);
const [editQuery, setEditQuery] = useState("");
const [editing, setEditing] = useState(false);
const [showInput, setShowInput] = useState(false);
const textareaRef = useRef<HTMLInputElement>(null);
const inputContainerRef = useRef<HTMLDivElement>(null);
const memory = searchSpace?.shared_memory_md || "";
useEffect(() => {
if (!showInput) return;
const handlePointerDownOutside = (event: MouseEvent | TouchEvent) => {
const target = event.target;
if (!(target instanceof Node)) return;
if (inputContainerRef.current?.contains(target)) return;
setShowInput(false);
};
document.addEventListener("mousedown", handlePointerDownOutside);
document.addEventListener("touchstart", handlePointerDownOutside, { passive: true });
return () => {
document.removeEventListener("mousedown", handlePointerDownOutside);
document.removeEventListener("touchstart", handlePointerDownOutside);
};
}, [showInput]);
const handleClear = async () => {
try {
setSaving(true);
await updateSearchSpace({
id: searchSpaceId,
data: { shared_memory_md: "" },
});
toast.success("Team memory cleared");
} catch {
toast.error("Failed to clear team memory");
} finally {
setSaving(false);
}
};
const handleEdit = async () => {
const query = editQuery.trim();
if (!query) return;
try {
setEditing(true);
await baseApiService.post(
`/api/v1/searchspaces/${searchSpaceId}/memory/edit`,
SearchSpaceSchema,
{ body: { query } }
);
setEditQuery("");
setShowInput(false);
await queryClient.invalidateQueries({
queryKey: cacheKeys.searchSpaces.detail(searchSpaceId.toString()),
});
toast.success("Team memory updated");
} catch {
toast.error("Failed to edit team memory");
} finally {
setEditing(false);
}
};
const openInput = () => {
setShowInput(true);
requestAnimationFrame(() => textareaRef.current?.focus());
};
const handleDownload = () => {
if (!memory) return;
try {
const blob = new Blob([memory], { type: "text/markdown;charset=utf-8" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = "team-memory.md";
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(url);
} catch {
toast.error("Failed to download team memory");
}
};
const handleCopyMarkdown = async () => {
if (!memory) return;
try {
await navigator.clipboard.writeText(memory);
toast.success("Copied to clipboard");
} catch {
toast.error("Failed to copy team memory");
}
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
handleEdit();
}
};
const displayMemory = memory.replace(/\(\d{4}-\d{2}-\d{2}\)\s*\[(fact|pref|instr)\]\s*/g, "");
const charCount = memory.length;
const getCounterColor = () => {
if (charCount > MEMORY_HARD_LIMIT) return "text-red-500";
if (charCount > 15_000) return "text-orange-500";
if (charCount > 10_000) return "text-yellow-500";
return "text-muted-foreground";
};
if (loading) {
return (
<div className="flex items-center justify-center py-12">
<Spinner size="md" className="text-muted-foreground" />
</div>
);
}
if (!memory) {
return (
<div className="flex flex-col items-center justify-center py-16 text-center">
<h3 className="text-base font-medium text-foreground">
What does SurfSense remember about your team?
</h3>
<p className="mt-2 max-w-sm text-sm text-muted-foreground">
Nothing yet. SurfSense picks up on team decisions and conventions as your team chats.
</p>
</div>
);
}
return (
<div className="space-y-4">
<Alert>
<Info />
<AlertDescription>
<p>
SurfSense uses this shared memory to provide team-wide context across all conversations
in this search space.
</p>
</AlertDescription>
</Alert>
<div className="relative h-[380px] rounded-lg border bg-background">
<div className="h-full overflow-y-auto scrollbar-thin">
<PlateEditor
markdown={displayMemory}
readOnly
preset="readonly"
variant="default"
editorVariant="none"
className="px-5 py-4 text-sm min-h-full"
/>
</div>
{showInput ? (
<div className="absolute bottom-3 inset-x-3 z-10">
<div
ref={inputContainerRef}
className="relative flex h-[54px] items-center gap-2 rounded-[9999px] border bg-muted/60 backdrop-blur-sm pl-4 pr-1 shadow-sm"
>
<input
ref={textareaRef}
type="text"
value={editQuery}
onChange={(e) => setEditQuery(e.target.value)}
onKeyDown={handleKeyDown}
placeholder="Tell SurfSense what to remember or forget about your team"
disabled={editing}
className="flex-1 bg-transparent text-sm outline-none placeholder:text-muted-foreground/70"
/>
<Button
type="button"
size="icon"
variant="ghost"
onClick={handleEdit}
disabled={editing || !editQuery.trim()}
className={`h-11 w-11 shrink-0 rounded-full ${
editing
? ""
: "bg-muted-foreground/15 hover:bg-accent hover:text-accent-foreground"
}`}
>
{editing ? (
<Spinner size="sm" />
) : (
<ArrowUp className="!h-5 !w-5 text-foreground" strokeWidth={2.25} />
)}
</Button>
</div>
</div>
) : (
<Button
type="button"
size="icon"
variant="secondary"
onClick={openInput}
className="absolute bottom-3 right-3 z-10 h-[54px] w-[54px] rounded-full border bg-muted/60 backdrop-blur-sm shadow-sm"
>
<Pencil className="!h-5 !w-5" />
</Button>
)}
</div>
<div className="flex items-center justify-between gap-2">
<span className={`text-xs shrink-0 ${getCounterColor()}`}>
{charCount.toLocaleString()} / {MEMORY_HARD_LIMIT.toLocaleString()}
<span className="hidden sm:inline"> characters</span>
<span className="sm:hidden"> chars</span>
{charCount > 15_000 && charCount <= MEMORY_HARD_LIMIT && " - Approaching limit"}
{charCount > MEMORY_HARD_LIMIT && " - Exceeds limit"}
</span>
<div className="flex items-center gap-1.5 sm:gap-2">
<Button
type="button"
variant="destructive"
size="sm"
className="text-xs sm:text-sm"
onClick={handleClear}
disabled={saving || editing || !memory}
>
<span className="hidden sm:inline">Reset Memory</span>
<span className="sm:hidden">Reset</span>
</Button>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button type="button" variant="secondary" size="sm" disabled={!memory}>
Export
<ChevronDown className="h-3 w-3 opacity-60" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuItem onClick={handleCopyMarkdown}>
<ClipboardCopy className="h-4 w-4 mr-2" />
Copy as Markdown
</DropdownMenuItem>
<DropdownMenuItem onClick={handleDownload}>
<Download className="h-4 w-4 mr-2" />
Download as Markdown
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
</div>
</div>
);
}

2
surfsense_web/components/ui/fixed-toolbar.tsx
View file

@ -14,7 +14,7 @@ export function FixedToolbar({
return (
<Toolbar
className={cn(
"scrollbar-hide absolute top-0 left-0 z-40 w-full justify-between overflow-x-auto border-b bg-background p-1",
"scrollbar-hide sticky top-0 z-40 w-full shrink-0 justify-between overflow-x-auto border-b bg-background p-1",
className
)}
{...props}

4
surfsense_web/contracts/enums/connectorIcons.tsx
View file

@ -1,6 +1,7 @@
import { IconUsersGroup } from "@tabler/icons-react";
import {
BookOpen,
Brain,
File,
FileText,
Globe,
@ -120,6 +121,9 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas
return <Webhook {...iconProps} />;
case "SURFSENSE_DOCS":
return <BookOpen {...iconProps} />;
case "USER_MEMORY":
case "TEAM_MEMORY":
return <Brain {...iconProps} />;
case "DEEP":
return <Sparkles {...iconProps} />;
case "DEEPER":

2
surfsense_web/contracts/types/document.types.ts
View file

@ -29,6 +29,8 @@ export const documentTypeEnum = z.enum([
"LOCAL_FOLDER_FILE",
"SURFSENSE_DOCS",
"NOTE",
"USER_MEMORY",
"TEAM_MEMORY",
"COMPOSIO_GOOGLE_DRIVE_CONNECTOR",
"COMPOSIO_GMAIL_CONNECTOR",
"COMPOSIO_GOOGLE_CALENDAR_CONNECTOR",

1
surfsense_web/contracts/types/search-space.types.ts
View file

@ -56,7 +56,6 @@ export const updateSearchSpaceRequest = z.object({
description: true,
citations_enabled: true,
qna_custom_instructions: true,
shared_memory_md: true,
ai_file_sort_enabled: true,
})
.partial(),