refactor: extract shared memory service

surfsense_backend/app/services/memory/__init__.py

@@ -0,0 +1,29 @@
+"""First-class memory service for user and team markdown memory."""
+
+from .service import (
+    MemoryScope,
+    SaveResult,
+    extract_and_save,
+    read_memory,
+    reset_memory,
+    save_memory,
+)
+from .validation import (
+    MEMORY_HARD_LIMIT,
+    MEMORY_SOFT_LIMIT,
+    validate_bullet_format,
+    validate_memory_scope,
+)
+
+__all__ = [
+    "MEMORY_HARD_LIMIT",
+    "MEMORY_SOFT_LIMIT",
+    "MemoryScope",
+    "SaveResult",
+    "extract_and_save",
+    "read_memory",
+    "reset_memory",
+    "save_memory",
+    "validate_bullet_format",
+    "validate_memory_scope",
+]

surfsense_backend/app/services/memory/prompts.py

@@ -0,0 +1,110 @@
+"""Prompts used by the memory service."""
+
+FORCED_REWRITE_PROMPT = """\
+You are a memory curator. The following memory document exceeds the character \
+limit and must be shortened.
+
+RULES:
+1. Rewrite the document to be under {target} characters.
+2. Output Markdown only. Use clear `##` headings and concise bullet points.
+3. New-format bullets should look like: `- YYYY-MM-DD: memory text`.
+4. If the input contains legacy markers like `(YYYY-MM-DD) [fact]`, preserve the
+   information but remove the inline marker in the output.
+5. Preserve durable instructions and preferences before generic facts when
+   compressing personal memory.
+6. Preserve existing headings when useful; merge duplicate headings and bullets.
+7. Output ONLY the consolidated markdown — no explanations, no wrapping.
+
+<memory_document>
+{content}
+</memory_document>"""
+
+USER_MEMORY_EXTRACT_PROMPT = """\
+You are a memory extraction assistant. Analyze the user's message and decide \
+if it contains any long-term information worth persisting to personal memory.
+
+Worth remembering: preferences, background/identity, goals, projects, \
+instructions, tools/languages they use, decisions, expertise, workplace — \
+durable facts that will matter in future conversations.
+
+NOT worth remembering: greetings, one-off factual questions, session \
+logistics, ephemeral requests, follow-up clarifications with no new personal \
+info, things that only matter for the current task.
+
+If there is nothing durable to remember, choose `action = no_update`.
+
+If the message contains memorizable information, choose `action = save` and \
+return the FULL updated memory document with the new information merged into \
+existing content.
+
+FORMAT RULES FOR `updated_memory`:
+- Markdown only.
+- Every entry should be under a `##` heading.
+- Recommended headings: `## Facts`, `## Preferences`, `## Instructions`.
+- New bullets should use: `- YYYY-MM-DD: memory text`.
+- If current memory uses legacy `(YYYY-MM-DD) [fact|pref|instr]` markers,
+  preserve the information but write the updated document in the new
+  heading-based format.
+- Use the user's first name from `<user_name>` when helpful, not "the user".
+- Do not duplicate existing information.
+
+<user_name>{user_name}</user_name>
+
+<current_memory>
+{current_memory}
+</current_memory>
+
+<user_message>
+{user_message}
+</user_message>"""
+
+TEAM_MEMORY_EXTRACT_PROMPT = """\
+You are a team-memory extraction assistant. Analyze the latest message and \
+decide if it contains durable TEAM-level information worth persisting.
+
+Decision policy:
+- Prioritize recall for durable team context, while avoiding personal-only facts.
+- Do NOT require explicit consensus language. A direct team-level statement can
+  be stored if it is stable and broadly useful for future team chats.
+- If evidence is weak or clearly tentative, choose `action = no_update`.
+
+Worth remembering (team-level only):
+- Decisions and defaults that guide future team work
+- Team conventions/standards (naming, review policy, coding norms)
+- Stable org/project facts (locations, ownership, constraints)
+- Long-lived architecture/process facts
+- Ongoing priorities that are likely relevant beyond this turn
+
+NOT worth remembering:
+- Personal preferences or biography of one person
+- Questions, brainstorming, tentative ideas, or speculation
+- One-off requests, status updates, TODOs, logistics for this session
+- Information scoped only to a single ephemeral task
+
+If the message contains memorizable team information, choose `action = save` \
+and return the FULL updated team memory document with new facts merged into \
+existing content.
+
+FORMAT RULES FOR `updated_memory`:
+- Markdown only.
+- Every entry should be under a `##` heading.
+- Recommended headings: `## Product Decisions`, `## Engineering Conventions`,
+  `## Project Facts`, `## Open Questions`.
+- New bullets should use: `- YYYY-MM-DD: memory text`.
+- If current memory uses legacy `(YYYY-MM-DD) [fact]` markers, preserve the
+  information but write the updated document in the new heading-based format.
+- Do not create personal headings such as `## Preferences`, `## Instructions`,
+  or `## Personal Notes`.
+- Preserve neutral team phrasing; avoid person-specific memory unless role-anchored.
+
+<current_team_memory>
+{current_memory}
+</current_team_memory>
+
+<latest_message_author>
+{author}
+</latest_message_author>
+
+<latest_message>
+{user_message}
+</latest_message>"""

surfsense_backend/app/services/memory/rewrite.py

@@ -0,0 +1,35 @@
+"""LLM-backed memory rewrite helpers."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from langchain_core.messages import HumanMessage
+
+from app.services.memory.prompts import FORCED_REWRITE_PROMPT
+from app.services.memory.validation import MEMORY_HARD_LIMIT
+from app.utils.content_utils import extract_text_content
+
+logger = logging.getLogger(__name__)
+
+
+async def forced_rewrite(content: str, llm: Any) -> str | None:
+    """Use a focused LLM call to compress memory under the hard limit."""
+    try:
+        prompt = FORCED_REWRITE_PROMPT.format(
+            target=MEMORY_HARD_LIMIT,
+            content=content,
+        )
+        response = await llm.ainvoke(
+            [HumanMessage(content=prompt)],
+            config={"tags": ["surfsense:internal", "memory-rewrite"]},
+        )
+        text = extract_text_content(response.content).strip()
+        if not text:
+            logger.warning("Forced memory rewrite returned empty text")
+            return None
+        return text
+    except Exception:
+        logger.exception("Forced memory rewrite LLM call failed")
+        return None

surfsense_backend/app/services/memory/schemas.py

@@ -0,0 +1,23 @@
+"""Structured output schemas for memory extraction."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class MemoryExtractionDecision(BaseModel):
+    """Structured extraction result; avoids string sentinel parsing."""
+
+    action: Literal["no_update", "save"] = Field(
+        description="Choose no_update when nothing durable should be saved; choose save otherwise."
+    )
+    reason: str | None = Field(
+        default=None,
+        description="Short reason for no_update, or brief summary of the memory update.",
+    )
+    updated_memory: str | None = Field(
+        default=None,
+        description="The full updated markdown memory document when action is save.",
+    )

surfsense_backend/app/services/memory/service.py

@@ -0,0 +1,300 @@
+"""Canonical read/write/reset/extract service for markdown memory."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any, Literal
+from uuid import UUID
+
+from langchain_core.messages import HumanMessage
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db import SearchSpace, User
+from app.services.memory.prompts import (
+    TEAM_MEMORY_EXTRACT_PROMPT,
+    USER_MEMORY_EXTRACT_PROMPT,
+)
+from app.services.memory.rewrite import forced_rewrite
+from app.services.memory.schemas import MemoryExtractionDecision
+from app.services.memory.validation import (
+    MEMORY_HARD_LIMIT,
+    soft_limit_warning,
+    strip_preamble_to_first_heading,
+    validate_bullet_format,
+    validate_diff,
+    validate_heading_sanity,
+    validate_memory_scope,
+    validate_memory_size,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class MemoryScope(StrEnum):
+    USER = "user"
+    TEAM = "team"
+
+
+@dataclass(frozen=True)
+class SaveResult:
+    status: Literal["saved", "error", "no_op"]
+    message: str
+    memory_md: str = ""
+    warnings: list[str] = field(default_factory=list)
+    diff_warnings: list[str] = field(default_factory=list)
+    format_warnings: list[str] = field(default_factory=list)
+    notice: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "status": self.status,
+            "message": self.message,
+            "memory_md": self.memory_md,
+        }
+        if self.notice:
+            data["notice"] = self.notice
+        if self.warnings:
+            data["warnings"] = self.warnings
+            if len(self.warnings) == 1:
+                data["warning"] = self.warnings[0]
+        if self.diff_warnings:
+            data["diff_warnings"] = self.diff_warnings
+        if self.format_warnings:
+            data["format_warnings"] = self.format_warnings
+        return data
+
+
+class MemoryRead(BaseModel):
+    memory_md: str
+
+
+def _normalize_scope(scope: MemoryScope | str) -> MemoryScope:
+    return scope if isinstance(scope, MemoryScope) else MemoryScope(scope)
+
+
+def _normalize_user_id(target_id: str | UUID) -> UUID:
+    return UUID(target_id) if isinstance(target_id, str) else target_id
+
+
+async def _load_target(
+    *,
+    scope: MemoryScope | str,
+    target_id: str | int | UUID,
+    session: AsyncSession,
+) -> User | SearchSpace | None:
+    normalized = _normalize_scope(scope)
+    if normalized is MemoryScope.USER:
+        result = await session.execute(
+            select(User).where(User.id == _normalize_user_id(target_id))  # type: ignore[arg-type]
+        )
+        return result.scalars().first()
+    result = await session.execute(select(SearchSpace).where(SearchSpace.id == int(target_id)))
+    return result.scalars().first()
+
+
+def _get_memory(target: User | SearchSpace, scope: MemoryScope) -> str:
+    if scope is MemoryScope.USER:
+        return getattr(target, "memory_md", None) or ""
+    return getattr(target, "shared_memory_md", None) or ""
+
+
+def _set_memory(target: User | SearchSpace, scope: MemoryScope, content: str) -> None:
+    if scope is MemoryScope.USER:
+        target.memory_md = content
+    else:
+        target.shared_memory_md = content
+
+
+async def read_memory(
+    *,
+    scope: MemoryScope | str,
+    target_id: str | int | UUID,
+    session: AsyncSession,
+) -> str:
+    normalized = _normalize_scope(scope)
+    target = await _load_target(scope=normalized, target_id=target_id, session=session)
+    if target is None:
+        return ""
+    return _get_memory(target, normalized)
+
+
+async def save_memory(
+    *,
+    scope: MemoryScope | str,
+    target_id: str | int | UUID,
+    content: str,
+    session: AsyncSession,
+    llm: Any | None = None,
+) -> SaveResult:
+    normalized = _normalize_scope(scope)
+    if not isinstance(content, str):
+        return SaveResult(
+            status="error",
+            message="Internal error: memory payload must be a string.",
+        )
+
+    target = await _load_target(scope=normalized, target_id=target_id, session=session)
+    if target is None:
+        return SaveResult(
+            status="error",
+            message="User not found." if normalized is MemoryScope.USER else "Search space not found.",
+        )
+
+    old_memory = _get_memory(target, normalized)
+    next_content = strip_preamble_to_first_heading(content.strip())
+    notice: str | None = None
+    warnings: list[str] = []
+
+    if len(next_content) > MEMORY_HARD_LIMIT and llm is not None:
+        rewritten = await forced_rewrite(next_content, llm)
+        if rewritten is not None and len(rewritten) < len(next_content):
+            next_content = strip_preamble_to_first_heading(rewritten)
+            notice = "Memory was automatically rewritten to fit within limits."
+
+    for validation in (
+        validate_memory_size(next_content),
+        validate_heading_sanity(next_content),
+    ):
+        if validation:
+            return SaveResult(
+                status="error",
+                message=validation["message"],
+                memory_md=old_memory,
+            )
+
+    scope_error, scope_warnings = validate_memory_scope(
+        next_content,
+        normalized.value,
+        old_memory=old_memory,
+    )
+    warnings.extend(scope_warnings)
+    if scope_error:
+        return SaveResult(
+            status="error",
+            message=scope_error["message"],
+            memory_md=old_memory,
+            warnings=warnings,
+        )
+
+    try:
+        _set_memory(target, normalized, next_content)
+        session.add(target)
+        await session.commit()
+    except Exception as e:
+        logger.exception("Failed to update %s memory: %s", normalized.value, e)
+        await session.rollback()
+        return SaveResult(
+            status="error",
+            message=f"Failed to update {normalized.value} memory: {e}",
+            memory_md=old_memory,
+        )
+
+    diff_warnings = validate_diff(old_memory, next_content)
+    format_warnings = validate_bullet_format(next_content)
+    warning = soft_limit_warning(next_content)
+    if warning:
+        warnings.append(warning)
+
+    return SaveResult(
+        status="saved",
+        message=(
+            "Memory updated."
+            if normalized is MemoryScope.USER
+            else "Team memory updated."
+        ),
+        memory_md=next_content,
+        warnings=warnings,
+        diff_warnings=diff_warnings,
+        format_warnings=format_warnings,
+        notice=notice,
+    )
+
+
+async def reset_memory(
+    *,
+    scope: MemoryScope | str,
+    target_id: str | int | UUID,
+    session: AsyncSession,
+) -> SaveResult:
+    return await save_memory(
+        scope=scope,
+        target_id=target_id,
+        content="",
+        session=session,
+        llm=None,
+    )
+
+
+async def extract_and_save(
+    *,
+    scope: MemoryScope | str,
+    target_id: str | int | UUID,
+    user_message: str,
+    actor_display_name: str | None,
+    session: AsyncSession,
+    llm: Any,
+) -> SaveResult:
+    normalized = _normalize_scope(scope)
+    current_memory = await read_memory(
+        scope=normalized,
+        target_id=target_id,
+        session=session,
+    )
+
+    if normalized is MemoryScope.USER:
+        first_name = (
+            actor_display_name.strip().split()[0]
+            if actor_display_name and actor_display_name.strip()
+            else "The user"
+        )
+        prompt = USER_MEMORY_EXTRACT_PROMPT.format(
+            current_memory=current_memory or "(empty)",
+            user_message=user_message,
+            user_name=first_name,
+        )
+    else:
+        prompt = TEAM_MEMORY_EXTRACT_PROMPT.format(
+            current_memory=current_memory or "(empty)",
+            author=actor_display_name or "Unknown team member",
+            user_message=user_message,
+        )
+
+    try:
+        structured = llm.with_structured_output(MemoryExtractionDecision)
+        decision = await structured.ainvoke(
+            [HumanMessage(content=prompt)],
+            config={"tags": ["surfsense:internal", "memory-extraction"]},
+        )
+    except Exception:
+        logger.exception("Structured memory extraction failed")
+        return SaveResult(
+            status="error",
+            message="Structured memory extraction failed.",
+            memory_md=current_memory,
+        )
+
+    if decision.action == "no_update":
+        return SaveResult(
+            status="no_op",
+            message=decision.reason or "No durable memory to persist.",
+            memory_md=current_memory,
+        )
+
+    if not decision.updated_memory:
+        return SaveResult(
+            status="error",
+            message="Structured memory extraction chose save without updated_memory.",
+            memory_md=current_memory,
+        )
+
+    return await save_memory(
+        scope=normalized,
+        target_id=target_id,
+        content=decision.updated_memory,
+        session=session,
+        llm=llm,
+    )

surfsense_backend/app/services/memory/validation.py

@@ -0,0 +1,158 @@
+"""Validation helpers for markdown-backed memory."""
+
+from __future__ import annotations
+
+import re
+from typing import Literal
+
+MEMORY_SOFT_LIMIT = 18_000
+MEMORY_HARD_LIMIT = 25_000
+
+_SECTION_HEADING_RE = re.compile(r"^##\s+(.+)$", re.MULTILINE)
+_HEADING_LINE_RE = re.compile(r"^##\s+\S+", re.MULTILINE)
+_HEADING_NORMALIZE_RE = re.compile(r"[^a-z0-9]+")
+_LEGACY_BULLET_RE = re.compile(r"^-\s+\(\d{4}-\d{2}-\d{2}\)\s+\[(fact|pref|instr)\]\s+.+$")
+_NEW_BULLET_RE = re.compile(r"^-\s+\d{4}-\d{2}-\d{2}:\s+.+$")
+
+_FORBIDDEN_TEAM_HEADINGS = {
+    "preferences",
+    "instructions",
+    "personal notes",
+    "personal instructions",
+}
+
+
+def has_markdown_heading(content: str) -> bool:
+    return bool(_HEADING_LINE_RE.search(content))
+
+
+def strip_preamble_to_first_heading(content: str) -> str:
+    """Drop model preamble before the first ``##`` heading, if one exists."""
+    match = _HEADING_LINE_RE.search(content)
+    if not match:
+        return content.strip()
+    return content[match.start() :].strip()
+
+
+def extract_headings(memory: str | None) -> set[str]:
+    if not memory:
+        return set()
+    return {_normalize_heading(h) for h in _SECTION_HEADING_RE.findall(memory)}
+
+
+def _normalize_heading(heading: str) -> str:
+    return _HEADING_NORMALIZE_RE.sub(" ", heading.strip().lower()).strip()
+
+
+def validate_memory_size(content: str) -> dict[str, str] | None:
+    length = len(content)
+    if length > MEMORY_HARD_LIMIT:
+        return {
+            "status": "error",
+            "message": (
+                f"Memory exceeds {MEMORY_HARD_LIMIT:,} character limit "
+                f"({length:,} chars). Consolidate by merging related items, "
+                "removing outdated entries, and shortening descriptions."
+            ),
+        }
+    return None
+
+
+def validate_heading_sanity(content: str) -> dict[str, str] | None:
+    """Block long prose blobs without headings unless they are legacy bullets."""
+    stripped = content.strip()
+    if not stripped:
+        return None
+    if has_markdown_heading(stripped):
+        return None
+    if len(stripped) <= 40:
+        return None
+    if any(_LEGACY_BULLET_RE.match(line.strip()) for line in stripped.splitlines()):
+        return None
+    return {
+        "status": "error",
+        "message": "Memory must be markdown with at least one ## heading.",
+    }
+
+
+def validate_memory_scope(
+    content: str,
+    scope: Literal["user", "team"],
+    *,
+    old_memory: str | None = None,
+) -> tuple[dict[str, str] | None, list[str]]:
+    """Reject new personal headings in team memory, grandfather existing ones."""
+    if scope != "team":
+        return None, []
+
+    old_forbidden = extract_headings(old_memory) & _FORBIDDEN_TEAM_HEADINGS
+    new_forbidden = extract_headings(content) & _FORBIDDEN_TEAM_HEADINGS
+    introduced = sorted(new_forbidden - old_forbidden)
+    grandfathered = sorted(new_forbidden & old_forbidden)
+
+    warnings: list[str] = []
+    if grandfathered:
+        warnings.append(
+            "Team memory contains legacy personal headings: "
+            + ", ".join(grandfathered)
+            + ". Please consolidate them into team-safe headings."
+        )
+    if introduced:
+        return (
+            {
+                "status": "error",
+                "message": (
+                    "Team memory cannot introduce personal headings: "
+                    + ", ".join(introduced)
+                    + ". Use team-safe headings instead."
+                ),
+            },
+            warnings,
+        )
+    return None, warnings
+
+
+def validate_bullet_format(content: str) -> list[str]:
+    warnings: list[str] = []
+    for line in content.splitlines():
+        stripped = line.strip()
+        if not stripped.startswith("- "):
+            continue
+        if _NEW_BULLET_RE.match(stripped) or _LEGACY_BULLET_RE.match(stripped):
+            continue
+        short = stripped[:80] + ("..." if len(stripped) > 80 else "")
+        warnings.append(f"Non-standard memory bullet: {short}")
+    return warnings
+
+
+def validate_diff(old_memory: str | None, new_memory: str) -> list[str]:
+    if not old_memory:
+        return []
+
+    warnings: list[str] = []
+    old_headings = extract_headings(old_memory)
+    new_headings = extract_headings(new_memory)
+    dropped = old_headings - new_headings
+    if dropped:
+        names = ", ".join(sorted(dropped))
+        warnings.append(
+            f"Sections removed: {names}. If unintentional, restore from the settings page."
+        )
+
+    old_len = len(old_memory)
+    new_len = len(new_memory)
+    if old_len > 0 and new_len < old_len * 0.4:
+        warnings.append(
+            f"Memory shrank significantly ({old_len:,} -> {new_len:,} chars). Possible data loss."
+        )
+    return warnings
+
+
+def soft_limit_warning(content: str) -> str | None:
+    length = len(content)
+    if length > MEMORY_SOFT_LIMIT:
+        return (
+            f"Memory is at {length:,}/{MEMORY_HARD_LIMIT:,} characters. "
+            "Consolidate by merging related items and removing less important entries."
+        )
+    return None