Merge pull request #1617 from CREDO23/feature-okf

[Feat] Serve the knowledge base in Open Knowledge Format (OKF)

surfsense_backend/app/routes/documents_routes.py

@@ -1,7 +1,8 @@
 # Force asyncio to use standard event loop before unstructured imports
 import asyncio
 
-from fastapi import APIRouter, Depends, Form, HTTPException, Query, UploadFile
+from fastapi import APIRouter, Depends, Form, HTTPException, Query, Request, UploadFile
+from fastapi.responses import PlainTextResponse
 from pydantic import BaseModel as PydanticBaseModel, Field
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
@@ -34,6 +35,8 @@ from app.schemas import (
     FolderRead,
     PaginatedResponse,
 )
+from app.services.export_service import resolve_document_markdown
+from app.services.okf import document_to_concept
 from app.services.task_dispatcher import TaskDispatcher, get_task_dispatcher
 from app.users import get_auth_context
 from app.utils.rbac import check_permission
@@ -1255,12 +1258,14 @@ async def get_document_chunks_paginated(
 @router.get("/documents/{document_id}", response_model=DocumentRead)
 async def read_document(
     document_id: int,
+    request: Request,
     session: AsyncSession = Depends(get_async_session),
     auth: AuthContext = Depends(get_auth_context),
 ):
-    """
-    Get a specific document by ID.
-    Requires DOCUMENTS_READ permission for the workspace.
+    """Get a document by ID.
+
+    ``Accept: text/markdown`` returns the OKF concept; otherwise the JSON record.
+    Requires DOCUMENTS_READ permission.
     """
     try:
         result = await session.execute(
@@ -1282,6 +1287,16 @@ async def read_document(
             "You don't have permission to read documents in this workspace",
         )
 
+        # ponytail: substring match, not RFC 7231 q-values (OKF is the only non-JSON view).
+        if "text/markdown" in request.headers.get("accept", ""):
+            body = (
+                await resolve_document_markdown(session, document)
+                or document.content
+                or ""
+            )
+            concept = document_to_concept(document, body=body)
+            return PlainTextResponse(concept, media_type="text/markdown")
+
         raw_content = document.content or ""
         return DocumentRead(
             id=document.id,

surfsense_backend/app/services/export_service.py

@@ -12,9 +12,26 @@ from sqlalchemy.future import select
 
 from app.db import Chunk, Document, Folder
 from app.services.folder_service import get_folder_subtree_ids
+from app.services.okf import (
+    INDEX_FILENAME,
+    LOG_FILENAME,
+    ConceptRef,
+    LogEntry,
+    SubdirRef,
+    document_to_concept,
+    folder_to_index,
+    folder_to_log,
+    okf_type,
+)
 
 logger = logging.getLogger(__name__)
 
+# Root index.md declares the targeted OKF version in frontmatter - the one place
+# the spec permits frontmatter in an index file.
+_ROOT_INDEX_FRONTMATTER = '---\nokf_version: "0.1"\n---\n\n'
+
+_RESERVED_STEMS = {"index", "log"}
+
 
 def _sanitize_filename(title: str) -> str:
     safe = "".join(c if c.isalnum() or c in " -_." else "_" for c in title).strip()
@@ -43,7 +60,7 @@ def _build_folder_path_map(folders: list[Folder]) -> dict[int, str]:
     return cache
 
 
-async def _get_document_markdown(
+async def resolve_document_markdown(
     session: AsyncSession, document: Document
 ) -> str | None:
     """Resolve markdown content using the 3-tier fallback:
@@ -71,6 +88,67 @@ async def _get_document_markdown(
     return None
 
 
+def _build_index_files(
+    dir_concepts: dict[str, list[ConceptRef]],
+) -> list[tuple[str, str]]:
+    """Build ``index.md`` files for every directory (and ancestor) with content.
+
+    Produces OKF progressive-disclosure listings: each directory lists its
+    concepts (grouped by type) and its immediate subdirectories. The bundle-root
+    index also declares ``okf_version``.
+    """
+    all_dirs: set[str] = {""}
+    for dir_path in dir_concepts:
+        all_dirs.add(dir_path)
+        parts = dir_path.split("/") if dir_path else []
+        for i in range(1, len(parts)):
+            all_dirs.add("/".join(parts[:i]))
+
+    children_by_dir: dict[str, list[str]] = {}
+    for dir_path in all_dirs:
+        if not dir_path:
+            continue
+        parent = dir_path.rsplit("/", 1)[0] if "/" in dir_path else ""
+        children_by_dir.setdefault(parent, []).append(dir_path)
+
+    index_files: list[tuple[str, str]] = []
+    for dir_path in all_dirs:
+        subdirs = [
+            SubdirRef(name=child.rsplit("/", 1)[-1])
+            for child in children_by_dir.get(dir_path, [])
+        ]
+        body = folder_to_index(
+            concepts=dir_concepts.get(dir_path, []),
+            subdirectories=subdirs,
+        )
+        if not body:
+            continue
+        if dir_path:
+            index_files.append((f"{dir_path}/{INDEX_FILENAME}", body))
+        else:
+            index_files.append((INDEX_FILENAME, _ROOT_INDEX_FRONTMATTER + body))
+
+    return index_files
+
+
+def _build_log_files(
+    dir_logs: dict[str, list[LogEntry]],
+) -> list[tuple[str, str]]:
+    """Build ``log.md`` files for every directory that holds concepts.
+
+    Unlike ``index.md``, logs are only synthesized where documents actually live
+    (no ancestor synthesis): an empty intermediate directory has nothing to log.
+    """
+    log_files: list[tuple[str, str]] = []
+    for dir_path, entries in dir_logs.items():
+        body = folder_to_log(entries)
+        if not body:
+            continue
+        path = f"{dir_path}/{LOG_FILENAME}" if dir_path else LOG_FILENAME
+        log_files.append((path, body))
+    return log_files
+
+
 @dataclass
 class ExportResult:
     zip_path: str
@@ -120,6 +198,10 @@ async def build_export_zip(
     used_paths: dict[str, int] = {}
     skipped_docs: list[str] = []
     is_first_batch = True
+    # dir path -> concepts it holds, accumulated across batches for index.md.
+    dir_concepts: dict[str, list[ConceptRef]] = {}
+    # dir path -> log entries it holds, accumulated across batches for log.md.
+    dir_logs: dict[str, list[LogEntry]] = {}
 
     try:
         offset = 0
@@ -143,7 +225,7 @@ async def build_export_zip(
                     skipped_docs.append(doc.title or "Untitled")
                     continue
 
-                markdown = await _get_document_markdown(session, doc)
+                markdown = await resolve_document_markdown(session, doc)
                 if not markdown or not markdown.strip():
                     continue
 
@@ -153,6 +235,9 @@ async def build_export_zip(
                     dir_path = ""
 
                 base_name = _sanitize_filename(doc.title or "Untitled")
+                # Never collide with reserved OKF filenames (index.md, log.md).
+                if base_name.lower() in _RESERVED_STEMS:
+                    base_name = f"{base_name}_"
                 file_path = (
                     f"{dir_path}/{base_name}.md" if dir_path else f"{base_name}.md"
                 )
@@ -160,14 +245,41 @@ async def build_export_zip(
                 if file_path in used_paths:
                     used_paths[file_path] += 1
                     suffix = used_paths[file_path]
+                    base_name = f"{base_name}_{suffix}"
                     file_path = (
-                        f"{dir_path}/{base_name}_{suffix}.md"
+                        f"{dir_path}/{base_name}.md"
                         if dir_path
-                        else f"{base_name}_{suffix}.md"
+                        else f"{base_name}.md"
                     )
                 used_paths[file_path] = used_paths.get(file_path, 0) + 1
 
-                entries.append((file_path, markdown))
+                concept = document_to_concept(doc, body=markdown)
+                entries.append((file_path, concept))
+
+                metadata = (
+                    doc.document_metadata
+                    if isinstance(doc.document_metadata, dict)
+                    else {}
+                )
+                description = metadata.get("description")
+                dir_concepts.setdefault(dir_path, []).append(
+                    ConceptRef(
+                        title=doc.title or "Untitled",
+                        filename=f"{base_name}.md",
+                        type=okf_type(doc.document_type),
+                        description=description
+                        if isinstance(description, str) and description.strip()
+                        else None,
+                    )
+                )
+
+                changed_at = doc.updated_at or doc.created_at
+                dir_logs.setdefault(dir_path, []).append(
+                    LogEntry(
+                        title=doc.title or "Untitled",
+                        timestamp=changed_at.isoformat() if changed_at else None,
+                    )
+                )
 
             if entries:
                 mode = "w" if is_first_batch else "a"
@@ -183,6 +295,30 @@ async def build_export_zip(
 
             offset += batch_size
 
+        index_files = _build_index_files(dir_concepts)
+        if index_files:
+            mode = "w" if is_first_batch else "a"
+
+            def _write_indexes(m: str = mode, e: list = index_files) -> None:
+                with zipfile.ZipFile(tmp_path, m, zipfile.ZIP_DEFLATED) as zf:
+                    for path, content in e:
+                        zf.writestr(path, content)
+
+            await asyncio.to_thread(_write_indexes)
+            is_first_batch = False
+
+        log_files = _build_log_files(dir_logs)
+        if log_files:
+            mode = "w" if is_first_batch else "a"
+
+            def _write_logs(m: str = mode, e: list = log_files) -> None:
+                with zipfile.ZipFile(tmp_path, m, zipfile.ZIP_DEFLATED) as zf:
+                    for path, content in e:
+                        zf.writestr(path, content)
+
+            await asyncio.to_thread(_write_logs)
+            is_first_batch = False
+
         export_name = "knowledge-base"
         if folder_id is not None and folder_id in folder_path_map:
             export_name = _sanitize_filename(folder_path_map[folder_id].split("/")[0])

surfsense_backend/app/services/okf/__init__.py

@@ -0,0 +1,56 @@
+"""Open Knowledge Format (OKF v0.1) serialization for the SurfSense KB.
+
+Single source of truth for turning documents and folders into OKF concepts,
+``index.md`` listings and ``log.md`` logs. Pure KB-layer functions; every
+consumer (ZIP export, REST, MCP, agents) calls in.
+
+The OKF-native model: ``Document`` rows are canonical, and frontmatter is
+*derived* from their columns on read (never stored), so rows are conformant by
+construction. Chunks and embeddings are a *derived*, rebuildable search
+projection - never a source of truth.
+
+Spec: https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md
+"""
+
+from app.services.okf.serializer import (
+    INDEX_FILENAME,
+    LOG_FILENAME,
+    ConceptRef,
+    LogEntry,
+    SubdirRef,
+    build_frontmatter,
+    document_to_concept,
+    folder_to_index,
+    folder_to_log,
+    render_frontmatter,
+)
+from app.services.okf.type_mapping import okf_resource, okf_type
+from app.services.okf.validator import (
+    RECOMMENDED_FRONTMATTER_KEYS,
+    REQUIRED_FRONTMATTER_KEYS,
+    is_conformant_concept,
+    parse_frontmatter,
+    validate_bundle,
+    validate_concept,
+)
+
+__all__ = [
+    "INDEX_FILENAME",
+    "LOG_FILENAME",
+    "ConceptRef",
+    "LogEntry",
+    "SubdirRef",
+    "build_frontmatter",
+    "document_to_concept",
+    "folder_to_index",
+    "folder_to_log",
+    "render_frontmatter",
+    "okf_resource",
+    "okf_type",
+    "RECOMMENDED_FRONTMATTER_KEYS",
+    "REQUIRED_FRONTMATTER_KEYS",
+    "is_conformant_concept",
+    "parse_frontmatter",
+    "validate_bundle",
+    "validate_concept",
+]

surfsense_backend/app/services/okf/serializer.py

@@ -0,0 +1,193 @@
+"""Serialize SurfSense knowledge into Open Knowledge Format (OKF v0.1).
+
+Pure functions with no HTTP / MCP / framework dependencies: given a
+:class:`~app.db.Document` (and, for listings, its neighbours) they return
+OKF-conformant markdown. Every consumer (ZIP export, REST, MCP, agents) calls
+these rather than re-implementing frontmatter.
+
+Spec: https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import yaml
+
+from app.db import Document
+from app.services.okf.type_mapping import okf_resource, okf_type
+
+# Reserved OKF filenames; never used for concept documents.
+INDEX_FILENAME = "index.md"
+LOG_FILENAME = "log.md"
+
+_FRONTMATTER_DELIMITER = "---"
+
+
+def _tags_from_metadata(metadata: dict[str, Any] | None) -> list[str] | None:
+    if not isinstance(metadata, dict):
+        return None
+    raw = metadata.get("tags")
+    if isinstance(raw, list):
+        tags = [str(t).strip() for t in raw if str(t).strip()]
+        return tags or None
+    return None
+
+
+def _timestamp(document: Document) -> str | None:
+    when = document.updated_at or document.created_at
+    if when is None:
+        return None
+    # ISO 8601, matching Google's sample bundles (e.g. 2026-05-28T22:49:59+00:00).
+    return when.isoformat()
+
+
+def build_frontmatter(document: Document) -> dict[str, Any]:
+    """Build the ordered OKF frontmatter mapping for a document.
+
+    Only ``type`` is required; recommended keys are included only when we have a
+    value. Insertion order is preserved in the emitted YAML.
+    """
+    metadata = document.document_metadata if isinstance(document.document_metadata, dict) else {}
+
+    frontmatter: dict[str, Any] = {"type": okf_type(document.document_type)}
+
+    resource = okf_resource(document.document_type, metadata)
+    if resource:
+        frontmatter["resource"] = resource
+
+    title = (document.title or "").strip()
+    if title:
+        frontmatter["title"] = title
+
+    description = metadata.get("description")
+    if isinstance(description, str) and description.strip():
+        frontmatter["description"] = description.strip()
+
+    tags = _tags_from_metadata(metadata)
+    if tags:
+        frontmatter["tags"] = tags
+
+    timestamp = _timestamp(document)
+    if timestamp:
+        frontmatter["timestamp"] = timestamp
+
+    return frontmatter
+
+
+def render_frontmatter(frontmatter: dict[str, Any]) -> str:
+    """Render a frontmatter mapping as a YAML block delimited by ``---``."""
+    body = yaml.safe_dump(
+        frontmatter,
+        sort_keys=False,
+        allow_unicode=True,
+        default_flow_style=False,
+    )
+    return f"{_FRONTMATTER_DELIMITER}\n{body}{_FRONTMATTER_DELIMITER}\n"
+
+
+def document_to_concept(document: Document, *, body: str) -> str:
+    """Serialize a document as an OKF concept: frontmatter + ``body``.
+
+    ``body`` is caller-resolved markdown, keeping this a pure formatting step.
+    """
+    frontmatter = render_frontmatter(build_frontmatter(document))
+    body_text = (body or "").strip("\n")
+    return f"{frontmatter}\n{body_text}\n"
+
+
+@dataclass(frozen=True)
+class ConceptRef:
+    """One concept entry for an ``index.md`` listing."""
+
+    title: str
+    filename: str  # relative to the directory, e.g. "orders.md"
+    type: str  # OKF type, used as the grouping heading
+    description: str | None = None
+
+
+@dataclass(frozen=True)
+class SubdirRef:
+    """One subdirectory entry for an ``index.md`` listing."""
+
+    name: str  # directory name, e.g. "tables"
+    description: str | None = None
+
+
+@dataclass(frozen=True)
+class LogEntry:
+    """One line of an OKF ``log.md``: a concept and when it last changed."""
+
+    title: str
+    timestamp: str | None = None  # ISO-8601, or None when unknown
+
+
+def folder_to_log(entries: list[LogEntry]) -> str:
+    """Build a minimal OKF ``log.md`` body for one directory.
+
+    Lists each concept newest-first with the time it last changed (from a
+    document's ``updated_at``/``created_at``); undated entries sort last. Returns
+    an empty string when there is nothing to log.
+
+    ponytail: this is a last-touched summary, not a per-field change history. A
+    fuller changelog would read ``DocumentVersion`` rows; upgrade there if
+    consumers ever need diffs rather than "what changed and when".
+    """
+    if not entries:
+        return ""
+    ordered = sorted(
+        entries,
+        key=lambda e: (e.timestamp is not None, e.timestamp or "", e.title),
+        reverse=True,
+    )
+    lines = ["# Change Log", ""]
+    for entry in ordered:
+        when = f" - {entry.timestamp}" if entry.timestamp else ""
+        lines.append(f"* {entry.title}{when}")
+    return "\n".join(lines) + "\n"
+
+
+def _index_bullet(title: str, link: str, description: str | None) -> str:
+    bullet = f"* [{title}]({link})"
+    if description:
+        # Keep index descriptions to a single line.
+        bullet += f" - {' '.join(description.split())}"
+    return bullet
+
+
+def folder_to_index(
+    *,
+    concepts: list[ConceptRef] | None = None,
+    subdirectories: list[SubdirRef] | None = None,
+) -> str:
+    """Build an OKF ``index.md`` body (no frontmatter) for one directory.
+
+    Subdirectories are listed under a ``# Subdirectories`` heading and concepts
+    are grouped under their ``type`` heading, mirroring Google's sample bundles.
+    Returns an empty string when there is nothing to list.
+    """
+    concepts = concepts or []
+    subdirectories = subdirectories or []
+    sections: list[str] = []
+
+    if subdirectories:
+        lines = ["# Subdirectories", ""]
+        for sub in sorted(subdirectories, key=lambda s: s.name.lower()):
+            lines.append(
+                _index_bullet(sub.name, f"{sub.name}/{INDEX_FILENAME}", sub.description)
+            )
+        sections.append("\n".join(lines))
+
+    by_type: dict[str, list[ConceptRef]] = {}
+    for concept in concepts:
+        by_type.setdefault(concept.type, []).append(concept)
+    for type_heading in sorted(by_type):
+        lines = [f"# {type_heading}", ""]
+        for concept in sorted(by_type[type_heading], key=lambda c: c.title.lower()):
+            lines.append(
+                _index_bullet(concept.title, concept.filename, concept.description)
+            )
+        sections.append("\n".join(lines))
+
+    return ("\n\n".join(sections) + "\n") if sections else ""

surfsense_backend/app/services/okf/type_mapping.py

@@ -0,0 +1,124 @@
+"""Map SurfSense document types to OKF concept ``type`` strings and ``resource`` URIs.
+
+OKF (Open Knowledge Format v0.1) requires a non-empty, human-friendly ``type`` on
+every concept and recommends a ``resource`` URI pointing at the underlying asset.
+SurfSense stores the type as a :class:`~app.db.DocumentType` enum and the source
+URL (when one exists) inside the free-form ``document_metadata`` JSON, under keys
+that differ per connector. This module is the single place that knows both mappings.
+
+Spec: https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from app.db import DocumentType
+
+# Title-cased type strings per Google's sample bundles. Types absent here fall
+# back to a derived string (see ``okf_type``), so this need not be exhaustive.
+OKF_TYPE_BY_DOCUMENT_TYPE: dict[DocumentType, str] = {
+    DocumentType.EXTENSION: "Web Page",
+    DocumentType.CRAWLED_URL: "Web Page",
+    DocumentType.FILE: "File",
+    DocumentType.NOTE: "Note",
+    DocumentType.SLACK_CONNECTOR: "Slack Message",
+    DocumentType.TEAMS_CONNECTOR: "Teams Message",
+    DocumentType.ONEDRIVE_FILE: "OneDrive File",
+    DocumentType.NOTION_CONNECTOR: "Notion Page",
+    DocumentType.YOUTUBE_VIDEO: "YouTube Video",
+    DocumentType.GITHUB_CONNECTOR: "GitHub Document",
+    DocumentType.LINEAR_CONNECTOR: "Linear Issue",
+    DocumentType.DISCORD_CONNECTOR: "Discord Message",
+    DocumentType.JIRA_CONNECTOR: "Jira Issue",
+    DocumentType.CONFLUENCE_CONNECTOR: "Confluence Page",
+    DocumentType.CLICKUP_CONNECTOR: "ClickUp Task",
+    DocumentType.GOOGLE_CALENDAR_CONNECTOR: "Google Calendar Event",
+    DocumentType.GOOGLE_GMAIL_CONNECTOR: "Gmail Message",
+    DocumentType.GOOGLE_DRIVE_FILE: "Google Drive File",
+    DocumentType.AIRTABLE_CONNECTOR: "Airtable Record",
+    DocumentType.LUMA_CONNECTOR: "Luma Event",
+    DocumentType.ELASTICSEARCH_CONNECTOR: "Elasticsearch Document",
+    DocumentType.BOOKSTACK_CONNECTOR: "BookStack Page",
+    DocumentType.CIRCLEBACK: "Circleback Meeting",
+    DocumentType.OBSIDIAN_CONNECTOR: "Obsidian Note",
+    DocumentType.DROPBOX_FILE: "Dropbox File",
+    DocumentType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR: "Google Drive File",
+    DocumentType.COMPOSIO_GMAIL_CONNECTOR: "Gmail Message",
+    DocumentType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: "Google Calendar Event",
+    DocumentType.LOCAL_FOLDER_FILE: "File",
+}
+
+
+def _coerce_document_type(document_type: DocumentType | str | None) -> DocumentType | None:
+    if document_type is None:
+        return None
+    try:
+        return DocumentType(document_type)
+    except ValueError:
+        return None
+
+
+def okf_type(document_type: DocumentType | str | None) -> str:
+    """Return the OKF ``type`` string for a SurfSense document type.
+
+    Falls back to a Title-Cased version of the enum value so the required
+    ``type`` field is never empty, even for types added after this map.
+    """
+    dt = _coerce_document_type(document_type)
+    if dt is None:
+        if document_type is None:
+            return "Document"
+        return str(document_type).replace("_", " ").title() or "Document"
+    return OKF_TYPE_BY_DOCUMENT_TYPE.get(dt) or dt.value.replace("_", " ").title()
+
+
+# Per-type metadata keys that hold the canonical source URL, checked first.
+_RESOURCE_KEYS_BY_TYPE: dict[DocumentType, tuple[str, ...]] = {
+    DocumentType.GOOGLE_DRIVE_FILE: ("webViewLink", "web_view_link"),
+    DocumentType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR: ("webViewLink", "web_view_link"),
+    DocumentType.GITHUB_CONNECTOR: ("html_url", "url"),
+    DocumentType.NOTION_CONNECTOR: ("url",),
+    DocumentType.LINEAR_CONNECTOR: ("url",),
+    DocumentType.JIRA_CONNECTOR: ("url",),
+    DocumentType.CONFLUENCE_CONNECTOR: ("url",),
+    DocumentType.SLACK_CONNECTOR: ("permalink", "url"),
+    DocumentType.DISCORD_CONNECTOR: ("url",),
+    DocumentType.CLICKUP_CONNECTOR: ("url",),
+    DocumentType.YOUTUBE_VIDEO: ("url", "video_url"),
+    DocumentType.LUMA_CONNECTOR: ("url",),
+    DocumentType.BOOKSTACK_CONNECTOR: ("url",),
+}
+
+# Generic URL-bearing keys checked for any type as a fallback.
+_GENERIC_RESOURCE_KEYS: tuple[str, ...] = (
+    "url",
+    "URL",
+    "source_url",
+    "sourceUrl",
+    "source",
+    "link",
+    "permalink",
+    "web_url",
+    "webViewLink",
+    "html_url",
+)
+
+
+def okf_resource(
+    document_type: DocumentType | str | None, metadata: dict[str, Any] | None
+) -> str | None:
+    """Return the canonical source URI for a document, or ``None`` if it has none."""
+    if not isinstance(metadata, dict) or not metadata:
+        return None
+
+    dt = _coerce_document_type(document_type)
+    type_keys = _RESOURCE_KEYS_BY_TYPE.get(dt, ()) if dt is not None else ()
+
+    for key in (*type_keys, *_GENERIC_RESOURCE_KEYS):
+        value = metadata.get(key)
+        if isinstance(value, str):
+            candidate = value.strip()
+            if candidate.startswith(("http://", "https://")):
+                return candidate
+    return None

surfsense_backend/app/services/okf/validator.py

@@ -0,0 +1,90 @@
+"""Minimal OKF v0.1 conformance checks.
+
+A bundle conforms if every non-reserved ``.md`` file has a parseable YAML
+frontmatter block whose ``type`` is a non-empty string. Reserved files
+(``index.md``, ``log.md``) are exempt from the frontmatter requirement.
+Consumers must stay permissive, so this is for producers to self-check what they
+emit, not to reject foreign bundles.
+
+Spec: https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md
+"""
+
+from __future__ import annotations
+
+import yaml
+
+from app.services.okf.serializer import INDEX_FILENAME, LOG_FILENAME
+
+_DELIMITER = "---"
+RESERVED_FILENAMES = frozenset({INDEX_FILENAME, LOG_FILENAME})
+
+# The OKF v0.1 frontmatter contract, shared by serializer and validator. Only
+# ``type`` is mandatory; the rest are recommended.
+REQUIRED_FRONTMATTER_KEYS: tuple[str, ...] = ("type",)
+RECOMMENDED_FRONTMATTER_KEYS: tuple[str, ...] = (
+    "title",
+    "description",
+    "resource",
+    "tags",
+    "timestamp",
+)
+
+
+def parse_frontmatter(text: str) -> tuple[dict | None, str | None]:
+    """Split an OKF concept into ``(frontmatter_dict, error)``.
+
+    Returns ``(dict, None)`` on success or ``(None, reason)`` if the frontmatter
+    block is missing or not a parseable YAML mapping.
+    """
+    if not text.startswith(_DELIMITER + "\n"):
+        return None, "missing opening '---' frontmatter delimiter"
+    rest = text[len(_DELIMITER) + 1 :]
+    end = rest.find("\n" + _DELIMITER)
+    if end == -1:
+        return None, "missing closing '---' frontmatter delimiter"
+    block = rest[:end]
+    try:
+        data = yaml.safe_load(block)
+    except yaml.YAMLError as exc:
+        return None, f"frontmatter is not valid YAML: {exc}"
+    if not isinstance(data, dict):
+        return None, "frontmatter is not a YAML mapping"
+    return data, None
+
+
+def validate_concept(text: str) -> list[str]:
+    """Return conformance errors for a single concept document (empty == conforms)."""
+    frontmatter, error = parse_frontmatter(text)
+    if error:
+        return [error]
+    errors: list[str] = []
+    for key in REQUIRED_FRONTMATTER_KEYS:
+        value = frontmatter.get(key)
+        if not isinstance(value, str) or not value.strip():
+            errors.append(f"frontmatter '{key}' is missing or empty")
+    return errors
+
+
+def is_conformant_concept(text: str) -> bool:
+    """True if a single concept document conforms to OKF v0.1."""
+    return not validate_concept(text)
+
+
+def validate_bundle(files: dict[str, str]) -> dict[str, list[str]]:
+    """Validate a bundle given a mapping of relative path -> file contents.
+
+    Only non-reserved ``.md`` files are checked for concept conformance. Returns
+    a mapping of path -> errors for every non-conformant file (an empty mapping
+    means the whole bundle conforms).
+    """
+    problems: dict[str, list[str]] = {}
+    for path, contents in files.items():
+        if not path.endswith(".md"):
+            continue
+        filename = path.rsplit("/", 1)[-1]
+        if filename in RESERVED_FILENAMES:
+            continue
+        errors = validate_concept(contents)
+        if errors:
+            problems[path] = errors
+    return problems

surfsense_backend/tests/integration/document_upload/test_okf_read.py

@@ -0,0 +1,66 @@
+"""GET /documents/{id} content-negotiates its representation (real HTTP, DB).
+
+Proves the live wiring the unit tests can't: ``Accept: text/markdown`` returns a
+conformant OKF concept, and the default request still returns the JSON record -
+both off the same endpoint, through the real FastAPI stack.
+"""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from app.services.okf import is_conformant_concept
+from tests.utils.helpers import poll_document_status, upload_file
+
+pytestmark = pytest.mark.integration
+
+
+async def _ready_doc_id(
+    client: httpx.AsyncClient,
+    headers: dict[str, str],
+    workspace_id: int,
+    cleanup_doc_ids: list[int],
+) -> int:
+    resp = await upload_file(client, headers, "sample.txt", workspace_id=workspace_id)
+    assert resp.status_code == 200
+    doc_ids = resp.json()["document_ids"]
+    cleanup_doc_ids.extend(doc_ids)
+    statuses = await poll_document_status(
+        client, headers, doc_ids, workspace_id=workspace_id
+    )
+    assert statuses[doc_ids[0]]["status"]["state"] == "ready"
+    return doc_ids[0]
+
+
+async def test_accept_markdown_returns_conformant_okf_concept(
+    client: httpx.AsyncClient,
+    headers: dict[str, str],
+    workspace_id: int,
+    cleanup_doc_ids: list[int],
+):
+    doc_id = await _ready_doc_id(client, headers, workspace_id, cleanup_doc_ids)
+
+    resp = await client.get(
+        f"/api/v1/documents/{doc_id}",
+        headers={**headers, "Accept": "text/markdown"},
+    )
+
+    assert resp.status_code == 200
+    assert resp.headers["content-type"].startswith("text/markdown")
+    assert is_conformant_concept(resp.text)
+
+
+async def test_default_accept_returns_json_record(
+    client: httpx.AsyncClient,
+    headers: dict[str, str],
+    workspace_id: int,
+    cleanup_doc_ids: list[int],
+):
+    doc_id = await _ready_doc_id(client, headers, workspace_id, cleanup_doc_ids)
+
+    resp = await client.get(f"/api/v1/documents/{doc_id}", headers=headers)
+
+    assert resp.status_code == 200
+    assert resp.headers["content-type"].startswith("application/json")
+    assert resp.json()["id"] == doc_id

surfsense_backend/tests/integration/test_okf_export_bundle.py

@@ -0,0 +1,77 @@
+"""A real export must be a conformant OKF bundle end-to-end (real DB).
+
+Unit tests cover the pure serializer; this drives the whole export pipeline
+(folder-path map, batching, ZIP writing) and asserts the emitted artifact -
+concept files plus reserved ``index.md``/``log.md`` - passes ``validate_bundle``.
+"""
+
+import os
+import zipfile
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db import Document, DocumentType, Folder, User, Workspace
+from app.services.export_service import build_export_zip
+from app.services.okf import validate_bundle
+
+pytestmark = pytest.mark.integration
+
+
+async def _add_doc(
+    session: AsyncSession,
+    *,
+    workspace: Workspace,
+    user: User,
+    title: str,
+    folder_id: int | None,
+    uid: str,
+) -> Document:
+    doc = Document(
+        title=title,
+        document_type=DocumentType.NOTE,
+        document_metadata={"tags": ["team"]},
+        content="body text",
+        content_hash=uid,
+        unique_identifier_hash=uid,
+        source_markdown=f"# {title}\n\nBody.",
+        workspace_id=workspace.id,
+        created_by_id=user.id,
+        folder_id=folder_id,
+    )
+    session.add(doc)
+    await session.flush()
+    return doc
+
+
+async def test_export_bundle_is_okf_conformant(
+    db_session: AsyncSession, db_user: User, db_workspace: Workspace
+):
+    folder = Folder(name="Research", position="0", workspace_id=db_workspace.id)
+    db_session.add(folder)
+    await db_session.flush()
+
+    await _add_doc(
+        db_session, workspace=db_workspace, user=db_user,
+        title="Root Note", folder_id=None, uid="okf-export-root",
+    )
+    await _add_doc(
+        db_session, workspace=db_workspace, user=db_user,
+        title="Nested Note", folder_id=folder.id, uid="okf-export-nested",
+    )
+
+    result = await build_export_zip(db_session, db_workspace.id)
+    try:
+        with zipfile.ZipFile(result.zip_path) as zf:
+            files = {name: zf.read(name).decode("utf-8") for name in zf.namelist()}
+    finally:
+        os.unlink(result.zip_path)
+
+    # Directory structure: concepts nested by folder, plus reserved files.
+    assert "Root Note.md" in files
+    assert "Research/Nested Note.md" in files
+    assert files["index.md"].startswith('---\nokf_version: "0.1"\n---')
+    assert any(name.endswith("log.md") for name in files)
+
+    # The whole bundle conforms; reserved index/log files are exempt.
+    assert validate_bundle(files) == {}

surfsense_backend/tests/integration/test_okf_path_identity.py

@@ -0,0 +1,116 @@
+"""A document's virtual path must resolve back to the same row: concepts are
+identified by path (``doc_to_virtual_path`` <-> ``virtual_path_to_doc``). Covers
+the hard cases - folder nesting and colliding titles.
+"""
+
+import pytest
+import pytest_asyncio
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.agents.chat.runtime.path_resolver import (
+    build_path_index,
+    doc_to_virtual_path,
+    virtual_path_to_doc,
+)
+from app.db import Document, DocumentType, Folder, User, Workspace
+
+pytestmark = pytest.mark.integration
+
+
+async def _add_document(
+    session: AsyncSession,
+    *,
+    workspace: Workspace,
+    user: User,
+    title: str,
+    folder_id: int | None,
+    unique_hash: str,
+) -> Document:
+    doc = Document(
+        title=title,
+        document_type=DocumentType.NOTE,
+        document_metadata={},
+        content="body",
+        content_hash=unique_hash,
+        unique_identifier_hash=unique_hash,
+        source_markdown="body",
+        workspace_id=workspace.id,
+        created_by_id=user.id,
+        folder_id=folder_id,
+    )
+    session.add(doc)
+    await session.flush()
+    return doc
+
+
+async def _roundtrip(
+    session: AsyncSession, workspace: Workspace, doc: Document
+) -> Document | None:
+    index = await build_path_index(session, workspace.id)
+    path = doc_to_virtual_path(
+        doc_id=doc.id, title=doc.title, folder_id=doc.folder_id, index=index
+    )
+    return await virtual_path_to_doc(
+        session, workspace_id=workspace.id, virtual_path=path
+    )
+
+
+@pytest_asyncio.fixture
+async def research_folder(
+    db_session: AsyncSession, db_workspace: Workspace
+) -> Folder:
+    folder = Folder(name="Research", position="0", workspace_id=db_workspace.id)
+    db_session.add(folder)
+    await db_session.flush()
+    return folder
+
+
+async def test_folder_nested_document_roundtrips(
+    db_session, db_user, db_workspace, research_folder
+):
+    doc = await _add_document(
+        db_session,
+        workspace=db_workspace,
+        user=db_user,
+        title="My Note",
+        folder_id=research_folder.id,
+        unique_hash="hash-nested",
+    )
+    assert await _roundtrip(db_session, db_workspace, doc) is doc
+
+
+async def test_colliding_titles_get_distinct_resolvable_paths(
+    db_session, db_user, db_workspace
+):
+    first = await _add_document(
+        db_session,
+        workspace=db_workspace,
+        user=db_user,
+        title="Hello",
+        folder_id=None,
+        unique_hash="hash-a",
+    )
+    second = await _add_document(
+        db_session,
+        workspace=db_workspace,
+        user=db_user,
+        title="Hello",
+        folder_id=None,
+        unique_hash="hash-b",
+    )
+
+    index = await build_path_index(db_session, db_workspace.id)
+    first_path = doc_to_virtual_path(
+        doc_id=first.id, title=first.title, folder_id=None, index=index
+    )
+    second_path = doc_to_virtual_path(
+        doc_id=second.id, title=second.title, folder_id=None, index=index
+    )
+    # Distinct identities: the collision is broken by a " (<id>).xml" suffix.
+    assert first_path != second_path
+
+    # Only the disambiguated path is stable; the bare title stays ambiguous while a twin exists.
+    resolved = await virtual_path_to_doc(
+        db_session, workspace_id=db_workspace.id, virtual_path=second_path
+    )
+    assert resolved is second

surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py

@@ -79,6 +79,33 @@ class TestDocToVirtualPath:
         assert path == f"{DOCUMENTS_ROOT}/notes/A.xml"
 
 
+class TestConceptIdentityRoundTrip:
+    """A document's virtual path must parse back to the same (folder, title) - the
+    OKF path-identity guarantee at the grammar level, no DB required."""
+
+    def test_folder_nested_document_roundtrips(self):
+        index = PathIndex(folder_paths={5: f"{DOCUMENTS_ROOT}/Research/AI"})
+        path = doc_to_virtual_path(
+            doc_id=2, title="My Note", folder_id=5, index=index
+        )
+        assert path == f"{DOCUMENTS_ROOT}/Research/AI/My Note.xml"
+        folder_parts, title = parse_documents_path(path)
+        assert folder_parts == ["Research", "AI"]
+        assert title == "My Note"
+
+    def test_colliding_title_roundtrips_to_base_title(self):
+        # Second doc with the same title gets a " (<id>)" suffix; parsing the
+        # path must strip the disambiguator and recover the original title.
+        index = PathIndex(occupants={f"{DOCUMENTS_ROOT}/Hello.xml": 7})
+        path = doc_to_virtual_path(
+            doc_id=8, title="Hello", folder_id=None, index=index
+        )
+        assert path == f"{DOCUMENTS_ROOT}/Hello (8).xml"
+        folder_parts, title = parse_documents_path(path)
+        assert folder_parts == []
+        assert title == "Hello"
+
+
 class TestParseDocumentsPath:
     def test_extracts_folder_parts_and_title(self):
         parts, title = parse_documents_path(f"{DOCUMENTS_ROOT}/foo/bar/baz.xml")

surfsense_backend/tests/unit/services/okf/test_audit.py

@@ -0,0 +1,44 @@
+"""A real export bundle - concepts plus reserved ``index.md``/``log.md`` - must
+pass the same ``validate_bundle`` a consumer would run.
+"""
+
+from datetime import UTC, datetime
+
+from app.db import Document, DocumentType
+from app.services.okf import (
+    LogEntry,
+    document_to_concept,
+    folder_to_index,
+    folder_to_log,
+    validate_bundle,
+)
+
+
+def _sample_bundle() -> dict[str, str]:
+    note = Document(
+        title="Weekly Sync",
+        document_type=DocumentType.NOTE,
+        document_metadata={"tags": ["team"]},
+        updated_at=datetime(2026, 5, 28, tzinfo=UTC),
+    )
+    page = Document(title="Docs Home", document_type=DocumentType.CRAWLED_URL)
+    return {
+        "weekly-sync.md": document_to_concept(note, body="# Agenda"),
+        "docs-home.md": document_to_concept(page, body="content"),
+        # Reserved files carry no frontmatter and must be exempt from the check.
+        "index.md": folder_to_index(),
+        "log.md": folder_to_log(
+            [LogEntry(title="Weekly Sync", timestamp="2026-05-28T00:00:00+00:00")]
+        ),
+    }
+
+
+def test_real_export_bundle_is_conformant() -> None:
+    assert validate_bundle(_sample_bundle()) == {}
+
+
+def test_audit_flags_a_drifted_concept() -> None:
+    bundle = _sample_bundle()
+    bundle["broken.md"] = "no frontmatter at all"
+    problems = validate_bundle(bundle)
+    assert list(problems) == ["broken.md"]

surfsense_backend/tests/unit/services/okf/test_conformance.py

@@ -0,0 +1,51 @@
+"""Every ``DocumentType`` must serialize to a concept a permissive OKF consumer
+can read: parseable frontmatter with a non-empty ``type``. Covers all types plus
+missing metadata, empty body, and non-ASCII titles.
+"""
+
+from datetime import UTC, datetime
+
+import pytest
+
+from app.db import Document, DocumentType
+from app.services.okf import document_to_concept, is_conformant_concept
+from app.services.okf.validator import (
+    RECOMMENDED_FRONTMATTER_KEYS,
+    REQUIRED_FRONTMATTER_KEYS,
+)
+
+
+@pytest.mark.parametrize("document_type", list(DocumentType))
+def test_every_document_type_serializes_to_conformant_concept(
+    document_type: DocumentType,
+) -> None:
+    doc = Document(
+        title="Sample",
+        document_type=document_type,
+        document_metadata={"url": "https://example.com/x"},
+        updated_at=datetime(2026, 5, 28, tzinfo=UTC),
+    )
+    assert is_conformant_concept(document_to_concept(doc, body="body"))
+
+
+def test_conformant_without_metadata_or_body() -> None:
+    doc = Document(title="Bare", document_type=DocumentType.NOTE)
+    assert is_conformant_concept(document_to_concept(doc, body=""))
+
+
+def test_conformant_with_non_ascii_title() -> None:
+    doc = Document(title="日本語ノート", document_type=DocumentType.NOTE)
+    concept = document_to_concept(doc, body="本文")
+    assert is_conformant_concept(concept)
+    assert "日本語ノート" in concept
+
+
+def test_contract_marks_only_type_as_required() -> None:
+    assert REQUIRED_FRONTMATTER_KEYS == ("type",)
+    assert set(RECOMMENDED_FRONTMATTER_KEYS) == {
+        "title",
+        "description",
+        "resource",
+        "tags",
+        "timestamp",
+    }

surfsense_backend/tests/unit/services/okf/test_ingestion.py

@@ -0,0 +1,31 @@
+"""A ConnectorDocument (the indexing write door) must serialize to a valid OKF
+concept. Per-``DocumentType`` conformance lives in ``test_conformance.py``.
+"""
+
+from app.db import Document, DocumentType
+from app.indexing_pipeline.connector_document import ConnectorDocument
+from app.services.okf import document_to_concept, is_conformant_concept
+
+
+def _document_from(connector_doc: ConnectorDocument) -> Document:
+    """Mirror how prepare_for_indexing builds a Document from a ConnectorDocument."""
+    return Document(
+        title=connector_doc.title,
+        document_type=connector_doc.document_type,
+        source_markdown=connector_doc.source_markdown,
+        document_metadata=connector_doc.metadata,
+    )
+
+
+def test_minimal_connector_document_yields_conformant_concept() -> None:
+    connector_doc = ConnectorDocument(
+        title="Bare",
+        source_markdown="just a body",
+        unique_id="u1",
+        document_type=DocumentType.FILE,
+        workspace_id=1,
+        created_by_id="user-1",
+    )
+    doc = _document_from(connector_doc)
+    concept = document_to_concept(doc, body=doc.source_markdown)
+    assert is_conformant_concept(concept)

surfsense_backend/tests/unit/services/okf/test_serializer.py

@@ -0,0 +1,130 @@
+"""OKF serializer/validator self-checks: emitted concepts stay conformant and the
+frontmatter fields consumers rely on (type/title/timestamp) round-trip.
+"""
+
+from datetime import UTC, datetime
+
+from app.db import Document, DocumentType
+from app.services.okf import (
+    ConceptRef,
+    LogEntry,
+    SubdirRef,
+    document_to_concept,
+    folder_to_index,
+    folder_to_log,
+    is_conformant_concept,
+    parse_frontmatter,
+    validate_concept,
+)
+
+
+def _make_document() -> Document:
+    return Document(
+        title="Weekly Sync Notes",
+        document_type=DocumentType.NOTE,
+        document_metadata={"tags": ["team", "meeting"], "url": "https://example.com/n"},
+        updated_at=datetime(2026, 5, 28, 22, 49, 59, tzinfo=UTC),
+    )
+
+
+def test_concept_is_conformant_and_roundtrips() -> None:
+    concept = document_to_concept(_make_document(), body="# Agenda\n\nShip OKF.")
+
+    assert is_conformant_concept(concept)
+
+    frontmatter, error = parse_frontmatter(concept)
+    assert error is None
+    assert frontmatter["type"] == "Note"
+    assert frontmatter["title"] == "Weekly Sync Notes"
+    assert frontmatter["tags"] == ["team", "meeting"]
+    assert frontmatter["resource"] == "https://example.com/n"
+    # timestamp must survive as an ISO-8601 string, not a parsed datetime.
+    assert frontmatter["timestamp"] == "2026-05-28T22:49:59+00:00"
+    assert "# Agenda" in concept
+
+
+def test_type_is_always_present_even_without_metadata() -> None:
+    doc = Document(title="Raw", document_type=DocumentType.CRAWLED_URL)
+    concept = document_to_concept(doc, body="body")
+    frontmatter, error = parse_frontmatter(concept)
+    assert error is None
+    assert frontmatter["type"] == "Web Page"
+    # No source URL / tags available -> those recommended keys are omitted.
+    assert "resource" not in frontmatter
+    assert "tags" not in frontmatter
+
+
+def test_validator_rejects_non_conformant_documents() -> None:
+    assert validate_concept("no frontmatter here")
+    assert validate_concept("---\ntitle: Missing type\n---\nbody")
+
+
+def test_folder_index_groups_by_type_and_lists_subdirs() -> None:
+    index = folder_to_index(
+        concepts=[
+            ConceptRef(title="Orders", filename="orders.md", type="Note", description="x"),
+        ],
+        subdirectories=[SubdirRef(name="tables", description="Table docs")],
+    )
+    assert "# Subdirectories" in index
+    assert "* [tables](tables/index.md) - Table docs" in index
+    assert "# Note" in index
+    assert "* [Orders](orders.md) - x" in index
+
+
+def test_folder_log_lists_concepts_newest_first() -> None:
+    log = folder_to_log(
+        [
+            LogEntry(title="Older", timestamp="2026-01-01T00:00:00+00:00"),
+            LogEntry(title="Newer", timestamp="2026-06-01T00:00:00+00:00"),
+            LogEntry(title="Undated", timestamp=None),
+        ]
+    )
+    assert "# Change Log" in log
+    # Newest dated entry precedes the older one; undated sorts last.
+    assert log.index("Newer") < log.index("Older") < log.index("Undated")
+    assert "* Newer - 2026-06-01T00:00:00+00:00" in log
+
+
+def test_folder_log_is_empty_when_no_entries() -> None:
+    assert folder_to_log([]) == ""
+
+
+def test_export_log_files_synthesized_only_where_docs_live() -> None:
+    from app.services.export_service import _build_log_files
+
+    files = dict(
+        _build_log_files(
+            {
+                "": [LogEntry(title="Root Doc", timestamp="2026-05-01T00:00:00+00:00")],
+                "Research/AI": [LogEntry(title="Nested", timestamp=None)],
+            }
+        )
+    )
+    assert "# Change Log" in files["log.md"]
+    assert "Root Doc" in files["log.md"]
+    assert "Nested" in files["Research/AI/log.md"]
+    # No empty intermediate log: "Research" holds no concepts of its own.
+    assert "Research/log.md" not in files
+
+
+def test_export_index_files_include_root_version_and_ancestors() -> None:
+    from app.services.export_service import _build_index_files
+
+    # A concept nested two levels deep, with no direct docs in the middle dir.
+    files = dict(
+        _build_index_files(
+            {
+                "Research/AI": [
+                    ConceptRef(title="Note", filename="note.md", type="Note")
+                ]
+            }
+        )
+    )
+
+    # Root, the empty intermediate dir, and the leaf all get an index.md so the
+    # hierarchy is fully navigable.
+    assert files["index.md"].startswith('---\nokf_version: "0.1"\n---')
+    assert "* [Research](Research/index.md)" in files["index.md"]
+    assert "* [AI](AI/index.md)" in files["Research/index.md"]
+    assert "* [Note](note.md)" in files["Research/AI/index.md"]

surfsense_mcp/mcp_server/core/client.py

@@ -61,13 +61,17 @@ class SurfSenseClient:
         json: Any | None = None,
         data: dict[str, Any] | None = None,
         files: Any | None = None,
+        headers: dict[str, str] | None = None,
     ) -> Any:
-        """Send a request and return the parsed body, or raise ``ToolError``."""
+        """Send a request and return the parsed body, or raise ``ToolError``.
+
+        ``headers`` overrides the client defaults for this call.
+        """
         # Omit unset query params: sending them empty makes the API parse ""
         # as a value (e.g. int("") on folder_id) and fail.
         if params is not None:
             params = {key: value for key, value in params.items() if value is not None}
-        headers = self._auth_headers()
+        headers = {**self._auth_headers(), **(headers or {})}
         try:
             response = await self._http.request(
                 method,

surfsense_mcp/mcp_server/features/knowledge_base/search_tools.py

@@ -123,11 +123,19 @@ def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -
         Use this after surfsense_search_knowledge_base or
         surfsense_list_documents to open a specific document — search results
         only include the matching passages, this returns the whole text.
+        The markdown form is an Open Knowledge Format (OKF) concept: a YAML
+        frontmatter block (type, title, tags, resource, timestamp) followed by
+        the document body.
         """
-        document = await client.request("GET", f"/documents/{document_id}")
         if response_format == "json":
+            document = await client.request("GET", f"/documents/{document_id}")
             return clip(to_json(document))
-        return _render_document(document)
+        concept = await client.request(
+            "GET",
+            f"/documents/{document_id}",
+            headers={"Accept": "text/markdown"},
+        )
+        return clip(concept if isinstance(concept, str) else str(concept))
 
 
 def _join(values: list[str] | None) -> str | None:
@@ -169,14 +177,3 @@ def _render_document_list(result: dict | None) -> str:
         + (" · more available_" if has_more else "_")
     )
     return "\n".join(lines)
-
-
-def _render_document(document: dict) -> str:
-    content = clip(document.get("content", "") or "(empty)")
-    return (
-        f"# {document.get('title', 'Untitled')} (id {document.get('id')})\n"
-        f"- type: {document.get('document_type')}\n"
-        f"- workspace: {document.get('workspace_id')}\n"
-        f"- updated: {document.get('updated_at')}\n\n"
-        f"{content}"
-    )

surfsense_mcp/tests/test_get_document_okf.py

@@ -0,0 +1,68 @@
+"""surfsense_get_document round-trips through the real tool registration.
+
+The markdown form must ask the backend for the OKF concept via content
+negotiation (``Accept: text/markdown``) and pass it through untouched; the JSON
+form must leave the default ``application/json`` Accept in place. This is the
+only coverage of the MCP-side glue that forwards the header.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from unittest.mock import MagicMock
+
+import httpx
+from mcp.server.fastmcp import FastMCP
+
+from mcp_server.core.client import SurfSenseClient
+from mcp_server.features.knowledge_base import search_tools
+
+_CONCEPT = "---\ntype: Note\ntitle: T\n---\n\nBody."
+
+
+def _client_recording(seen: dict) -> SurfSenseClient:
+    async def handler(request: httpx.Request) -> httpx.Response:
+        seen["path"] = request.url.path
+        seen["accept"] = request.headers.get("accept")
+        if "text/markdown" in (seen["accept"] or ""):
+            return httpx.Response(
+                200, text=_CONCEPT, headers={"content-type": "text/markdown"}
+            )
+        return httpx.Response(200, json={"id": 1, "title": "T"})
+
+    client = SurfSenseClient(
+        api_base="http://test/api/v1", timeout=5, fallback_api_key="ss_pat_x"
+    )
+    client._http = httpx.AsyncClient(
+        base_url="http://test/api/v1",
+        headers={"Accept": "application/json"},
+        transport=httpx.MockTransport(handler),
+    )
+    return client
+
+
+def _call_get_document(client: SurfSenseClient, **arguments) -> str:
+    mcp = FastMCP("test")
+    search_tools.register(mcp, client, MagicMock())
+    blocks = asyncio.run(mcp.call_tool("surfsense_get_document", arguments))
+    return "".join(block.text for block in blocks)
+
+
+def test_markdown_requests_okf_concept_and_passes_it_through():
+    seen: dict = {}
+    text = _call_get_document(_client_recording(seen), document_id=1)
+
+    assert seen["path"] == "/api/v1/documents/1"
+    assert "text/markdown" in seen["accept"]
+    assert text == _CONCEPT
+
+
+def test_json_keeps_default_accept():
+    seen: dict = {}
+    text = _call_get_document(
+        _client_recording(seen), document_id=1, response_format="json"
+    )
+
+    assert seen["path"] == "/api/v1/documents/1"
+    assert seen["accept"] == "application/json"
+    assert '"id": 1' in text