From 444839b0ec3c02f367dfc4b9e14d932501230cd9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:58:52 +0200 Subject: [PATCH 01/17] add OKF type mapping --- .../app/services/okf/type_mapping.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 surfsense_backend/app/services/okf/type_mapping.py diff --git a/surfsense_backend/app/services/okf/type_mapping.py b/surfsense_backend/app/services/okf/type_mapping.py new file mode 100644 index 000000000..2f1155369 --- /dev/null +++ b/surfsense_backend/app/services/okf/type_mapping.py @@ -0,0 +1,124 @@ +"""Map SurfSense document types to OKF concept ``type`` strings and ``resource`` URIs. + +OKF (Open Knowledge Format v0.1) requires a non-empty, human-friendly ``type`` on +every concept and recommends a ``resource`` URI pointing at the underlying asset. +SurfSense stores the type as a :class:`~app.db.DocumentType` enum and the source +URL (when one exists) inside the free-form ``document_metadata`` JSON, under keys +that differ per connector. This module is the single place that knows both mappings. + +Spec: https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md +""" + +from __future__ import annotations + +from typing import Any + +from app.db import DocumentType + +# Title-cased type strings per Google's sample bundles. Types absent here fall +# back to a derived string (see ``okf_type``), so this need not be exhaustive. +OKF_TYPE_BY_DOCUMENT_TYPE: dict[DocumentType, str] = { + DocumentType.EXTENSION: "Web Page", + DocumentType.CRAWLED_URL: "Web Page", + DocumentType.FILE: "File", + DocumentType.NOTE: "Note", + DocumentType.SLACK_CONNECTOR: "Slack Message", + DocumentType.TEAMS_CONNECTOR: "Teams Message", + DocumentType.ONEDRIVE_FILE: "OneDrive File", + DocumentType.NOTION_CONNECTOR: "Notion Page", + DocumentType.YOUTUBE_VIDEO: "YouTube Video", + DocumentType.GITHUB_CONNECTOR: "GitHub Document", + DocumentType.LINEAR_CONNECTOR: "Linear Issue", + DocumentType.DISCORD_CONNECTOR: "Discord Message", + DocumentType.JIRA_CONNECTOR: "Jira Issue", + DocumentType.CONFLUENCE_CONNECTOR: "Confluence Page", + DocumentType.CLICKUP_CONNECTOR: "ClickUp Task", + DocumentType.GOOGLE_CALENDAR_CONNECTOR: "Google Calendar Event", + DocumentType.GOOGLE_GMAIL_CONNECTOR: "Gmail Message", + DocumentType.GOOGLE_DRIVE_FILE: "Google Drive File", + DocumentType.AIRTABLE_CONNECTOR: "Airtable Record", + DocumentType.LUMA_CONNECTOR: "Luma Event", + DocumentType.ELASTICSEARCH_CONNECTOR: "Elasticsearch Document", + DocumentType.BOOKSTACK_CONNECTOR: "BookStack Page", + DocumentType.CIRCLEBACK: "Circleback Meeting", + DocumentType.OBSIDIAN_CONNECTOR: "Obsidian Note", + DocumentType.DROPBOX_FILE: "Dropbox File", + DocumentType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR: "Google Drive File", + DocumentType.COMPOSIO_GMAIL_CONNECTOR: "Gmail Message", + DocumentType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR: "Google Calendar Event", + DocumentType.LOCAL_FOLDER_FILE: "File", +} + + +def _coerce_document_type(document_type: DocumentType | str | None) -> DocumentType | None: + if document_type is None: + return None + try: + return DocumentType(document_type) + except ValueError: + return None + + +def okf_type(document_type: DocumentType | str | None) -> str: + """Return the OKF ``type`` string for a SurfSense document type. + + Falls back to a Title-Cased version of the enum value so the required + ``type`` field is never empty, even for types added after this map. + """ + dt = _coerce_document_type(document_type) + if dt is None: + if document_type is None: + return "Document" + return str(document_type).replace("_", " ").title() or "Document" + return OKF_TYPE_BY_DOCUMENT_TYPE.get(dt) or dt.value.replace("_", " ").title() + + +# Per-type metadata keys that hold the canonical source URL, checked first. +_RESOURCE_KEYS_BY_TYPE: dict[DocumentType, tuple[str, ...]] = { + DocumentType.GOOGLE_DRIVE_FILE: ("webViewLink", "web_view_link"), + DocumentType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR: ("webViewLink", "web_view_link"), + DocumentType.GITHUB_CONNECTOR: ("html_url", "url"), + DocumentType.NOTION_CONNECTOR: ("url",), + DocumentType.LINEAR_CONNECTOR: ("url",), + DocumentType.JIRA_CONNECTOR: ("url",), + DocumentType.CONFLUENCE_CONNECTOR: ("url",), + DocumentType.SLACK_CONNECTOR: ("permalink", "url"), + DocumentType.DISCORD_CONNECTOR: ("url",), + DocumentType.CLICKUP_CONNECTOR: ("url",), + DocumentType.YOUTUBE_VIDEO: ("url", "video_url"), + DocumentType.LUMA_CONNECTOR: ("url",), + DocumentType.BOOKSTACK_CONNECTOR: ("url",), +} + +# Generic URL-bearing keys checked for any type as a fallback. +_GENERIC_RESOURCE_KEYS: tuple[str, ...] = ( + "url", + "URL", + "source_url", + "sourceUrl", + "source", + "link", + "permalink", + "web_url", + "webViewLink", + "html_url", +) + + +def okf_resource( + document_type: DocumentType | str | None, metadata: dict[str, Any] | None +) -> str | None: + """Return the canonical source URI for a document, or ``None`` if it has none.""" + if not isinstance(metadata, dict) or not metadata: + return None + + dt = _coerce_document_type(document_type) + type_keys = _RESOURCE_KEYS_BY_TYPE.get(dt, ()) if dt is not None else () + + for key in (*type_keys, *_GENERIC_RESOURCE_KEYS): + value = metadata.get(key) + if isinstance(value, str): + candidate = value.strip() + if candidate.startswith(("http://", "https://")): + return candidate + return None From d694bcd1914c2fde8742a1c13bf2bd23fc5ef7b5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:58:52 +0200 Subject: [PATCH 02/17] add OKF serializer --- .../app/services/okf/serializer.py | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 surfsense_backend/app/services/okf/serializer.py diff --git a/surfsense_backend/app/services/okf/serializer.py b/surfsense_backend/app/services/okf/serializer.py new file mode 100644 index 000000000..c866badf9 --- /dev/null +++ b/surfsense_backend/app/services/okf/serializer.py @@ -0,0 +1,193 @@ +"""Serialize SurfSense knowledge into Open Knowledge Format (OKF v0.1). + +Pure functions with no HTTP / MCP / framework dependencies: given a +:class:`~app.db.Document` (and, for listings, its neighbours) they return +OKF-conformant markdown. Every consumer (ZIP export, REST, MCP, agents) calls +these rather than re-implementing frontmatter. + +Spec: https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import yaml + +from app.db import Document +from app.services.okf.type_mapping import okf_resource, okf_type + +# Reserved OKF filenames; never used for concept documents. +INDEX_FILENAME = "index.md" +LOG_FILENAME = "log.md" + +_FRONTMATTER_DELIMITER = "---" + + +def _tags_from_metadata(metadata: dict[str, Any] | None) -> list[str] | None: + if not isinstance(metadata, dict): + return None + raw = metadata.get("tags") + if isinstance(raw, list): + tags = [str(t).strip() for t in raw if str(t).strip()] + return tags or None + return None + + +def _timestamp(document: Document) -> str | None: + when = document.updated_at or document.created_at + if when is None: + return None + # ISO 8601, matching Google's sample bundles (e.g. 2026-05-28T22:49:59+00:00). + return when.isoformat() + + +def build_frontmatter(document: Document) -> dict[str, Any]: + """Build the ordered OKF frontmatter mapping for a document. + + Only ``type`` is required; recommended keys are included only when we have a + value. Insertion order is preserved in the emitted YAML. + """ + metadata = document.document_metadata if isinstance(document.document_metadata, dict) else {} + + frontmatter: dict[str, Any] = {"type": okf_type(document.document_type)} + + resource = okf_resource(document.document_type, metadata) + if resource: + frontmatter["resource"] = resource + + title = (document.title or "").strip() + if title: + frontmatter["title"] = title + + description = metadata.get("description") + if isinstance(description, str) and description.strip(): + frontmatter["description"] = description.strip() + + tags = _tags_from_metadata(metadata) + if tags: + frontmatter["tags"] = tags + + timestamp = _timestamp(document) + if timestamp: + frontmatter["timestamp"] = timestamp + + return frontmatter + + +def render_frontmatter(frontmatter: dict[str, Any]) -> str: + """Render a frontmatter mapping as a YAML block delimited by ``---``.""" + body = yaml.safe_dump( + frontmatter, + sort_keys=False, + allow_unicode=True, + default_flow_style=False, + ) + return f"{_FRONTMATTER_DELIMITER}\n{body}{_FRONTMATTER_DELIMITER}\n" + + +def document_to_concept(document: Document, *, body: str) -> str: + """Serialize a document as an OKF concept: frontmatter + ``body``. + + ``body`` is caller-resolved markdown, keeping this a pure formatting step. + """ + frontmatter = render_frontmatter(build_frontmatter(document)) + body_text = (body or "").strip("\n") + return f"{frontmatter}\n{body_text}\n" + + +@dataclass(frozen=True) +class ConceptRef: + """One concept entry for an ``index.md`` listing.""" + + title: str + filename: str # relative to the directory, e.g. "orders.md" + type: str # OKF type, used as the grouping heading + description: str | None = None + + +@dataclass(frozen=True) +class SubdirRef: + """One subdirectory entry for an ``index.md`` listing.""" + + name: str # directory name, e.g. "tables" + description: str | None = None + + +@dataclass(frozen=True) +class LogEntry: + """One line of an OKF ``log.md``: a concept and when it last changed.""" + + title: str + timestamp: str | None = None # ISO-8601, or None when unknown + + +def folder_to_log(entries: list[LogEntry]) -> str: + """Build a minimal OKF ``log.md`` body for one directory. + + Lists each concept newest-first with the time it last changed (from a + document's ``updated_at``/``created_at``); undated entries sort last. Returns + an empty string when there is nothing to log. + + ponytail: this is a last-touched summary, not a per-field change history. A + fuller changelog would read ``DocumentVersion`` rows; upgrade there if + consumers ever need diffs rather than "what changed and when". + """ + if not entries: + return "" + ordered = sorted( + entries, + key=lambda e: (e.timestamp is not None, e.timestamp or "", e.title), + reverse=True, + ) + lines = ["# Change Log", ""] + for entry in ordered: + when = f" - {entry.timestamp}" if entry.timestamp else "" + lines.append(f"* {entry.title}{when}") + return "\n".join(lines) + "\n" + + +def _index_bullet(title: str, link: str, description: str | None) -> str: + bullet = f"* [{title}]({link})" + if description: + # Keep index descriptions to a single line. + bullet += f" - {' '.join(description.split())}" + return bullet + + +def folder_to_index( + *, + concepts: list[ConceptRef] | None = None, + subdirectories: list[SubdirRef] | None = None, +) -> str: + """Build an OKF ``index.md`` body (no frontmatter) for one directory. + + Subdirectories are listed under a ``# Subdirectories`` heading and concepts + are grouped under their ``type`` heading, mirroring Google's sample bundles. + Returns an empty string when there is nothing to list. + """ + concepts = concepts or [] + subdirectories = subdirectories or [] + sections: list[str] = [] + + if subdirectories: + lines = ["# Subdirectories", ""] + for sub in sorted(subdirectories, key=lambda s: s.name.lower()): + lines.append( + _index_bullet(sub.name, f"{sub.name}/{INDEX_FILENAME}", sub.description) + ) + sections.append("\n".join(lines)) + + by_type: dict[str, list[ConceptRef]] = {} + for concept in concepts: + by_type.setdefault(concept.type, []).append(concept) + for type_heading in sorted(by_type): + lines = [f"# {type_heading}", ""] + for concept in sorted(by_type[type_heading], key=lambda c: c.title.lower()): + lines.append( + _index_bullet(concept.title, concept.filename, concept.description) + ) + sections.append("\n".join(lines)) + + return ("\n\n".join(sections) + "\n") if sections else "" From f44810088846fe1889753f87c2e9e11dc28f0aa9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:58:52 +0200 Subject: [PATCH 03/17] add OKF conformance validator --- .../app/services/okf/validator.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 surfsense_backend/app/services/okf/validator.py diff --git a/surfsense_backend/app/services/okf/validator.py b/surfsense_backend/app/services/okf/validator.py new file mode 100644 index 000000000..4d11c4ba1 --- /dev/null +++ b/surfsense_backend/app/services/okf/validator.py @@ -0,0 +1,90 @@ +"""Minimal OKF v0.1 conformance checks. + +A bundle conforms if every non-reserved ``.md`` file has a parseable YAML +frontmatter block whose ``type`` is a non-empty string. Reserved files +(``index.md``, ``log.md``) are exempt from the frontmatter requirement. +Consumers must stay permissive, so this is for producers to self-check what they +emit, not to reject foreign bundles. + +Spec: https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md +""" + +from __future__ import annotations + +import yaml + +from app.services.okf.serializer import INDEX_FILENAME, LOG_FILENAME + +_DELIMITER = "---" +RESERVED_FILENAMES = frozenset({INDEX_FILENAME, LOG_FILENAME}) + +# The OKF v0.1 frontmatter contract, shared by serializer and validator. Only +# ``type`` is mandatory; the rest are recommended. +REQUIRED_FRONTMATTER_KEYS: tuple[str, ...] = ("type",) +RECOMMENDED_FRONTMATTER_KEYS: tuple[str, ...] = ( + "title", + "description", + "resource", + "tags", + "timestamp", +) + + +def parse_frontmatter(text: str) -> tuple[dict | None, str | None]: + """Split an OKF concept into ``(frontmatter_dict, error)``. + + Returns ``(dict, None)`` on success or ``(None, reason)`` if the frontmatter + block is missing or not a parseable YAML mapping. + """ + if not text.startswith(_DELIMITER + "\n"): + return None, "missing opening '---' frontmatter delimiter" + rest = text[len(_DELIMITER) + 1 :] + end = rest.find("\n" + _DELIMITER) + if end == -1: + return None, "missing closing '---' frontmatter delimiter" + block = rest[:end] + try: + data = yaml.safe_load(block) + except yaml.YAMLError as exc: + return None, f"frontmatter is not valid YAML: {exc}" + if not isinstance(data, dict): + return None, "frontmatter is not a YAML mapping" + return data, None + + +def validate_concept(text: str) -> list[str]: + """Return conformance errors for a single concept document (empty == conforms).""" + frontmatter, error = parse_frontmatter(text) + if error: + return [error] + errors: list[str] = [] + for key in REQUIRED_FRONTMATTER_KEYS: + value = frontmatter.get(key) + if not isinstance(value, str) or not value.strip(): + errors.append(f"frontmatter '{key}' is missing or empty") + return errors + + +def is_conformant_concept(text: str) -> bool: + """True if a single concept document conforms to OKF v0.1.""" + return not validate_concept(text) + + +def validate_bundle(files: dict[str, str]) -> dict[str, list[str]]: + """Validate a bundle given a mapping of relative path -> file contents. + + Only non-reserved ``.md`` files are checked for concept conformance. Returns + a mapping of path -> errors for every non-conformant file (an empty mapping + means the whole bundle conforms). + """ + problems: dict[str, list[str]] = {} + for path, contents in files.items(): + if not path.endswith(".md"): + continue + filename = path.rsplit("/", 1)[-1] + if filename in RESERVED_FILENAMES: + continue + errors = validate_concept(contents) + if errors: + problems[path] = errors + return problems From 3401b0ae0ea8d37422d65956ab2525f18d924939 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:58:52 +0200 Subject: [PATCH 04/17] add OKF package api --- .../app/services/okf/__init__.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 surfsense_backend/app/services/okf/__init__.py diff --git a/surfsense_backend/app/services/okf/__init__.py b/surfsense_backend/app/services/okf/__init__.py new file mode 100644 index 000000000..1bcb1ccb1 --- /dev/null +++ b/surfsense_backend/app/services/okf/__init__.py @@ -0,0 +1,56 @@ +"""Open Knowledge Format (OKF v0.1) serialization for the SurfSense KB. + +Single source of truth for turning documents and folders into OKF concepts, +``index.md`` listings and ``log.md`` logs. Pure KB-layer functions; every +consumer (ZIP export, REST, MCP, agents) calls in. + +The OKF-native model: ``Document`` rows are canonical, and frontmatter is +*derived* from their columns on read (never stored), so rows are conformant by +construction. Chunks and embeddings are a *derived*, rebuildable search +projection - never a source of truth. + +Spec: https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md +""" + +from app.services.okf.serializer import ( + INDEX_FILENAME, + LOG_FILENAME, + ConceptRef, + LogEntry, + SubdirRef, + build_frontmatter, + document_to_concept, + folder_to_index, + folder_to_log, + render_frontmatter, +) +from app.services.okf.type_mapping import okf_resource, okf_type +from app.services.okf.validator import ( + RECOMMENDED_FRONTMATTER_KEYS, + REQUIRED_FRONTMATTER_KEYS, + is_conformant_concept, + parse_frontmatter, + validate_bundle, + validate_concept, +) + +__all__ = [ + "INDEX_FILENAME", + "LOG_FILENAME", + "ConceptRef", + "LogEntry", + "SubdirRef", + "build_frontmatter", + "document_to_concept", + "folder_to_index", + "folder_to_log", + "render_frontmatter", + "okf_resource", + "okf_type", + "RECOMMENDED_FRONTMATTER_KEYS", + "REQUIRED_FRONTMATTER_KEYS", + "is_conformant_concept", + "parse_frontmatter", + "validate_bundle", + "validate_concept", +] From 6a72051e063b9136fd7001767d1a3e74982e84d3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:58:52 +0200 Subject: [PATCH 05/17] content-negotiate OKF on GET document --- .../app/routes/documents_routes.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index a5d31f07c..574bf78b5 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -1,7 +1,8 @@ # Force asyncio to use standard event loop before unstructured imports import asyncio -from fastapi import APIRouter, Depends, Form, HTTPException, Query, UploadFile +from fastapi import APIRouter, Depends, Form, HTTPException, Query, Request, UploadFile +from fastapi.responses import PlainTextResponse from pydantic import BaseModel as PydanticBaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select @@ -34,6 +35,8 @@ from app.schemas import ( FolderRead, PaginatedResponse, ) +from app.services.export_service import resolve_document_markdown +from app.services.okf import document_to_concept from app.services.task_dispatcher import TaskDispatcher, get_task_dispatcher from app.users import get_auth_context from app.utils.rbac import check_permission @@ -1255,12 +1258,14 @@ async def get_document_chunks_paginated( @router.get("/documents/{document_id}", response_model=DocumentRead) async def read_document( document_id: int, + request: Request, session: AsyncSession = Depends(get_async_session), auth: AuthContext = Depends(get_auth_context), ): - """ - Get a specific document by ID. - Requires DOCUMENTS_READ permission for the workspace. + """Get a document by ID. + + ``Accept: text/markdown`` returns the OKF concept; otherwise the JSON record. + Requires DOCUMENTS_READ permission. """ try: result = await session.execute( @@ -1282,6 +1287,16 @@ async def read_document( "You don't have permission to read documents in this workspace", ) + # ponytail: substring match, not RFC 7231 q-values (OKF is the only non-JSON view). + if "text/markdown" in request.headers.get("accept", ""): + body = ( + await resolve_document_markdown(session, document) + or document.content + or "" + ) + concept = document_to_concept(document, body=body) + return PlainTextResponse(concept, media_type="text/markdown") + raw_content = document.content or "" return DocumentRead( id=document.id, From 3cf315880d73aaf6361aeda982862c4c39c35313 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:58:52 +0200 Subject: [PATCH 06/17] export full OKF bundle with index and log --- .../app/services/export_service.py | 146 +++++++++++++++++- 1 file changed, 141 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/services/export_service.py b/surfsense_backend/app/services/export_service.py index 2d1a8c2cf..e718c25d1 100644 --- a/surfsense_backend/app/services/export_service.py +++ b/surfsense_backend/app/services/export_service.py @@ -12,9 +12,26 @@ from sqlalchemy.future import select from app.db import Chunk, Document, Folder from app.services.folder_service import get_folder_subtree_ids +from app.services.okf import ( + INDEX_FILENAME, + LOG_FILENAME, + ConceptRef, + LogEntry, + SubdirRef, + document_to_concept, + folder_to_index, + folder_to_log, + okf_type, +) logger = logging.getLogger(__name__) +# Root index.md declares the targeted OKF version in frontmatter - the one place +# the spec permits frontmatter in an index file. +_ROOT_INDEX_FRONTMATTER = '---\nokf_version: "0.1"\n---\n\n' + +_RESERVED_STEMS = {"index", "log"} + def _sanitize_filename(title: str) -> str: safe = "".join(c if c.isalnum() or c in " -_." else "_" for c in title).strip() @@ -43,7 +60,7 @@ def _build_folder_path_map(folders: list[Folder]) -> dict[int, str]: return cache -async def _get_document_markdown( +async def resolve_document_markdown( session: AsyncSession, document: Document ) -> str | None: """Resolve markdown content using the 3-tier fallback: @@ -71,6 +88,67 @@ async def _get_document_markdown( return None +def _build_index_files( + dir_concepts: dict[str, list[ConceptRef]], +) -> list[tuple[str, str]]: + """Build ``index.md`` files for every directory (and ancestor) with content. + + Produces OKF progressive-disclosure listings: each directory lists its + concepts (grouped by type) and its immediate subdirectories. The bundle-root + index also declares ``okf_version``. + """ + all_dirs: set[str] = {""} + for dir_path in dir_concepts: + all_dirs.add(dir_path) + parts = dir_path.split("/") if dir_path else [] + for i in range(1, len(parts)): + all_dirs.add("/".join(parts[:i])) + + children_by_dir: dict[str, list[str]] = {} + for dir_path in all_dirs: + if not dir_path: + continue + parent = dir_path.rsplit("/", 1)[0] if "/" in dir_path else "" + children_by_dir.setdefault(parent, []).append(dir_path) + + index_files: list[tuple[str, str]] = [] + for dir_path in all_dirs: + subdirs = [ + SubdirRef(name=child.rsplit("/", 1)[-1]) + for child in children_by_dir.get(dir_path, []) + ] + body = folder_to_index( + concepts=dir_concepts.get(dir_path, []), + subdirectories=subdirs, + ) + if not body: + continue + if dir_path: + index_files.append((f"{dir_path}/{INDEX_FILENAME}", body)) + else: + index_files.append((INDEX_FILENAME, _ROOT_INDEX_FRONTMATTER + body)) + + return index_files + + +def _build_log_files( + dir_logs: dict[str, list[LogEntry]], +) -> list[tuple[str, str]]: + """Build ``log.md`` files for every directory that holds concepts. + + Unlike ``index.md``, logs are only synthesized where documents actually live + (no ancestor synthesis): an empty intermediate directory has nothing to log. + """ + log_files: list[tuple[str, str]] = [] + for dir_path, entries in dir_logs.items(): + body = folder_to_log(entries) + if not body: + continue + path = f"{dir_path}/{LOG_FILENAME}" if dir_path else LOG_FILENAME + log_files.append((path, body)) + return log_files + + @dataclass class ExportResult: zip_path: str @@ -120,6 +198,10 @@ async def build_export_zip( used_paths: dict[str, int] = {} skipped_docs: list[str] = [] is_first_batch = True + # dir path -> concepts it holds, accumulated across batches for index.md. + dir_concepts: dict[str, list[ConceptRef]] = {} + # dir path -> log entries it holds, accumulated across batches for log.md. + dir_logs: dict[str, list[LogEntry]] = {} try: offset = 0 @@ -143,7 +225,7 @@ async def build_export_zip( skipped_docs.append(doc.title or "Untitled") continue - markdown = await _get_document_markdown(session, doc) + markdown = await resolve_document_markdown(session, doc) if not markdown or not markdown.strip(): continue @@ -153,6 +235,9 @@ async def build_export_zip( dir_path = "" base_name = _sanitize_filename(doc.title or "Untitled") + # Never collide with reserved OKF filenames (index.md, log.md). + if base_name.lower() in _RESERVED_STEMS: + base_name = f"{base_name}_" file_path = ( f"{dir_path}/{base_name}.md" if dir_path else f"{base_name}.md" ) @@ -160,14 +245,41 @@ async def build_export_zip( if file_path in used_paths: used_paths[file_path] += 1 suffix = used_paths[file_path] + base_name = f"{base_name}_{suffix}" file_path = ( - f"{dir_path}/{base_name}_{suffix}.md" + f"{dir_path}/{base_name}.md" if dir_path - else f"{base_name}_{suffix}.md" + else f"{base_name}.md" ) used_paths[file_path] = used_paths.get(file_path, 0) + 1 - entries.append((file_path, markdown)) + concept = document_to_concept(doc, body=markdown) + entries.append((file_path, concept)) + + metadata = ( + doc.document_metadata + if isinstance(doc.document_metadata, dict) + else {} + ) + description = metadata.get("description") + dir_concepts.setdefault(dir_path, []).append( + ConceptRef( + title=doc.title or "Untitled", + filename=f"{base_name}.md", + type=okf_type(doc.document_type), + description=description + if isinstance(description, str) and description.strip() + else None, + ) + ) + + changed_at = doc.updated_at or doc.created_at + dir_logs.setdefault(dir_path, []).append( + LogEntry( + title=doc.title or "Untitled", + timestamp=changed_at.isoformat() if changed_at else None, + ) + ) if entries: mode = "w" if is_first_batch else "a" @@ -183,6 +295,30 @@ async def build_export_zip( offset += batch_size + index_files = _build_index_files(dir_concepts) + if index_files: + mode = "w" if is_first_batch else "a" + + def _write_indexes(m: str = mode, e: list = index_files) -> None: + with zipfile.ZipFile(tmp_path, m, zipfile.ZIP_DEFLATED) as zf: + for path, content in e: + zf.writestr(path, content) + + await asyncio.to_thread(_write_indexes) + is_first_batch = False + + log_files = _build_log_files(dir_logs) + if log_files: + mode = "w" if is_first_batch else "a" + + def _write_logs(m: str = mode, e: list = log_files) -> None: + with zipfile.ZipFile(tmp_path, m, zipfile.ZIP_DEFLATED) as zf: + for path, content in e: + zf.writestr(path, content) + + await asyncio.to_thread(_write_logs) + is_first_batch = False + export_name = "knowledge-base" if folder_id is not None and folder_id in folder_path_map: export_name = _sanitize_filename(folder_path_map[folder_id].split("/")[0]) From edf38edc3fa975c233dac45259265a4a807b386d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:58:52 +0200 Subject: [PATCH 07/17] support per-request headers --- surfsense_mcp/mcp_server/core/client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/surfsense_mcp/mcp_server/core/client.py b/surfsense_mcp/mcp_server/core/client.py index 9ae446ae2..d65b47b7e 100644 --- a/surfsense_mcp/mcp_server/core/client.py +++ b/surfsense_mcp/mcp_server/core/client.py @@ -61,13 +61,17 @@ class SurfSenseClient: json: Any | None = None, data: dict[str, Any] | None = None, files: Any | None = None, + headers: dict[str, str] | None = None, ) -> Any: - """Send a request and return the parsed body, or raise ``ToolError``.""" + """Send a request and return the parsed body, or raise ``ToolError``. + + ``headers`` overrides the client defaults for this call. + """ # Omit unset query params: sending them empty makes the API parse "" # as a value (e.g. int("") on folder_id) and fail. if params is not None: params = {key: value for key, value in params.items() if value is not None} - headers = self._auth_headers() + headers = {**self._auth_headers(), **(headers or {})} try: response = await self._http.request( method, From a01b292886c5654595b80008b219bacd2faaaa3e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:58:53 +0200 Subject: [PATCH 08/17] request OKF concept via content negotiation --- .../features/knowledge_base/search_tools.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/surfsense_mcp/mcp_server/features/knowledge_base/search_tools.py b/surfsense_mcp/mcp_server/features/knowledge_base/search_tools.py index c0f7f83a9..dc085e00f 100644 --- a/surfsense_mcp/mcp_server/features/knowledge_base/search_tools.py +++ b/surfsense_mcp/mcp_server/features/knowledge_base/search_tools.py @@ -123,11 +123,19 @@ def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) - Use this after surfsense_search_knowledge_base or surfsense_list_documents to open a specific document — search results only include the matching passages, this returns the whole text. + The markdown form is an Open Knowledge Format (OKF) concept: a YAML + frontmatter block (type, title, tags, resource, timestamp) followed by + the document body. """ - document = await client.request("GET", f"/documents/{document_id}") if response_format == "json": + document = await client.request("GET", f"/documents/{document_id}") return clip(to_json(document)) - return _render_document(document) + concept = await client.request( + "GET", + f"/documents/{document_id}", + headers={"Accept": "text/markdown"}, + ) + return clip(concept if isinstance(concept, str) else str(concept)) def _join(values: list[str] | None) -> str | None: @@ -169,14 +177,3 @@ def _render_document_list(result: dict | None) -> str: + (" · more available_" if has_more else "_") ) return "\n".join(lines) - - -def _render_document(document: dict) -> str: - content = clip(document.get("content", "") or "(empty)") - return ( - f"# {document.get('title', 'Untitled')} (id {document.get('id')})\n" - f"- type: {document.get('document_type')}\n" - f"- workspace: {document.get('workspace_id')}\n" - f"- updated: {document.get('updated_at')}\n\n" - f"{content}" - ) From 7c53909b495d3b4c0f6e30de3734376f33430a52 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:59:00 +0200 Subject: [PATCH 09/17] test: OKF serializer self-checks --- .../unit/services/okf/test_serializer.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 surfsense_backend/tests/unit/services/okf/test_serializer.py diff --git a/surfsense_backend/tests/unit/services/okf/test_serializer.py b/surfsense_backend/tests/unit/services/okf/test_serializer.py new file mode 100644 index 000000000..157ed44c8 --- /dev/null +++ b/surfsense_backend/tests/unit/services/okf/test_serializer.py @@ -0,0 +1,130 @@ +"""OKF serializer/validator self-checks: emitted concepts stay conformant and the +frontmatter fields consumers rely on (type/title/timestamp) round-trip. +""" + +from datetime import UTC, datetime + +from app.db import Document, DocumentType +from app.services.okf import ( + ConceptRef, + LogEntry, + SubdirRef, + document_to_concept, + folder_to_index, + folder_to_log, + is_conformant_concept, + parse_frontmatter, + validate_concept, +) + + +def _make_document() -> Document: + return Document( + title="Weekly Sync Notes", + document_type=DocumentType.NOTE, + document_metadata={"tags": ["team", "meeting"], "url": "https://example.com/n"}, + updated_at=datetime(2026, 5, 28, 22, 49, 59, tzinfo=UTC), + ) + + +def test_concept_is_conformant_and_roundtrips() -> None: + concept = document_to_concept(_make_document(), body="# Agenda\n\nShip OKF.") + + assert is_conformant_concept(concept) + + frontmatter, error = parse_frontmatter(concept) + assert error is None + assert frontmatter["type"] == "Note" + assert frontmatter["title"] == "Weekly Sync Notes" + assert frontmatter["tags"] == ["team", "meeting"] + assert frontmatter["resource"] == "https://example.com/n" + # timestamp must survive as an ISO-8601 string, not a parsed datetime. + assert frontmatter["timestamp"] == "2026-05-28T22:49:59+00:00" + assert "# Agenda" in concept + + +def test_type_is_always_present_even_without_metadata() -> None: + doc = Document(title="Raw", document_type=DocumentType.CRAWLED_URL) + concept = document_to_concept(doc, body="body") + frontmatter, error = parse_frontmatter(concept) + assert error is None + assert frontmatter["type"] == "Web Page" + # No source URL / tags available -> those recommended keys are omitted. + assert "resource" not in frontmatter + assert "tags" not in frontmatter + + +def test_validator_rejects_non_conformant_documents() -> None: + assert validate_concept("no frontmatter here") + assert validate_concept("---\ntitle: Missing type\n---\nbody") + + +def test_folder_index_groups_by_type_and_lists_subdirs() -> None: + index = folder_to_index( + concepts=[ + ConceptRef(title="Orders", filename="orders.md", type="Note", description="x"), + ], + subdirectories=[SubdirRef(name="tables", description="Table docs")], + ) + assert "# Subdirectories" in index + assert "* [tables](tables/index.md) - Table docs" in index + assert "# Note" in index + assert "* [Orders](orders.md) - x" in index + + +def test_folder_log_lists_concepts_newest_first() -> None: + log = folder_to_log( + [ + LogEntry(title="Older", timestamp="2026-01-01T00:00:00+00:00"), + LogEntry(title="Newer", timestamp="2026-06-01T00:00:00+00:00"), + LogEntry(title="Undated", timestamp=None), + ] + ) + assert "# Change Log" in log + # Newest dated entry precedes the older one; undated sorts last. + assert log.index("Newer") < log.index("Older") < log.index("Undated") + assert "* Newer - 2026-06-01T00:00:00+00:00" in log + + +def test_folder_log_is_empty_when_no_entries() -> None: + assert folder_to_log([]) == "" + + +def test_export_log_files_synthesized_only_where_docs_live() -> None: + from app.services.export_service import _build_log_files + + files = dict( + _build_log_files( + { + "": [LogEntry(title="Root Doc", timestamp="2026-05-01T00:00:00+00:00")], + "Research/AI": [LogEntry(title="Nested", timestamp=None)], + } + ) + ) + assert "# Change Log" in files["log.md"] + assert "Root Doc" in files["log.md"] + assert "Nested" in files["Research/AI/log.md"] + # No empty intermediate log: "Research" holds no concepts of its own. + assert "Research/log.md" not in files + + +def test_export_index_files_include_root_version_and_ancestors() -> None: + from app.services.export_service import _build_index_files + + # A concept nested two levels deep, with no direct docs in the middle dir. + files = dict( + _build_index_files( + { + "Research/AI": [ + ConceptRef(title="Note", filename="note.md", type="Note") + ] + } + ) + ) + + # Root, the empty intermediate dir, and the leaf all get an index.md so the + # hierarchy is fully navigable. + assert files["index.md"].startswith('---\nokf_version: "0.1"\n---') + assert "* [Research](Research/index.md)" in files["index.md"] + assert "* [AI](AI/index.md)" in files["Research/index.md"] + assert "* [Note](note.md)" in files["Research/AI/index.md"] From 863d9a2248a7e1089b710a99a1914d1c8e8984b9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:59:00 +0200 Subject: [PATCH 10/17] test: per-type OKF conformance --- .../unit/services/okf/test_conformance.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 surfsense_backend/tests/unit/services/okf/test_conformance.py diff --git a/surfsense_backend/tests/unit/services/okf/test_conformance.py b/surfsense_backend/tests/unit/services/okf/test_conformance.py new file mode 100644 index 000000000..da1988224 --- /dev/null +++ b/surfsense_backend/tests/unit/services/okf/test_conformance.py @@ -0,0 +1,51 @@ +"""Every ``DocumentType`` must serialize to a concept a permissive OKF consumer +can read: parseable frontmatter with a non-empty ``type``. Covers all types plus +missing metadata, empty body, and non-ASCII titles. +""" + +from datetime import UTC, datetime + +import pytest + +from app.db import Document, DocumentType +from app.services.okf import document_to_concept, is_conformant_concept +from app.services.okf.validator import ( + RECOMMENDED_FRONTMATTER_KEYS, + REQUIRED_FRONTMATTER_KEYS, +) + + +@pytest.mark.parametrize("document_type", list(DocumentType)) +def test_every_document_type_serializes_to_conformant_concept( + document_type: DocumentType, +) -> None: + doc = Document( + title="Sample", + document_type=document_type, + document_metadata={"url": "https://example.com/x"}, + updated_at=datetime(2026, 5, 28, tzinfo=UTC), + ) + assert is_conformant_concept(document_to_concept(doc, body="body")) + + +def test_conformant_without_metadata_or_body() -> None: + doc = Document(title="Bare", document_type=DocumentType.NOTE) + assert is_conformant_concept(document_to_concept(doc, body="")) + + +def test_conformant_with_non_ascii_title() -> None: + doc = Document(title="日本語ノート", document_type=DocumentType.NOTE) + concept = document_to_concept(doc, body="本文") + assert is_conformant_concept(concept) + assert "日本語ノート" in concept + + +def test_contract_marks_only_type_as_required() -> None: + assert REQUIRED_FRONTMATTER_KEYS == ("type",) + assert set(RECOMMENDED_FRONTMATTER_KEYS) == { + "title", + "description", + "resource", + "tags", + "timestamp", + } From 5d2f9f1ca84a966b39830a5c043223ec878d69aa Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:59:01 +0200 Subject: [PATCH 11/17] test: connector document is OKF concept --- .../tests/unit/services/okf/test_ingestion.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 surfsense_backend/tests/unit/services/okf/test_ingestion.py diff --git a/surfsense_backend/tests/unit/services/okf/test_ingestion.py b/surfsense_backend/tests/unit/services/okf/test_ingestion.py new file mode 100644 index 000000000..e7c49bc33 --- /dev/null +++ b/surfsense_backend/tests/unit/services/okf/test_ingestion.py @@ -0,0 +1,31 @@ +"""A ConnectorDocument (the indexing write door) must serialize to a valid OKF +concept. Per-``DocumentType`` conformance lives in ``test_conformance.py``. +""" + +from app.db import Document, DocumentType +from app.indexing_pipeline.connector_document import ConnectorDocument +from app.services.okf import document_to_concept, is_conformant_concept + + +def _document_from(connector_doc: ConnectorDocument) -> Document: + """Mirror how prepare_for_indexing builds a Document from a ConnectorDocument.""" + return Document( + title=connector_doc.title, + document_type=connector_doc.document_type, + source_markdown=connector_doc.source_markdown, + document_metadata=connector_doc.metadata, + ) + + +def test_minimal_connector_document_yields_conformant_concept() -> None: + connector_doc = ConnectorDocument( + title="Bare", + source_markdown="just a body", + unique_id="u1", + document_type=DocumentType.FILE, + workspace_id=1, + created_by_id="user-1", + ) + doc = _document_from(connector_doc) + concept = document_to_concept(doc, body=doc.source_markdown) + assert is_conformant_concept(concept) From bdc03670a57b665d4583cf305f15e3be365d0b76 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:59:01 +0200 Subject: [PATCH 12/17] test: OKF bundle conformance audit --- .../tests/unit/services/okf/test_audit.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 surfsense_backend/tests/unit/services/okf/test_audit.py diff --git a/surfsense_backend/tests/unit/services/okf/test_audit.py b/surfsense_backend/tests/unit/services/okf/test_audit.py new file mode 100644 index 000000000..f8303e237 --- /dev/null +++ b/surfsense_backend/tests/unit/services/okf/test_audit.py @@ -0,0 +1,44 @@ +"""A real export bundle - concepts plus reserved ``index.md``/``log.md`` - must +pass the same ``validate_bundle`` a consumer would run. +""" + +from datetime import UTC, datetime + +from app.db import Document, DocumentType +from app.services.okf import ( + LogEntry, + document_to_concept, + folder_to_index, + folder_to_log, + validate_bundle, +) + + +def _sample_bundle() -> dict[str, str]: + note = Document( + title="Weekly Sync", + document_type=DocumentType.NOTE, + document_metadata={"tags": ["team"]}, + updated_at=datetime(2026, 5, 28, tzinfo=UTC), + ) + page = Document(title="Docs Home", document_type=DocumentType.CRAWLED_URL) + return { + "weekly-sync.md": document_to_concept(note, body="# Agenda"), + "docs-home.md": document_to_concept(page, body="content"), + # Reserved files carry no frontmatter and must be exempt from the check. + "index.md": folder_to_index(), + "log.md": folder_to_log( + [LogEntry(title="Weekly Sync", timestamp="2026-05-28T00:00:00+00:00")] + ), + } + + +def test_real_export_bundle_is_conformant() -> None: + assert validate_bundle(_sample_bundle()) == {} + + +def test_audit_flags_a_drifted_concept() -> None: + bundle = _sample_bundle() + bundle["broken.md"] = "no frontmatter at all" + problems = validate_bundle(bundle) + assert list(problems) == ["broken.md"] From ea20f0042def858821be1304c4cc16ae69cbed50 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:59:01 +0200 Subject: [PATCH 13/17] test: OKF export bundle is conformant --- .../integration/test_okf_export_bundle.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 surfsense_backend/tests/integration/test_okf_export_bundle.py diff --git a/surfsense_backend/tests/integration/test_okf_export_bundle.py b/surfsense_backend/tests/integration/test_okf_export_bundle.py new file mode 100644 index 000000000..bccd3ac4a --- /dev/null +++ b/surfsense_backend/tests/integration/test_okf_export_bundle.py @@ -0,0 +1,77 @@ +"""A real export must be a conformant OKF bundle end-to-end (real DB). + +Unit tests cover the pure serializer; this drives the whole export pipeline +(folder-path map, batching, ZIP writing) and asserts the emitted artifact - +concept files plus reserved ``index.md``/``log.md`` - passes ``validate_bundle``. +""" + +import os +import zipfile + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import Document, DocumentType, Folder, User, Workspace +from app.services.export_service import build_export_zip +from app.services.okf import validate_bundle + +pytestmark = pytest.mark.integration + + +async def _add_doc( + session: AsyncSession, + *, + workspace: Workspace, + user: User, + title: str, + folder_id: int | None, + uid: str, +) -> Document: + doc = Document( + title=title, + document_type=DocumentType.NOTE, + document_metadata={"tags": ["team"]}, + content="body text", + content_hash=uid, + unique_identifier_hash=uid, + source_markdown=f"# {title}\n\nBody.", + workspace_id=workspace.id, + created_by_id=user.id, + folder_id=folder_id, + ) + session.add(doc) + await session.flush() + return doc + + +async def test_export_bundle_is_okf_conformant( + db_session: AsyncSession, db_user: User, db_workspace: Workspace +): + folder = Folder(name="Research", position="0", workspace_id=db_workspace.id) + db_session.add(folder) + await db_session.flush() + + await _add_doc( + db_session, workspace=db_workspace, user=db_user, + title="Root Note", folder_id=None, uid="okf-export-root", + ) + await _add_doc( + db_session, workspace=db_workspace, user=db_user, + title="Nested Note", folder_id=folder.id, uid="okf-export-nested", + ) + + result = await build_export_zip(db_session, db_workspace.id) + try: + with zipfile.ZipFile(result.zip_path) as zf: + files = {name: zf.read(name).decode("utf-8") for name in zf.namelist()} + finally: + os.unlink(result.zip_path) + + # Directory structure: concepts nested by folder, plus reserved files. + assert "Root Note.md" in files + assert "Research/Nested Note.md" in files + assert files["index.md"].startswith('---\nokf_version: "0.1"\n---') + assert any(name.endswith("log.md") for name in files) + + # The whole bundle conforms; reserved index/log files are exempt. + assert validate_bundle(files) == {} From 826910f50d86943f1e5377dc28b2c3d805edd155 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:59:01 +0200 Subject: [PATCH 14/17] test: OKF path-identity roundtrip --- .../integration/test_okf_path_identity.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 surfsense_backend/tests/integration/test_okf_path_identity.py diff --git a/surfsense_backend/tests/integration/test_okf_path_identity.py b/surfsense_backend/tests/integration/test_okf_path_identity.py new file mode 100644 index 000000000..8c12d7844 --- /dev/null +++ b/surfsense_backend/tests/integration/test_okf_path_identity.py @@ -0,0 +1,116 @@ +"""A document's virtual path must resolve back to the same row: concepts are +identified by path (``doc_to_virtual_path`` <-> ``virtual_path_to_doc``). Covers +the hard cases - folder nesting and colliding titles. +""" + +import pytest +import pytest_asyncio +from sqlalchemy.ext.asyncio import AsyncSession + +from app.agents.chat.runtime.path_resolver import ( + build_path_index, + doc_to_virtual_path, + virtual_path_to_doc, +) +from app.db import Document, DocumentType, Folder, User, Workspace + +pytestmark = pytest.mark.integration + + +async def _add_document( + session: AsyncSession, + *, + workspace: Workspace, + user: User, + title: str, + folder_id: int | None, + unique_hash: str, +) -> Document: + doc = Document( + title=title, + document_type=DocumentType.NOTE, + document_metadata={}, + content="body", + content_hash=unique_hash, + unique_identifier_hash=unique_hash, + source_markdown="body", + workspace_id=workspace.id, + created_by_id=user.id, + folder_id=folder_id, + ) + session.add(doc) + await session.flush() + return doc + + +async def _roundtrip( + session: AsyncSession, workspace: Workspace, doc: Document +) -> Document | None: + index = await build_path_index(session, workspace.id) + path = doc_to_virtual_path( + doc_id=doc.id, title=doc.title, folder_id=doc.folder_id, index=index + ) + return await virtual_path_to_doc( + session, workspace_id=workspace.id, virtual_path=path + ) + + +@pytest_asyncio.fixture +async def research_folder( + db_session: AsyncSession, db_workspace: Workspace +) -> Folder: + folder = Folder(name="Research", position="0", workspace_id=db_workspace.id) + db_session.add(folder) + await db_session.flush() + return folder + + +async def test_folder_nested_document_roundtrips( + db_session, db_user, db_workspace, research_folder +): + doc = await _add_document( + db_session, + workspace=db_workspace, + user=db_user, + title="My Note", + folder_id=research_folder.id, + unique_hash="hash-nested", + ) + assert await _roundtrip(db_session, db_workspace, doc) is doc + + +async def test_colliding_titles_get_distinct_resolvable_paths( + db_session, db_user, db_workspace +): + first = await _add_document( + db_session, + workspace=db_workspace, + user=db_user, + title="Hello", + folder_id=None, + unique_hash="hash-a", + ) + second = await _add_document( + db_session, + workspace=db_workspace, + user=db_user, + title="Hello", + folder_id=None, + unique_hash="hash-b", + ) + + index = await build_path_index(db_session, db_workspace.id) + first_path = doc_to_virtual_path( + doc_id=first.id, title=first.title, folder_id=None, index=index + ) + second_path = doc_to_virtual_path( + doc_id=second.id, title=second.title, folder_id=None, index=index + ) + # Distinct identities: the collision is broken by a " ().xml" suffix. + assert first_path != second_path + + # Only the disambiguated path is stable; the bare title stays ambiguous while a twin exists. + resolved = await virtual_path_to_doc( + db_session, workspace_id=db_workspace.id, virtual_path=second_path + ) + assert resolved is second From c99a8154c0ff61522061b62ced7e8f3472afa8af Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:59:01 +0200 Subject: [PATCH 15/17] test: content-negotiated OKF read over HTTP --- .../document_upload/test_okf_read.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 surfsense_backend/tests/integration/document_upload/test_okf_read.py diff --git a/surfsense_backend/tests/integration/document_upload/test_okf_read.py b/surfsense_backend/tests/integration/document_upload/test_okf_read.py new file mode 100644 index 000000000..67a11a767 --- /dev/null +++ b/surfsense_backend/tests/integration/document_upload/test_okf_read.py @@ -0,0 +1,66 @@ +"""GET /documents/{id} content-negotiates its representation (real HTTP, DB). + +Proves the live wiring the unit tests can't: ``Accept: text/markdown`` returns a +conformant OKF concept, and the default request still returns the JSON record - +both off the same endpoint, through the real FastAPI stack. +""" + +from __future__ import annotations + +import httpx +import pytest + +from app.services.okf import is_conformant_concept +from tests.utils.helpers import poll_document_status, upload_file + +pytestmark = pytest.mark.integration + + +async def _ready_doc_id( + client: httpx.AsyncClient, + headers: dict[str, str], + workspace_id: int, + cleanup_doc_ids: list[int], +) -> int: + resp = await upload_file(client, headers, "sample.txt", workspace_id=workspace_id) + assert resp.status_code == 200 + doc_ids = resp.json()["document_ids"] + cleanup_doc_ids.extend(doc_ids) + statuses = await poll_document_status( + client, headers, doc_ids, workspace_id=workspace_id + ) + assert statuses[doc_ids[0]]["status"]["state"] == "ready" + return doc_ids[0] + + +async def test_accept_markdown_returns_conformant_okf_concept( + client: httpx.AsyncClient, + headers: dict[str, str], + workspace_id: int, + cleanup_doc_ids: list[int], +): + doc_id = await _ready_doc_id(client, headers, workspace_id, cleanup_doc_ids) + + resp = await client.get( + f"/api/v1/documents/{doc_id}", + headers={**headers, "Accept": "text/markdown"}, + ) + + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("text/markdown") + assert is_conformant_concept(resp.text) + + +async def test_default_accept_returns_json_record( + client: httpx.AsyncClient, + headers: dict[str, str], + workspace_id: int, + cleanup_doc_ids: list[int], +): + doc_id = await _ready_doc_id(client, headers, workspace_id, cleanup_doc_ids) + + resp = await client.get(f"/api/v1/documents/{doc_id}", headers=headers) + + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("application/json") + assert resp.json()["id"] == doc_id From 9a183924842205b680c6441faff3e7024105ce6b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:59:01 +0200 Subject: [PATCH 16/17] test: tidy concept-identity roundtrip docstring --- .../agents/new_chat/test_path_resolver.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py b/surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py index ba9dc242b..ad3b17c9c 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py @@ -79,6 +79,33 @@ class TestDocToVirtualPath: assert path == f"{DOCUMENTS_ROOT}/notes/A.xml" +class TestConceptIdentityRoundTrip: + """A document's virtual path must parse back to the same (folder, title) - the + OKF path-identity guarantee at the grammar level, no DB required.""" + + def test_folder_nested_document_roundtrips(self): + index = PathIndex(folder_paths={5: f"{DOCUMENTS_ROOT}/Research/AI"}) + path = doc_to_virtual_path( + doc_id=2, title="My Note", folder_id=5, index=index + ) + assert path == f"{DOCUMENTS_ROOT}/Research/AI/My Note.xml" + folder_parts, title = parse_documents_path(path) + assert folder_parts == ["Research", "AI"] + assert title == "My Note" + + def test_colliding_title_roundtrips_to_base_title(self): + # Second doc with the same title gets a " ()" suffix; parsing the + # path must strip the disambiguator and recover the original title. + index = PathIndex(occupants={f"{DOCUMENTS_ROOT}/Hello.xml": 7}) + path = doc_to_virtual_path( + doc_id=8, title="Hello", folder_id=None, index=index + ) + assert path == f"{DOCUMENTS_ROOT}/Hello (8).xml" + folder_parts, title = parse_documents_path(path) + assert folder_parts == [] + assert title == "Hello" + + class TestParseDocumentsPath: def test_extracts_folder_parts_and_title(self): parts, title = parse_documents_path(f"{DOCUMENTS_ROOT}/foo/bar/baz.xml") From 5f1cc7a6341990ac97068b966e1e2e494706a8e8 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 21 Jul 2026 19:59:01 +0200 Subject: [PATCH 17/17] test: MCP get_document forwards Accept header --- surfsense_mcp/tests/test_get_document_okf.py | 68 ++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 surfsense_mcp/tests/test_get_document_okf.py diff --git a/surfsense_mcp/tests/test_get_document_okf.py b/surfsense_mcp/tests/test_get_document_okf.py new file mode 100644 index 000000000..8b5cdae2f --- /dev/null +++ b/surfsense_mcp/tests/test_get_document_okf.py @@ -0,0 +1,68 @@ +"""surfsense_get_document round-trips through the real tool registration. + +The markdown form must ask the backend for the OKF concept via content +negotiation (``Accept: text/markdown``) and pass it through untouched; the JSON +form must leave the default ``application/json`` Accept in place. This is the +only coverage of the MCP-side glue that forwards the header. +""" + +from __future__ import annotations + +import asyncio +from unittest.mock import MagicMock + +import httpx +from mcp.server.fastmcp import FastMCP + +from mcp_server.core.client import SurfSenseClient +from mcp_server.features.knowledge_base import search_tools + +_CONCEPT = "---\ntype: Note\ntitle: T\n---\n\nBody." + + +def _client_recording(seen: dict) -> SurfSenseClient: + async def handler(request: httpx.Request) -> httpx.Response: + seen["path"] = request.url.path + seen["accept"] = request.headers.get("accept") + if "text/markdown" in (seen["accept"] or ""): + return httpx.Response( + 200, text=_CONCEPT, headers={"content-type": "text/markdown"} + ) + return httpx.Response(200, json={"id": 1, "title": "T"}) + + client = SurfSenseClient( + api_base="http://test/api/v1", timeout=5, fallback_api_key="ss_pat_x" + ) + client._http = httpx.AsyncClient( + base_url="http://test/api/v1", + headers={"Accept": "application/json"}, + transport=httpx.MockTransport(handler), + ) + return client + + +def _call_get_document(client: SurfSenseClient, **arguments) -> str: + mcp = FastMCP("test") + search_tools.register(mcp, client, MagicMock()) + blocks = asyncio.run(mcp.call_tool("surfsense_get_document", arguments)) + return "".join(block.text for block in blocks) + + +def test_markdown_requests_okf_concept_and_passes_it_through(): + seen: dict = {} + text = _call_get_document(_client_recording(seen), document_id=1) + + assert seen["path"] == "/api/v1/documents/1" + assert "text/markdown" in seen["accept"] + assert text == _CONCEPT + + +def test_json_keeps_default_accept(): + seen: dict = {} + text = _call_get_document( + _client_recording(seen), document_id=1, response_format="json" + ) + + assert seen["path"] == "/api/v1/documents/1" + assert seen["accept"] == "application/json" + assert '"id": 1' in text