Merge pull request #1617 from CREDO23/feature-okf

[Feat] Serve the knowledge base in Open Knowledge Format (OKF)

surfsense_backend/tests/integration/document_upload/test_okf_read.py

@@ -0,0 +1,66 @@
+"""GET /documents/{id} content-negotiates its representation (real HTTP, DB).
+
+Proves the live wiring the unit tests can't: ``Accept: text/markdown`` returns a
+conformant OKF concept, and the default request still returns the JSON record -
+both off the same endpoint, through the real FastAPI stack.
+"""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from app.services.okf import is_conformant_concept
+from tests.utils.helpers import poll_document_status, upload_file
+
+pytestmark = pytest.mark.integration
+
+
+async def _ready_doc_id(
+    client: httpx.AsyncClient,
+    headers: dict[str, str],
+    workspace_id: int,
+    cleanup_doc_ids: list[int],
+) -> int:
+    resp = await upload_file(client, headers, "sample.txt", workspace_id=workspace_id)
+    assert resp.status_code == 200
+    doc_ids = resp.json()["document_ids"]
+    cleanup_doc_ids.extend(doc_ids)
+    statuses = await poll_document_status(
+        client, headers, doc_ids, workspace_id=workspace_id
+    )
+    assert statuses[doc_ids[0]]["status"]["state"] == "ready"
+    return doc_ids[0]
+
+
+async def test_accept_markdown_returns_conformant_okf_concept(
+    client: httpx.AsyncClient,
+    headers: dict[str, str],
+    workspace_id: int,
+    cleanup_doc_ids: list[int],
+):
+    doc_id = await _ready_doc_id(client, headers, workspace_id, cleanup_doc_ids)
+
+    resp = await client.get(
+        f"/api/v1/documents/{doc_id}",
+        headers={**headers, "Accept": "text/markdown"},
+    )
+
+    assert resp.status_code == 200
+    assert resp.headers["content-type"].startswith("text/markdown")
+    assert is_conformant_concept(resp.text)
+
+
+async def test_default_accept_returns_json_record(
+    client: httpx.AsyncClient,
+    headers: dict[str, str],
+    workspace_id: int,
+    cleanup_doc_ids: list[int],
+):
+    doc_id = await _ready_doc_id(client, headers, workspace_id, cleanup_doc_ids)
+
+    resp = await client.get(f"/api/v1/documents/{doc_id}", headers=headers)
+
+    assert resp.status_code == 200
+    assert resp.headers["content-type"].startswith("application/json")
+    assert resp.json()["id"] == doc_id

surfsense_backend/tests/integration/test_okf_export_bundle.py

@@ -0,0 +1,77 @@
+"""A real export must be a conformant OKF bundle end-to-end (real DB).
+
+Unit tests cover the pure serializer; this drives the whole export pipeline
+(folder-path map, batching, ZIP writing) and asserts the emitted artifact -
+concept files plus reserved ``index.md``/``log.md`` - passes ``validate_bundle``.
+"""
+
+import os
+import zipfile
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db import Document, DocumentType, Folder, User, Workspace
+from app.services.export_service import build_export_zip
+from app.services.okf import validate_bundle
+
+pytestmark = pytest.mark.integration
+
+
+async def _add_doc(
+    session: AsyncSession,
+    *,
+    workspace: Workspace,
+    user: User,
+    title: str,
+    folder_id: int | None,
+    uid: str,
+) -> Document:
+    doc = Document(
+        title=title,
+        document_type=DocumentType.NOTE,
+        document_metadata={"tags": ["team"]},
+        content="body text",
+        content_hash=uid,
+        unique_identifier_hash=uid,
+        source_markdown=f"# {title}\n\nBody.",
+        workspace_id=workspace.id,
+        created_by_id=user.id,
+        folder_id=folder_id,
+    )
+    session.add(doc)
+    await session.flush()
+    return doc
+
+
+async def test_export_bundle_is_okf_conformant(
+    db_session: AsyncSession, db_user: User, db_workspace: Workspace
+):
+    folder = Folder(name="Research", position="0", workspace_id=db_workspace.id)
+    db_session.add(folder)
+    await db_session.flush()
+
+    await _add_doc(
+        db_session, workspace=db_workspace, user=db_user,
+        title="Root Note", folder_id=None, uid="okf-export-root",
+    )
+    await _add_doc(
+        db_session, workspace=db_workspace, user=db_user,
+        title="Nested Note", folder_id=folder.id, uid="okf-export-nested",
+    )
+
+    result = await build_export_zip(db_session, db_workspace.id)
+    try:
+        with zipfile.ZipFile(result.zip_path) as zf:
+            files = {name: zf.read(name).decode("utf-8") for name in zf.namelist()}
+    finally:
+        os.unlink(result.zip_path)
+
+    # Directory structure: concepts nested by folder, plus reserved files.
+    assert "Root Note.md" in files
+    assert "Research/Nested Note.md" in files
+    assert files["index.md"].startswith('---\nokf_version: "0.1"\n---')
+    assert any(name.endswith("log.md") for name in files)
+
+    # The whole bundle conforms; reserved index/log files are exempt.
+    assert validate_bundle(files) == {}

surfsense_backend/tests/integration/test_okf_path_identity.py

@@ -0,0 +1,116 @@
+"""A document's virtual path must resolve back to the same row: concepts are
+identified by path (``doc_to_virtual_path`` <-> ``virtual_path_to_doc``). Covers
+the hard cases - folder nesting and colliding titles.
+"""
+
+import pytest
+import pytest_asyncio
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.agents.chat.runtime.path_resolver import (
+    build_path_index,
+    doc_to_virtual_path,
+    virtual_path_to_doc,
+)
+from app.db import Document, DocumentType, Folder, User, Workspace
+
+pytestmark = pytest.mark.integration
+
+
+async def _add_document(
+    session: AsyncSession,
+    *,
+    workspace: Workspace,
+    user: User,
+    title: str,
+    folder_id: int | None,
+    unique_hash: str,
+) -> Document:
+    doc = Document(
+        title=title,
+        document_type=DocumentType.NOTE,
+        document_metadata={},
+        content="body",
+        content_hash=unique_hash,
+        unique_identifier_hash=unique_hash,
+        source_markdown="body",
+        workspace_id=workspace.id,
+        created_by_id=user.id,
+        folder_id=folder_id,
+    )
+    session.add(doc)
+    await session.flush()
+    return doc
+
+
+async def _roundtrip(
+    session: AsyncSession, workspace: Workspace, doc: Document
+) -> Document | None:
+    index = await build_path_index(session, workspace.id)
+    path = doc_to_virtual_path(
+        doc_id=doc.id, title=doc.title, folder_id=doc.folder_id, index=index
+    )
+    return await virtual_path_to_doc(
+        session, workspace_id=workspace.id, virtual_path=path
+    )
+
+
+@pytest_asyncio.fixture
+async def research_folder(
+    db_session: AsyncSession, db_workspace: Workspace
+) -> Folder:
+    folder = Folder(name="Research", position="0", workspace_id=db_workspace.id)
+    db_session.add(folder)
+    await db_session.flush()
+    return folder
+
+
+async def test_folder_nested_document_roundtrips(
+    db_session, db_user, db_workspace, research_folder
+):
+    doc = await _add_document(
+        db_session,
+        workspace=db_workspace,
+        user=db_user,
+        title="My Note",
+        folder_id=research_folder.id,
+        unique_hash="hash-nested",
+    )
+    assert await _roundtrip(db_session, db_workspace, doc) is doc
+
+
+async def test_colliding_titles_get_distinct_resolvable_paths(
+    db_session, db_user, db_workspace
+):
+    first = await _add_document(
+        db_session,
+        workspace=db_workspace,
+        user=db_user,
+        title="Hello",
+        folder_id=None,
+        unique_hash="hash-a",
+    )
+    second = await _add_document(
+        db_session,
+        workspace=db_workspace,
+        user=db_user,
+        title="Hello",
+        folder_id=None,
+        unique_hash="hash-b",
+    )
+
+    index = await build_path_index(db_session, db_workspace.id)
+    first_path = doc_to_virtual_path(
+        doc_id=first.id, title=first.title, folder_id=None, index=index
+    )
+    second_path = doc_to_virtual_path(
+        doc_id=second.id, title=second.title, folder_id=None, index=index
+    )
+    # Distinct identities: the collision is broken by a " (<id>).xml" suffix.
+    assert first_path != second_path
+
+    # Only the disambiguated path is stable; the bare title stays ambiguous while a twin exists.
+    resolved = await virtual_path_to_doc(
+        db_session, workspace_id=db_workspace.id, virtual_path=second_path
+    )
+    assert resolved is second

surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py

@@ -79,6 +79,33 @@ class TestDocToVirtualPath:
         assert path == f"{DOCUMENTS_ROOT}/notes/A.xml"
 
 
+class TestConceptIdentityRoundTrip:
+    """A document's virtual path must parse back to the same (folder, title) - the
+    OKF path-identity guarantee at the grammar level, no DB required."""
+
+    def test_folder_nested_document_roundtrips(self):
+        index = PathIndex(folder_paths={5: f"{DOCUMENTS_ROOT}/Research/AI"})
+        path = doc_to_virtual_path(
+            doc_id=2, title="My Note", folder_id=5, index=index
+        )
+        assert path == f"{DOCUMENTS_ROOT}/Research/AI/My Note.xml"
+        folder_parts, title = parse_documents_path(path)
+        assert folder_parts == ["Research", "AI"]
+        assert title == "My Note"
+
+    def test_colliding_title_roundtrips_to_base_title(self):
+        # Second doc with the same title gets a " (<id>)" suffix; parsing the
+        # path must strip the disambiguator and recover the original title.
+        index = PathIndex(occupants={f"{DOCUMENTS_ROOT}/Hello.xml": 7})
+        path = doc_to_virtual_path(
+            doc_id=8, title="Hello", folder_id=None, index=index
+        )
+        assert path == f"{DOCUMENTS_ROOT}/Hello (8).xml"
+        folder_parts, title = parse_documents_path(path)
+        assert folder_parts == []
+        assert title == "Hello"
+
+
 class TestParseDocumentsPath:
     def test_extracts_folder_parts_and_title(self):
         parts, title = parse_documents_path(f"{DOCUMENTS_ROOT}/foo/bar/baz.xml")

surfsense_backend/tests/unit/services/okf/test_audit.py

@@ -0,0 +1,44 @@
+"""A real export bundle - concepts plus reserved ``index.md``/``log.md`` - must
+pass the same ``validate_bundle`` a consumer would run.
+"""
+
+from datetime import UTC, datetime
+
+from app.db import Document, DocumentType
+from app.services.okf import (
+    LogEntry,
+    document_to_concept,
+    folder_to_index,
+    folder_to_log,
+    validate_bundle,
+)
+
+
+def _sample_bundle() -> dict[str, str]:
+    note = Document(
+        title="Weekly Sync",
+        document_type=DocumentType.NOTE,
+        document_metadata={"tags": ["team"]},
+        updated_at=datetime(2026, 5, 28, tzinfo=UTC),
+    )
+    page = Document(title="Docs Home", document_type=DocumentType.CRAWLED_URL)
+    return {
+        "weekly-sync.md": document_to_concept(note, body="# Agenda"),
+        "docs-home.md": document_to_concept(page, body="content"),
+        # Reserved files carry no frontmatter and must be exempt from the check.
+        "index.md": folder_to_index(),
+        "log.md": folder_to_log(
+            [LogEntry(title="Weekly Sync", timestamp="2026-05-28T00:00:00+00:00")]
+        ),
+    }
+
+
+def test_real_export_bundle_is_conformant() -> None:
+    assert validate_bundle(_sample_bundle()) == {}
+
+
+def test_audit_flags_a_drifted_concept() -> None:
+    bundle = _sample_bundle()
+    bundle["broken.md"] = "no frontmatter at all"
+    problems = validate_bundle(bundle)
+    assert list(problems) == ["broken.md"]

surfsense_backend/tests/unit/services/okf/test_conformance.py

@@ -0,0 +1,51 @@
+"""Every ``DocumentType`` must serialize to a concept a permissive OKF consumer
+can read: parseable frontmatter with a non-empty ``type``. Covers all types plus
+missing metadata, empty body, and non-ASCII titles.
+"""
+
+from datetime import UTC, datetime
+
+import pytest
+
+from app.db import Document, DocumentType
+from app.services.okf import document_to_concept, is_conformant_concept
+from app.services.okf.validator import (
+    RECOMMENDED_FRONTMATTER_KEYS,
+    REQUIRED_FRONTMATTER_KEYS,
+)
+
+
+@pytest.mark.parametrize("document_type", list(DocumentType))
+def test_every_document_type_serializes_to_conformant_concept(
+    document_type: DocumentType,
+) -> None:
+    doc = Document(
+        title="Sample",
+        document_type=document_type,
+        document_metadata={"url": "https://example.com/x"},
+        updated_at=datetime(2026, 5, 28, tzinfo=UTC),
+    )
+    assert is_conformant_concept(document_to_concept(doc, body="body"))
+
+
+def test_conformant_without_metadata_or_body() -> None:
+    doc = Document(title="Bare", document_type=DocumentType.NOTE)
+    assert is_conformant_concept(document_to_concept(doc, body=""))
+
+
+def test_conformant_with_non_ascii_title() -> None:
+    doc = Document(title="日本語ノート", document_type=DocumentType.NOTE)
+    concept = document_to_concept(doc, body="本文")
+    assert is_conformant_concept(concept)
+    assert "日本語ノート" in concept
+
+
+def test_contract_marks_only_type_as_required() -> None:
+    assert REQUIRED_FRONTMATTER_KEYS == ("type",)
+    assert set(RECOMMENDED_FRONTMATTER_KEYS) == {
+        "title",
+        "description",
+        "resource",
+        "tags",
+        "timestamp",
+    }

surfsense_backend/tests/unit/services/okf/test_ingestion.py

@@ -0,0 +1,31 @@
+"""A ConnectorDocument (the indexing write door) must serialize to a valid OKF
+concept. Per-``DocumentType`` conformance lives in ``test_conformance.py``.
+"""
+
+from app.db import Document, DocumentType
+from app.indexing_pipeline.connector_document import ConnectorDocument
+from app.services.okf import document_to_concept, is_conformant_concept
+
+
+def _document_from(connector_doc: ConnectorDocument) -> Document:
+    """Mirror how prepare_for_indexing builds a Document from a ConnectorDocument."""
+    return Document(
+        title=connector_doc.title,
+        document_type=connector_doc.document_type,
+        source_markdown=connector_doc.source_markdown,
+        document_metadata=connector_doc.metadata,
+    )
+
+
+def test_minimal_connector_document_yields_conformant_concept() -> None:
+    connector_doc = ConnectorDocument(
+        title="Bare",
+        source_markdown="just a body",
+        unique_id="u1",
+        document_type=DocumentType.FILE,
+        workspace_id=1,
+        created_by_id="user-1",
+    )
+    doc = _document_from(connector_doc)
+    concept = document_to_concept(doc, body=doc.source_markdown)
+    assert is_conformant_concept(concept)

surfsense_backend/tests/unit/services/okf/test_serializer.py

@@ -0,0 +1,130 @@
+"""OKF serializer/validator self-checks: emitted concepts stay conformant and the
+frontmatter fields consumers rely on (type/title/timestamp) round-trip.
+"""
+
+from datetime import UTC, datetime
+
+from app.db import Document, DocumentType
+from app.services.okf import (
+    ConceptRef,
+    LogEntry,
+    SubdirRef,
+    document_to_concept,
+    folder_to_index,
+    folder_to_log,
+    is_conformant_concept,
+    parse_frontmatter,
+    validate_concept,
+)
+
+
+def _make_document() -> Document:
+    return Document(
+        title="Weekly Sync Notes",
+        document_type=DocumentType.NOTE,
+        document_metadata={"tags": ["team", "meeting"], "url": "https://example.com/n"},
+        updated_at=datetime(2026, 5, 28, 22, 49, 59, tzinfo=UTC),
+    )
+
+
+def test_concept_is_conformant_and_roundtrips() -> None:
+    concept = document_to_concept(_make_document(), body="# Agenda\n\nShip OKF.")
+
+    assert is_conformant_concept(concept)
+
+    frontmatter, error = parse_frontmatter(concept)
+    assert error is None
+    assert frontmatter["type"] == "Note"
+    assert frontmatter["title"] == "Weekly Sync Notes"
+    assert frontmatter["tags"] == ["team", "meeting"]
+    assert frontmatter["resource"] == "https://example.com/n"
+    # timestamp must survive as an ISO-8601 string, not a parsed datetime.
+    assert frontmatter["timestamp"] == "2026-05-28T22:49:59+00:00"
+    assert "# Agenda" in concept
+
+
+def test_type_is_always_present_even_without_metadata() -> None:
+    doc = Document(title="Raw", document_type=DocumentType.CRAWLED_URL)
+    concept = document_to_concept(doc, body="body")
+    frontmatter, error = parse_frontmatter(concept)
+    assert error is None
+    assert frontmatter["type"] == "Web Page"
+    # No source URL / tags available -> those recommended keys are omitted.
+    assert "resource" not in frontmatter
+    assert "tags" not in frontmatter
+
+
+def test_validator_rejects_non_conformant_documents() -> None:
+    assert validate_concept("no frontmatter here")
+    assert validate_concept("---\ntitle: Missing type\n---\nbody")
+
+
+def test_folder_index_groups_by_type_and_lists_subdirs() -> None:
+    index = folder_to_index(
+        concepts=[
+            ConceptRef(title="Orders", filename="orders.md", type="Note", description="x"),
+        ],
+        subdirectories=[SubdirRef(name="tables", description="Table docs")],
+    )
+    assert "# Subdirectories" in index
+    assert "* [tables](tables/index.md) - Table docs" in index
+    assert "# Note" in index
+    assert "* [Orders](orders.md) - x" in index
+
+
+def test_folder_log_lists_concepts_newest_first() -> None:
+    log = folder_to_log(
+        [
+            LogEntry(title="Older", timestamp="2026-01-01T00:00:00+00:00"),
+            LogEntry(title="Newer", timestamp="2026-06-01T00:00:00+00:00"),
+            LogEntry(title="Undated", timestamp=None),
+        ]
+    )
+    assert "# Change Log" in log
+    # Newest dated entry precedes the older one; undated sorts last.
+    assert log.index("Newer") < log.index("Older") < log.index("Undated")
+    assert "* Newer - 2026-06-01T00:00:00+00:00" in log
+
+
+def test_folder_log_is_empty_when_no_entries() -> None:
+    assert folder_to_log([]) == ""
+
+
+def test_export_log_files_synthesized_only_where_docs_live() -> None:
+    from app.services.export_service import _build_log_files
+
+    files = dict(
+        _build_log_files(
+            {
+                "": [LogEntry(title="Root Doc", timestamp="2026-05-01T00:00:00+00:00")],
+                "Research/AI": [LogEntry(title="Nested", timestamp=None)],
+            }
+        )
+    )
+    assert "# Change Log" in files["log.md"]
+    assert "Root Doc" in files["log.md"]
+    assert "Nested" in files["Research/AI/log.md"]
+    # No empty intermediate log: "Research" holds no concepts of its own.
+    assert "Research/log.md" not in files
+
+
+def test_export_index_files_include_root_version_and_ancestors() -> None:
+    from app.services.export_service import _build_index_files
+
+    # A concept nested two levels deep, with no direct docs in the middle dir.
+    files = dict(
+        _build_index_files(
+            {
+                "Research/AI": [
+                    ConceptRef(title="Note", filename="note.md", type="Note")
+                ]
+            }
+        )
+    )
+
+    # Root, the empty intermediate dir, and the leaf all get an index.md so the
+    # hierarchy is fully navigable.
+    assert files["index.md"].startswith('---\nokf_version: "0.1"\n---')
+    assert "* [Research](Research/index.md)" in files["index.md"]
+    assert "* [AI](AI/index.md)" in files["Research/index.md"]
+    assert "* [Note](note.md)" in files["Research/AI/index.md"]