document-render: unify KB/web rendering on numbered [n] passages

Add a shared document_render package that renders sources as
<document view="excerpt|full"> blocks with server-assigned [n] passage
labels (KB locator {document_id, chunk_id}, web locator {url}). Wire the
KB read backend (kb_postgres) and read_file to the new renderer and drop
the legacy per-document XML renderer (document_xml, retrieved_context) and
the old chunk_index / matched="true" / <chunk id> read format.

surfsense_backend/tests/unit/agents/multi_agent_chat/shared/document_render/test_document.py

@@ -0,0 +1,152 @@
+"""Tests for the shared ``render_document`` (one ``<document>`` block)."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.agents.chat.multi_agent_chat.shared.citations import (
+    CitationRegistry,
+    CitationSourceType,
+)
+from app.agents.chat.multi_agent_chat.shared.document_render import (
+    RenderableDocument,
+    RenderablePassage,
+    render_document,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def _document(
+    document_id: int,
+    title: str,
+    chunk_ids: list[int],
+    *,
+    source: str | None = None,
+) -> RenderableDocument:
+    return RenderableDocument(
+        title=title,
+        source=source,
+        passages=[
+            RenderablePassage(
+                content=f"text {cid}",
+                locator={"document_id": document_id, "chunk_id": cid},
+            )
+            for cid in chunk_ids
+        ],
+    )
+
+
+def test_returns_none_when_no_passages() -> None:
+    registry = CitationRegistry()
+
+    assert (
+        render_document(_document(1, "Empty", []), view="excerpt", registry=registry)
+        is None
+    )
+
+
+def test_excerpt_open_and_close_tags() -> None:
+    registry = CitationRegistry()
+
+    block = render_document(
+        _document(1, "Q3 Launch Notes", [880], source="Slack · #launch"),
+        view="excerpt",
+        registry=registry,
+    )
+
+    assert block is not None
+    assert block.startswith(
+        '<document title="Q3 Launch Notes" source="Slack · #launch" view="excerpt">'
+    )
+    assert block.endswith("</document>")
+
+
+def test_full_view_renders_view_attribute() -> None:
+    registry = CitationRegistry()
+
+    block = render_document(_document(1, "Doc", [880]), view="full", registry=registry)
+
+    assert block is not None
+    assert '<document title="Doc" view="full">' in block
+
+
+def test_source_attribute_omitted_when_absent() -> None:
+    registry = CitationRegistry()
+
+    block = render_document(
+        _document(1, "Plain", [1]), view="excerpt", registry=registry
+    )
+
+    assert block is not None
+    assert block.startswith('<document title="Plain" view="excerpt">')
+
+
+def test_registers_passages_with_chunk_locators() -> None:
+    registry = CitationRegistry()
+
+    render_document(
+        _document(1, "Doc", [880], source="Slack"),
+        view="excerpt",
+        registry=registry,
+    )
+
+    entry = registry.resolve(1)
+    assert entry is not None
+    assert entry.source_type is CitationSourceType.KB_CHUNK
+    assert entry.locator == {"document_id": 1, "chunk_id": 880}
+    assert entry.display == {"title": "Doc", "source": "Slack"}
+
+
+def test_passages_get_monotonic_labels() -> None:
+    registry = CitationRegistry()
+
+    block = render_document(
+        _document(1, "Doc", [880, 881]), view="excerpt", registry=registry
+    )
+
+    assert block is not None
+    assert "  [1] text 880" in block
+    assert "  [2] text 881" in block
+
+
+def test_multiline_passage_indents_under_label() -> None:
+    registry = CitationRegistry()
+    document = RenderableDocument(
+        title="Doc",
+        passages=[
+            RenderablePassage(
+                content="line one\nline two",
+                locator={"document_id": 1, "chunk_id": 5},
+            )
+        ],
+    )
+
+    block = render_document(document, view="excerpt", registry=registry)
+
+    assert block is not None
+    assert "  [1] line one\n      line two" in block
+
+
+def test_attribute_values_are_escaped() -> None:
+    registry = CitationRegistry()
+
+    block = render_document(
+        _document(1, 'A & B <c> "d"', [1], source="x & y"),
+        view="excerpt",
+        registry=registry,
+    )
+
+    assert block is not None
+    assert 'title="A &amp; B &lt;c&gt; &quot;d&quot;"' in block
+    assert 'source="x &amp; y"' in block
+
+
+def test_same_passage_reuses_label_across_calls() -> None:
+    registry = CitationRegistry()
+    document = _document(1, "Doc", [880])
+
+    render_document(document, view="excerpt", registry=registry)
+    render_document(document, view="full", registry=registry)
+
+    assert registry.next_n == 2

surfsense_backend/tests/unit/agents/multi_agent_chat/shared/document_render/test_search_context.py

@@ -0,0 +1,94 @@
+"""Tests for the ``<retrieved_context>`` wrapper around excerpt documents."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.agents.chat.multi_agent_chat.shared.citations import CitationRegistry
+from app.agents.chat.multi_agent_chat.shared.document_render import (
+    RenderableDocument,
+    RenderablePassage,
+    render_search_context,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def _document(
+    document_id: int,
+    title: str,
+    chunk_ids: list[int],
+    *,
+    source: str | None = None,
+) -> RenderableDocument:
+    return RenderableDocument(
+        title=title,
+        source=source,
+        passages=[
+            RenderablePassage(
+                content=f"text {cid}",
+                locator={"document_id": document_id, "chunk_id": cid},
+            )
+            for cid in chunk_ids
+        ],
+    )
+
+
+def test_returns_none_when_nothing_to_show() -> None:
+    registry = CitationRegistry()
+
+    assert render_search_context([], registry) is None
+    assert render_search_context([_document(1, "Empty", [])], registry) is None
+
+
+def test_assigns_monotonic_labels_across_documents() -> None:
+    registry = CitationRegistry()
+
+    block = render_search_context(
+        [
+            _document(1, "Q3 Launch Notes", [880, 881], source="Slack"),
+            _document(2, "Timeline", [12], source="Notion"),
+        ],
+        registry,
+    )
+
+    assert block is not None
+    assert "[1] text 880" in block
+    assert "[2] text 881" in block
+    assert "[3] text 12" in block
+
+
+def test_wraps_in_retrieved_context_and_teaches_excerpt_and_citation() -> None:
+    registry = CitationRegistry()
+
+    block = render_search_context([_document(1, "Doc", [1])], registry)
+
+    assert block is not None
+    assert block.startswith("<retrieved_context>")
+    assert block.endswith("</retrieved_context>")
+    assert "excerpt view" in block
+    assert "Cite a chunk with its [n]." in block
+
+
+def test_documents_render_as_excerpt_blocks() -> None:
+    registry = CitationRegistry()
+
+    block = render_search_context(
+        [_document(1, "Q3", [1], source="Slack · #launch")], registry
+    )
+
+    assert block is not None
+    assert '<document title="Q3" source="Slack · #launch" view="excerpt">' in block
+    assert "</document>" in block
+
+
+def test_same_passage_reuses_label_across_calls() -> None:
+    registry = CitationRegistry()
+    document = _document(1, "Doc", [880])
+
+    render_search_context([document], registry)
+    block = render_search_context([document], registry)
+
+    assert block is not None
+    assert "[1] text 880" in block
+    assert registry.next_n == 2

surfsense_backend/tests/unit/agents/multi_agent_chat/shared/document_render/test_source_label.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import pytest
 
-from app.agents.chat.multi_agent_chat.shared.retrieval.source_label import source_label
+from app.agents.chat.multi_agent_chat.shared.document_render import source_label
 
 pytestmark = pytest.mark.unit
 

surfsense_backend/tests/unit/agents/multi_agent_chat/shared/document_render/test_web_results.py

@@ -0,0 +1,82 @@
+"""Tests for the ``<web_results>`` wrapper around web-result excerpt documents."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.agents.chat.multi_agent_chat.shared.citations import (
+    CitationRegistry,
+    CitationSourceType,
+)
+from app.agents.chat.multi_agent_chat.shared.document_render import (
+    RenderableDocument,
+    RenderablePassage,
+    render_web_results,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def _web_doc(url: str, title: str, content: str) -> RenderableDocument:
+    return RenderableDocument(
+        title=title,
+        source=f"Web · {url.split('//', 1)[-1].split('/', 1)[0]}",
+        passages=[
+            RenderablePassage(
+                content=content,
+                locator={"url": url},
+                source_type=CitationSourceType.WEB_RESULT,
+            )
+        ],
+    )
+
+
+def test_returns_none_when_nothing_to_show() -> None:
+    registry = CitationRegistry()
+
+    assert render_web_results([], registry) is None
+
+
+def test_wraps_in_web_results_container() -> None:
+    registry = CitationRegistry()
+
+    block = render_web_results(
+        [_web_doc("https://example.com/a", "Example", "the answer is 42")],
+        registry,
+    )
+
+    assert block is not None
+    assert block.startswith("<web_results>")
+    assert block.endswith("</web_results>")
+    assert "cite a result with its [n]" in block
+    assert '<document title="Example" source="Web · example.com" view="excerpt">' in block
+    assert "[1] the answer is 42" in block
+
+
+def test_registers_each_result_as_web_result_with_url_locator() -> None:
+    registry = CitationRegistry()
+
+    render_web_results(
+        [
+            _web_doc("https://a.com/x", "A", "alpha"),
+            _web_doc("https://b.com/y", "B", "beta"),
+        ],
+        registry,
+    )
+
+    first = registry.resolve(1)
+    second = registry.resolve(2)
+    assert first is not None and second is not None
+    assert first.source_type is CitationSourceType.WEB_RESULT
+    assert first.locator == {"url": "https://a.com/x"}
+    assert second.locator == {"url": "https://b.com/y"}
+
+
+def test_same_url_reuses_label_across_calls() -> None:
+    registry = CitationRegistry()
+    doc = _web_doc("https://example.com/a", "Example", "stable fact")
+
+    render_web_results([doc], registry)
+    render_web_results([doc], registry)
+
+    assert registry.next_n == 2

surfsense_backend/tests/unit/agents/multi_agent_chat/shared/retrieval/test_adapter.py

@@ -1,11 +1,11 @@
-"""Tests for mapping a DocumentHit to a renderable RetrievedDocument."""
+"""Tests for mapping a DocumentHit to a renderable document."""
 
 from __future__ import annotations
 
 import pytest
 
 from app.agents.chat.multi_agent_chat.shared.retrieval.adapter import (
-    to_retrieved_document,
+    to_renderable_document,
 )
 from app.agents.chat.multi_agent_chat.shared.retrieval.models import (
     ChunkHit,
@@ -15,7 +15,7 @@ from app.agents.chat.multi_agent_chat.shared.retrieval.models import (
 pytestmark = pytest.mark.unit
 
 
-def test_maps_identity_source_label_and_passages() -> None:
+def test_maps_identity_source_and_passages() -> None:
     hit = DocumentHit(
         document_id=42,
         title="Q3 Launch Notes",
@@ -28,13 +28,14 @@ def test_maps_identity_source_label_and_passages() -> None:
         ],
     )
 
-    document = to_retrieved_document(hit)
+    document = to_renderable_document(hit)
 
-    assert document.document_id == 42
     assert document.title == "Q3 Launch Notes"
-    assert document.source_label == "Slack"
-    assert [(p.chunk_id, p.content) for p in document.passages] == [(880, "a"), (881, "b")]
-    assert all(p.document_id == 42 for p in document.passages)
+    assert document.source == "Slack"
+    assert [
+        (p.locator["chunk_id"], p.content) for p in document.passages
+    ] == [(880, "a"), (881, "b")]
+    assert all(p.locator["document_id"] == 42 for p in document.passages)
 
 
 def test_document_with_no_chunks_maps_to_no_passages() -> None:
@@ -47,4 +48,4 @@ def test_document_with_no_chunks_maps_to_no_passages() -> None:
         chunks=[],
     )
 
-    assert to_retrieved_document(hit).passages == []
+    assert to_renderable_document(hit).passages == []

surfsense_backend/tests/unit/agents/multi_agent_chat/shared/retrieved_context/test_renderer.py

@@ -1,144 +0,0 @@
-"""Tests for the <retrieved_context> renderer and its citation registration."""
-
-from __future__ import annotations
-
-import pytest
-
-from app.agents.chat.multi_agent_chat.shared.citations import (
-    CitationRegistry,
-    CitationSourceType,
-)
-from app.agents.chat.multi_agent_chat.shared.retrieved_context import (
-    RetrievedDocument,
-    RetrievedPassage,
-    render_retrieved_context,
-)
-
-pytestmark = pytest.mark.unit
-
-
-def _document(
-    document_id: int,
-    title: str,
-    chunk_ids: list[int],
-    *,
-    source_label: str | None = None,
-) -> RetrievedDocument:
-    return RetrievedDocument(
-        document_id=document_id,
-        title=title,
-        source_label=source_label,
-        passages=[
-            RetrievedPassage(document_id=document_id, chunk_id=cid, content=f"text {cid}")
-            for cid in chunk_ids
-        ],
-    )
-
-
-def test_returns_none_when_nothing_to_show() -> None:
-    registry = CitationRegistry()
-
-    assert render_retrieved_context([], registry) is None
-    assert render_retrieved_context([_document(1, "Empty", [])], registry) is None
-
-
-def test_assigns_monotonic_labels_across_documents() -> None:
-    registry = CitationRegistry()
-
-    block = render_retrieved_context(
-        [
-            _document(1, "Q3 Launch Notes", [880, 881], source_label="Slack"),
-            _document(2, "Timeline", [12], source_label="Notion"),
-        ],
-        registry,
-    )
-
-    assert block is not None
-    assert "[1] text 880" in block
-    assert "[2] text 881" in block
-    assert "[3] text 12" in block
-
-
-def test_registers_passages_with_chunk_locators() -> None:
-    registry = CitationRegistry()
-
-    render_retrieved_context([_document(1, "Doc", [880])], registry)
-
-    entry = registry.resolve(1)
-    assert entry is not None
-    assert entry.source_type is CitationSourceType.KB_CHUNK
-    assert entry.locator == {"document_id": 1, "chunk_id": 880}
-    assert entry.display["title"] == "Doc"
-
-
-def test_header_shows_source_when_present() -> None:
-    registry = CitationRegistry()
-
-    block = render_retrieved_context(
-        [
-            _document(1, "Q3", [1], source_label="Slack · #launch"),
-            _document(2, "Plan", [2]),
-        ],
-        registry,
-    )
-
-    assert block is not None
-    assert 'Document: "Q3"  (Slack · #launch)' in block
-    assert 'Document: "Plan"' in block
-
-
-def test_wraps_block_and_explains_chunk_vs_document() -> None:
-    registry = CitationRegistry()
-
-    block = render_retrieved_context([_document(1, "Doc", [1])], registry)
-
-    assert block is not None
-    assert block.startswith("<retrieved_context>")
-    assert block.endswith("</retrieved_context>")
-    assert "Cite a chunk with [n]." in block
-
-
-def test_multiline_passage_is_indented_under_label() -> None:
-    registry = CitationRegistry()
-    document = RetrievedDocument(
-        document_id=1,
-        title="Doc",
-        passages=[RetrievedPassage(document_id=1, chunk_id=5, content="line one\nline two")],
-    )
-
-    block = render_retrieved_context([document], registry)
-
-    assert block is not None
-    assert "  [1] line one\n      line two" in block
-
-
-def test_continuation_indent_tracks_label_width() -> None:
-    registry = CitationRegistry()
-    # Burn labels 1..9 so the tenth passage renders as [10] (a 7-char label).
-    documents = [_document(i, f"Doc {i}", [i]) for i in range(1, 10)]
-    documents.append(
-        RetrievedDocument(
-            document_id=10,
-            title="Doc 10",
-            passages=[
-                RetrievedPassage(document_id=10, chunk_id=10, content="line one\nline two")
-            ],
-        )
-    )
-
-    block = render_retrieved_context(documents, registry)
-
-    assert block is not None
-    assert "  [10] line one\n       line two" in block
-
-
-def test_same_passage_reuses_label_across_calls() -> None:
-    registry = CitationRegistry()
-    document = _document(1, "Doc", [880])
-
-    render_retrieved_context([document], registry)
-    block = render_retrieved_context([document], registry)
-
-    assert block is not None
-    assert "[1] text 880" in block
-    assert registry.next_n == 2

surfsense_backend/tests/unit/middleware/test_kb_postgres_read.py

@@ -0,0 +1,124 @@
+"""Unit tests for the KB read path: full-view render + anonymous-doc loading.
+
+DB-backed loads are exercised by the integration suite; here we lock the pure
+pieces — ``render_full_document`` and the anonymous-upload branch of
+``aload_document`` — which need no database.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from app.agents.chat.multi_agent_chat.shared.citations import (
+    CitationRegistry,
+    CitationSourceType,
+)
+from app.agents.chat.multi_agent_chat.shared.document_render import (
+    RenderableDocument,
+    RenderablePassage,
+)
+from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import (
+    KBPostgresBackend,
+    render_full_document,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def _backend(state: dict) -> KBPostgresBackend:
+    return KBPostgresBackend(search_space_id=1, runtime=SimpleNamespace(state=state))
+
+
+def test_render_full_document_uses_full_view_and_registers() -> None:
+    registry = CitationRegistry()
+    document = RenderableDocument(
+        title="Launch Notes",
+        source="Slack",
+        passages=[
+            RenderablePassage(
+                content="push to March 10",
+                locator={"document_id": 7, "chunk_id": 880},
+            ),
+        ],
+    )
+
+    rendered = render_full_document(document, registry)
+
+    assert '<document title="Launch Notes" source="Slack" view="full">' in rendered
+    assert "[1] push to March 10" in rendered
+    entry = registry.resolve(1)
+    assert entry is not None
+    assert entry.locator == {"document_id": 7, "chunk_id": 880}
+
+
+def test_render_full_document_reuses_search_label() -> None:
+    """A chunk already registered from search keeps its [n] on a later full read."""
+    registry = CitationRegistry()
+    n = registry.register(
+        CitationSourceType.KB_CHUNK,
+        {"document_id": 7, "chunk_id": 880},
+        {"title": "Launch Notes", "source": "Slack"},
+    )
+    document = RenderableDocument(
+        title="Launch Notes",
+        source="Slack",
+        passages=[
+            RenderablePassage(
+                content="new chunk",
+                locator={"document_id": 7, "chunk_id": 881},
+            ),
+            RenderablePassage(
+                content="push to March 10",
+                locator={"document_id": 7, "chunk_id": 880},
+            ),
+        ],
+    )
+
+    rendered = render_full_document(document, registry)
+
+    assert f"[{n}] push to March 10" in rendered
+    assert "[2] new chunk" in rendered
+
+
+def test_render_full_document_empty_falls_back_to_notice() -> None:
+    registry = CitationRegistry()
+    document = RenderableDocument(title="Empty", passages=[])
+
+    assert render_full_document(document, registry) == (
+        "(This document has no readable content.)"
+    )
+
+
+async def test_aload_document_anonymous_upload() -> None:
+    backend = _backend(
+        {
+            "kb_anon_doc": {
+                "path": "/anon_upload.md",
+                "title": "Quarterly Report",
+                "chunks": [
+                    {"chunk_id": -1, "content": "revenue grew"},
+                    {"chunk_id": -2, "content": "costs fell"},
+                ],
+            }
+        }
+    )
+
+    loaded = await backend.aload_document("/anon_upload.md")
+
+    assert loaded is not None
+    document, doc_id = loaded
+    assert doc_id is None
+    assert document.title == "Quarterly Report"
+    assert [p.locator["chunk_id"] for p in document.passages] == [-1, -2]
+    assert all(p.locator["document_id"] == -1 for p in document.passages)
+    assert all(
+        p.source_type is CitationSourceType.ANON_CHUNK for p in document.passages
+    )
+
+
+async def test_aload_document_unknown_path_returns_none() -> None:
+    backend = _backend({})
+
+    assert await backend.aload_document("/not/under/documents.md") is None