Merge pull request #1619 from CREDO23/feat-run-citations

[Feat] Cite scraper runs as verifiable sources in chat

surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md

@@ -1,8 +1,9 @@
 <citations>
 Cite with one token: the bracket label `[n]`. Every citable result —
 prose from a `task` knowledge_base/research specialist (including the
-knowledge_base specialist's `[n]`-labelled workspace findings) — already
-carries `[n]` labels on a single shared count.
+knowledge_base specialist's `[n]`-labelled workspace findings) and
+scraper specialists' run-backed findings — already carries `[n]` labels
+on a single shared count.
 Those labels are the only citation you write; the server resolves each one
 back to its source after the turn.
 

surfsense_backend/app/agents/chat/multi_agent_chat/shared/citations/markers.py

@@ -1,10 +1,11 @@
 """Map a registered citation to the frontend ``[citation:<payload>]`` payload.
 
 The citation renderer understands a chunk id (``42``), a negative chunk id for
-anonymous uploads (``-3``), and a URL. This is the seam that turns a server-side
-source into one the renderer can resolve; it grows as more source kinds become
-renderable. Kinds with no renderable form yet return ``None`` so the marker is
-dropped rather than emitted broken.
+anonymous uploads (``-3``), a URL, and a scraper-run handle (``run_<uuid>``).
+This is the seam that turns a server-side source into one the renderer can
+resolve; it grows as more source kinds become renderable. Kinds with no
+renderable form yet return ``None`` so the marker is dropped rather than
+emitted broken.
 """
 
 from __future__ import annotations
@@ -22,6 +23,9 @@ def to_frontend_payload(entry: CitationEntry) -> str | None:
         case CitationSourceType.WEB_RESULT:
             url = locator.get("url")
             return url or None
+        case CitationSourceType.RUN:
+            run_id = locator.get("run_id")
+            return str(run_id) if run_id else None
         case _:
             # Connector items and chat turns have no client-side renderer yet
             # (the frontend resolves only chunk ids and URLs), so they stay

surfsense_backend/app/agents/chat/multi_agent_chat/shared/citations/models.py

@@ -17,6 +17,7 @@ class CitationSourceType(StrEnum):
     WEB_RESULT = "web_result"
     CHAT_TURN = "chat_turn"
     ANON_CHUNK = "anon_chunk"
+    RUN = "run"
 
 
 class CitationEntry(BaseModel):

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/middleware_stack.py

@@ -15,6 +15,9 @@ from __future__ import annotations
 from typing import Any
 
 from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags
+from app.agents.chat.multi_agent_chat.shared.middleware.citation_state import (
+    build_citation_state_mw,
+)
 from app.agents.chat.multi_agent_chat.shared.middleware.resilience import (
     ResilienceMiddlewares,
 )
@@ -45,6 +48,8 @@ def build_subagent_middleware_stack(
     return {
         "todos": build_todos_mw(),
         "permission": permission,
+        # Declares the citation_registry channel so run [n]s merge up from tools.
+        "citation": build_citation_state_mw(),
         "retry": resilience.retry,
         "fallback": resilience.fallback,
         "model_call_limit": resilience.model_call_limit,

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md

@@ -5,3 +5,4 @@ Rules (universal):
 - `status=blocked` due to missing required inputs -> `missing_fields` must be non-null.
 - `assumptions`: any inferences you made about the user's intent; `null` when no inferences were needed.
 - The `evidence` object's fields are documented in your route-specific `<output_contract>` above; never invent fields the tool did not return.
+- When a finding is drawn from a scraper run, append that run's `[n]` (the tool result states `Cite this scraper run as [n]`) to the finding text so the citation survives into the final answer. Copy the label exactly; never invent one.

surfsense_backend/app/capabilities/core/access/agent.py

@@ -11,9 +11,13 @@ subagent can follow a truncation reference without extra wiring.
 
 from __future__ import annotations
 
+import json
 import time
 
+from langchain.tools import ToolRuntime
+from langchain_core.messages import ToolMessage
 from langchain_core.tools import BaseTool, StructuredTool
+from langgraph.types import Command
 
 from app.capabilities.core.billing import charge_capability, gate_capability
 from app.capabilities.core.progress import progress_scope
@@ -64,7 +68,7 @@ def _capability_tool(capability: Capability, workspace_id: int) -> BaseTool:
     executor = capability.executor
     name = capability.name
 
-    async def _run(**kwargs: object) -> dict | str:
+    async def _run(runtime: ToolRuntime, **kwargs: object) -> dict | str | Command:
         payload = input_model(**kwargs)
         input_dump = payload.model_dump(exclude_none=True)
         thread_id = _current_thread_id()
@@ -119,13 +123,42 @@ def _capability_tool(capability: Capability, workspace_id: int) -> BaseTool:
                     progress=reporter.coarse,
                 )
 
+        # No stored run to cite: keep the legacy return shape, no citation.
+        if run_id is None:
+            if serialized.char_count <= RUN_OUTPUT_CHAR_CAP:
+                return output.model_dump(exclude_none=True)
+            return _build_preview(serialized, run_id)
+
+        run_external_id = f"run_{run_id}"
         if serialized.char_count <= RUN_OUTPUT_CHAR_CAP:
             dump = output.model_dump(exclude_none=True)
-            if run_id is not None:
-                dump["run_id"] = f"run_{run_id}"
-            return dump
+            dump["run_id"] = run_external_id
+            content = json.dumps(dump, ensure_ascii=False, default=str)
+        else:
+            content = _build_preview(serialized, run_id)
 
-        return _build_preview(serialized, run_id)
+        # Deferred import: the citation spine imports from here; lazy avoids a cycle.
+        from app.agents.chat.multi_agent_chat.shared.citations import load_registry
+        from app.capabilities.core.access.run_citation import attach_run_citation
+
+        registry = load_registry(getattr(runtime, "state", None))
+        _, label = attach_run_citation(
+            registry, run_external_id=run_external_id, capability=name
+        )
+        return Command(
+            update={
+                "messages": [
+                    ToolMessage(
+                        content=content + label,
+                        tool_call_id=runtime.tool_call_id,
+                    )
+                ],
+                "citation_registry": registry,
+            }
+        )
+
+    # Un-stringify for StructuredTool's signature-based runtime injection.
+    _run.__annotations__["runtime"] = ToolRuntime
 
     return StructuredTool.from_function(
         coroutine=_run,

surfsense_backend/app/capabilities/core/access/run_citation.py

@@ -0,0 +1,23 @@
+"""Register a recorded scraper run as a citable ``[n]``."""
+
+from __future__ import annotations
+
+from app.agents.chat.multi_agent_chat.shared.citations import (
+    CitationRegistry,
+    CitationSourceType,
+)
+
+
+def attach_run_citation(
+    registry: CitationRegistry,
+    *,
+    run_external_id: str,
+    capability: str,
+) -> tuple[int, str]:
+    """Register the ``run_<uuid>`` handle; return its ``[n]`` and the label line."""
+    n = registry.register(
+        CitationSourceType.RUN,
+        {"run_id": run_external_id},
+        {"capability": capability},
+    )
+    return n, f"\n\nCite this scraper run as [{n}] after any claim drawn from its data."

surfsense_backend/tests/unit/agents/multi_agent_chat/shared/citations/test_markers.py

@@ -37,6 +37,18 @@ def test_web_result_maps_to_url() -> None:
     assert to_frontend_payload(entry) == "https://example.com/a"
 
 
+def test_run_maps_to_run_handle() -> None:
+    entry = _entry(CitationSourceType.RUN, {"run_id": "run_abc-123"})
+
+    assert to_frontend_payload(entry) == "run_abc-123"
+
+
+def test_run_without_handle_is_dropped() -> None:
+    entry = _entry(CitationSourceType.RUN, {})
+
+    assert to_frontend_payload(entry) is None
+
+
 def test_not_yet_renderable_kind_is_dropped() -> None:
     entry = _entry(CitationSourceType.CHAT_TURN, {"thread_id": 1, "turn": 2})
 

surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py

@@ -69,6 +69,12 @@ def _verb_tool(tools, name: str):
     return next(t for t in tools if t.name == name)
 
 
+def _invoke(tool, text: str, *, state=None):
+    """Call the coroutine with a stand-in runtime (ToolNode injects it in prod)."""
+    runtime = SimpleNamespace(state=state or {}, tool_call_id="tc_1", context=None)
+    return tool.coroutine(runtime, text=text)
+
+
 async def test_registry_becomes_one_tool_per_verb_plus_readers(isolate):
     caps = [
         _capability(name="web.scrape", output=_EchoOutput(echoed="a")),
@@ -104,20 +110,51 @@ async def test_tool_runs_executor_and_returns_serialized_output(isolate):
     tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap])
     tool = _verb_tool(tools, "web_scrape")
 
-    result = await tool.ainvoke({"text": "ping"})
+    result = await _invoke(tool, "ping")
 
     # Fake session makes record_run fail -> no run_id key, plain serialized output.
     assert result == {"echoed": "hi there"}
     assert cap.executor.seen.text == "ping"
 
 
+async def test_tool_registers_run_citation_when_stored(isolate, monkeypatch):
+    from langgraph.types import Command
+
+    cap = _capability(name="web.scrape", output=_EchoOutput(echoed="hi"))
+    monkeypatch.setattr(isolate.module, "record_run", AsyncMock(return_value="abc-123"))
+    tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap])
+    tool = _verb_tool(tools, "web_scrape")
+
+    result = await _invoke(tool, "ping")
+
+    assert isinstance(result, Command)
+    registry = result.update["citation_registry"]
+    entry = registry.resolve(1)
+    assert entry is not None
+    assert entry.locator["run_id"] == "run_abc-123"
+    message = result.update["messages"][0]
+    assert "[1]" in message.content
+    assert "run_abc-123" in message.content
+
+
+async def test_runtime_survives_langchain_arg_parsing(isolate):
+    """runtime must survive langchain arg parsing (else ToolNode drops it)."""
+    cap = _capability(name="web.scrape", output=_EchoOutput(echoed="hi"))
+    tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap])
+    tool = _verb_tool(tools, "web_scrape")
+
+    parsed = tool._parse_input({"text": "x", "runtime": "RT"}, "tc_1")
+
+    assert parsed["runtime"] == "RT"
+
+
 async def test_tool_charges_owner(isolate):
     output = _EchoOutput(echoed="hi")
     cap = _capability(name="web.scrape", output=output)
     tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap])
     tool = _verb_tool(tools, "web_scrape")
 
-    await tool.ainvoke({"text": "ping"})
+    await _invoke(tool, "ping")
 
     isolate.charge.assert_awaited_once()
     (charged_output, unit, ctx), _ = isolate.charge.call_args
@@ -136,7 +173,7 @@ async def test_over_budget_returns_friendly_message(isolate):
     tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap])
     tool = _verb_tool(tools, "web_scrape")
 
-    result = await tool.ainvoke({"text": "ping"})
+    result = await _invoke(tool, "ping")
 
     assert isinstance(result, str)
     assert "credit" in result.lower()

surfsense_backend/tests/unit/capabilities/access/test_run_citation.py

@@ -0,0 +1,43 @@
+"""Unit tests for registering a scraper run as a citation."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.agents.chat.multi_agent_chat.shared.citations import (
+    CitationRegistry,
+    CitationSourceType,
+    to_frontend_payload,
+)
+from app.capabilities.core.access.run_citation import attach_run_citation
+
+pytestmark = pytest.mark.unit
+
+
+def test_attaches_run_and_returns_label_with_ordinal() -> None:
+    registry = CitationRegistry()
+
+    n, label = attach_run_citation(
+        registry, run_external_id="run_abc-123", capability="walmart.scrape"
+    )
+
+    assert n == 1
+    assert f"[{n}]" in label
+    entry = registry.resolve(n)
+    assert entry is not None
+    assert entry.source_type is CitationSourceType.RUN
+    assert to_frontend_payload(entry) == "run_abc-123"
+
+
+def test_same_run_dedups_to_one_label() -> None:
+    registry = CitationRegistry()
+
+    first, _ = attach_run_citation(
+        registry, run_external_id="run_x", capability="walmart.scrape"
+    )
+    again, _ = attach_run_citation(
+        registry, run_external_id="run_x", capability="walmart.reviews"
+    )
+
+    assert first == again
+    assert len(registry.by_n) == 1