apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-26 23:51:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1619 from CREDO23/feat-run-citations

Browse source 
[Feat] Cite scraper runs as verifiable sources in chat
This commit is contained in:
Rohan Verma 2026-07-22 14:53:45 -07:00 • committed by GitHub
commit ca4f231577
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 339 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

35
surfsense_web/lib/citations/citation-parser.test.ts Normal file
View file

@ -0,0 +1,35 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { parseTextWithCitations } from "./citation-parser";
// Run with: pnpm exec tsx --test lib/citations/citation-parser.test.ts
const NO_URLS = new Map<string, string>();
test("parses a scraper-run handle into a run token", () => {
const segments = parseTextWithCitations(
"Price is $19 [citation:run_1b2c3d4e-5f60-7081-9abc-def012345678].",
NO_URLS
);
assert.deepEqual(segments, [
"Price is $19 ",
{ kind: "run", runId: "run_1b2c3d4e-5f60-7081-9abc-def012345678" },
".",
]);
});
test("run handles do not collide with numeric chunk citations", () => {
const segments = parseTextWithCitations(
"chunk [citation:42] vs run [citation:run_ab-12].",
NO_URLS
);
assert.deepEqual(segments, [
"chunk ",
{ kind: "chunk", chunkId: 42, isDocsChunk: false },
" vs run ",
{ kind: "run", runId: "run_ab-12" },
".",
]);
});

10
surfsense_web/lib/citations/citation-parser.ts
View file

@ -11,18 +11,20 @@ import { FENCED_OR_INLINE_CODE } from "@/lib/markdown/code-regions";
/**
* Matches `[citation:...]` with numeric IDs (incl. negative, doc- prefix,
* comma-separated), URL-based IDs from live web search, or `urlciteN`
* placeholders produced by `preprocessCitationMarkdown`.
* comma-separated), URL-based IDs from live web search, `urlciteN`
* placeholders produced by `preprocessCitationMarkdown`, or a scraper-run
* handle (`run_<uuid>`).
*
* Also matches Chinese brackets 【】 and zero-width spaces that LLMs
* sometimes emit.
*/
export const CITATION_REGEX =
/[[【]\u200B?citation:\s*(https?:\/\/[^\]】\u200B]+|urlcite\d+|(?:doc-)?-?\d+(?:\s*,\s*(?:doc-)?-?\d+)*)\s*\u200B?[\]】]/g;
/[[【]\u200B?citation:\s*(https?:\/\/[^\]】\u200B]+|urlcite\d+|run_[0-9a-fA-F-]+|(?:doc-)?-?\d+(?:\s*,\s*(?:doc-)?-?\d+)*)\s*\u200B?[\]】]/g;
/** A single parsed citation reference. */
export type CitationToken =
| { kind: "url"; url: string }
| { kind: "run"; runId: string }
| { kind: "chunk"; chunkId: number; isDocsChunk: boolean };
/** Output of `parseTextWithCitations` — interleaved text + citation tokens. */
@ -102,6 +104,8 @@ export function parseTextWithCitations(text: string, urlMap: CitationUrlMap): Pa
if (url) {
segments.push({ kind: "url", url });
}
} else if (captured.startsWith("run_")) {
segments.push({ kind: "run", runId: captured.trim() });
} else {
const rawIds = captured.split(",").map((s) => s.trim());
for (const rawId of rawIds) {