From 633f9359c201cbc5f4daec99d0dc9f44c0f4553a Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov <7889985+andreybavt@users.noreply.github.com> Date: Mon, 11 May 2026 17:25:46 +0200 Subject: [PATCH] feat: return sl search snippets --- packages/context/src/sl/ports.ts | 2 +- .../context/src/sl/sl-search.service.test.ts | 32 +++++++++++++++++++ packages/context/src/sl/sl-search.service.ts | 8 +++-- .../src/sl/sqlite-sl-sources-index.test.ts | 18 +++++++++-- .../context/src/sl/sqlite-sl-sources-index.ts | 22 ++++++++++--- 5 files changed, 73 insertions(+), 9 deletions(-) diff --git a/packages/context/src/sl/ports.ts b/packages/context/src/sl/ports.ts index d2426460..08e1ca6d 100644 --- a/packages/context/src/sl/ports.ts +++ b/packages/context/src/sl/ports.ts @@ -49,5 +49,5 @@ export interface SlSourcesIndexPort { queryText: string, limit: number, minRrfScore?: number, - ): Promise>; + ): Promise>; } diff --git a/packages/context/src/sl/sl-search.service.test.ts b/packages/context/src/sl/sl-search.service.test.ts index 1051eaeb..ffe27cbc 100644 --- a/packages/context/src/sl/sl-search.service.test.ts +++ b/packages/context/src/sl/sl-search.service.test.ts @@ -191,4 +191,36 @@ describe('SlSearchService', () => { expect(text).toContain('commonly joined to public.customers on customer_id'); expect(text).toContain('stale since 2026-05-01T00:00:00.000Z'); }); + + it('preserves FTS snippets returned by the source index', async () => { + const service = new SlSearchService( + { + maxBatchSize: 16, + computeEmbedding: vi.fn(async () => [1, 0]), + computeEmbeddingsBulk: vi.fn(), + }, + { + upsertSources: vi.fn(), + getExistingSearchTexts: vi.fn(), + deleteStale: vi.fn(), + deleteByConnection: vi.fn(), + deleteByConnectionAndName: vi.fn(), + search: vi.fn(async () => [ + { + sourceName: 'orders', + rrfScore: 0.75, + snippet: 'usage: paid order lifecycle', + }, + ]), + }, + ); + + await expect(service.search('warehouse', 'order lifecycle', 10)).resolves.toEqual([ + { + sourceName: 'orders', + score: 0.75, + snippet: 'usage: paid order lifecycle', + }, + ]); + }); }); diff --git a/packages/context/src/sl/sl-search.service.ts b/packages/context/src/sl/sl-search.service.ts index e2a42906..68ae1557 100644 --- a/packages/context/src/sl/sl-search.service.ts +++ b/packages/context/src/sl/sl-search.service.ts @@ -168,7 +168,7 @@ export class SlSearchService { query: string, limit = 15, minRrfScore = 0, - ): Promise> { + ): Promise> { let queryEmbedding: number[] | null = null; try { queryEmbedding = await this.embeddingService.computeEmbedding(query); @@ -179,7 +179,11 @@ export class SlSearchService { } const results = await this.slSourcesRepository.search(connectionId, queryEmbedding, query, limit, minRrfScore); - return results.map((r) => ({ sourceName: r.sourceName, score: r.rrfScore })); + return results.map((result) => ({ + sourceName: result.sourceName, + score: result.rrfScore, + ...(result.snippet ? { snippet: result.snippet } : {}), + })); } buildSearchText(source: SemanticLayerSource, priority: string[] = DEFAULT_PRIORITY): string { diff --git a/packages/context/src/sl/sqlite-sl-sources-index.test.ts b/packages/context/src/sl/sqlite-sl-sources-index.test.ts index 01f1be37..18258000 100644 --- a/packages/context/src/sl/sqlite-sl-sources-index.test.ts +++ b/packages/context/src/sl/sqlite-sl-sources-index.test.ts @@ -17,7 +17,7 @@ describe('SqliteSlSourcesIndex', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('creates SQLite tables and searches indexed source text', async () => { + it('creates SQLite tables and searches indexed source text with FTS snippets', async () => { const index = new SqliteSlSourcesIndex({ dbPath }); await index.upsertSources('warehouse', [ @@ -34,10 +34,24 @@ describe('SqliteSlSourcesIndex', () => { ]); await expect(access(dbPath)).resolves.toBeUndefined(); - expect(await index.search('warehouse', null, 'gross revenue', 10)).toEqual([ + + const directResults = await index.search('warehouse', null, 'gross revenue', 10); + expect(directResults).toEqual([ expect.objectContaining({ sourceName: 'orders', rrfScore: expect.any(Number), + snippet: expect.stringContaining(''), + }), + ]); + expect(directResults[0]?.snippet).toContain('revenue'); + + const lexicalCandidates = await index.searchLexicalCandidates({ queryText: 'gross revenue', limit: 10 }); + expect(lexicalCandidates).toEqual([ + expect.objectContaining({ + id: 'warehouse/orders', + connectionId: 'warehouse', + sourceName: 'orders', + snippet: expect.stringContaining(''), }), ]); }); diff --git a/packages/context/src/sl/sqlite-sl-sources-index.ts b/packages/context/src/sl/sqlite-sl-sources-index.ts index a5000976..f53c8eef 100644 --- a/packages/context/src/sl/sqlite-sl-sources-index.ts +++ b/packages/context/src/sl/sqlite-sl-sources-index.ts @@ -19,6 +19,7 @@ type SearchRow = { connection_id?: string; source_name: string; rank: number; + snippet?: string | null; }; export interface SlSqliteLaneCandidate { @@ -27,6 +28,7 @@ export interface SlSqliteLaneCandidate { sourceName: string; rank: number; rawScore: number; + snippet?: string; } export interface SlSqliteDictionaryCandidate extends SlSqliteLaneCandidate { @@ -334,7 +336,11 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { const rows = this.db .prepare( ` - SELECT connection_id, source_name, bm25(local_sl_sources_fts) AS rank + SELECT + connection_id, + source_name, + bm25(local_sl_sources_fts) AS rank, + snippet(local_sl_sources_fts, 2, '', '', '...', 12) AS snippet FROM local_sl_sources_fts WHERE local_sl_sources_fts MATCH ? ${connectionPredicate} @@ -350,6 +356,7 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { sourceName: row.source_name, rank: index + 1, rawScore: Number(row.rank), + ...(typeof row.snippet === 'string' && row.snippet.length > 0 ? { snippet: row.snippet } : {}), })); } @@ -499,7 +506,7 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { queryText: string, limit: number, minRrfScore = 0, - ): Promise> { + ): Promise> { const ftsQuery = normalizeFtsQuery(queryText); if (!ftsQuery) { return []; @@ -508,7 +515,10 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { const rows = this.db .prepare( ` - SELECT source_name, bm25(local_sl_sources_fts) AS rank + SELECT + source_name, + bm25(local_sl_sources_fts) AS rank, + snippet(local_sl_sources_fts, 2, '', '', '...', 12) AS snippet FROM local_sl_sources_fts WHERE connection_id = ? AND local_sl_sources_fts MATCH ? @@ -519,7 +529,11 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { .all(connectionId, ftsQuery, Math.max(1, limit)) as SearchRow[]; return rows - .map((row) => ({ sourceName: row.source_name, rrfScore: scoreFromRank(row.rank) })) + .map((row) => ({ + sourceName: row.source_name, + rrfScore: scoreFromRank(row.rank), + ...(typeof row.snippet === 'string' && row.snippet.length > 0 ? { snippet: row.snippet } : {}), + })) .filter((row) => row.rrfScore >= minRrfScore); }