From d1b59364412b70ebec1f9b0d7180a5b518ac63bc Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Wed, 13 May 2026 19:32:49 +0200 Subject: [PATCH 1/2] feat(cli): add text ingest command (#72) --- packages/cli/src/cli-program.ts | 4 + packages/cli/src/cli-runtime.ts | 2 + packages/cli/src/commands/ingest-commands.ts | 31 +- packages/cli/src/context-build-view.test.ts | 24 ++ packages/cli/src/context-build-view.ts | 48 ++- packages/cli/src/index.test.ts | 59 ++++ packages/cli/src/text-ingest.test.ts | 339 ++++++++++++++++++ packages/cli/src/text-ingest.ts | 354 +++++++++++++++++++ 8 files changed, 850 insertions(+), 11 deletions(-) create mode 100644 packages/cli/src/text-ingest.test.ts create mode 100644 packages/cli/src/text-ingest.ts diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index efe2e5bb..c1935495 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -316,6 +316,10 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command { registerIngestCommands(program, context, { runIngestWithProgress: async (ingestArgs, ingestIo, ingestDeps, defaultRunIngest) => await (ingestDeps.ingest ?? defaultRunIngest)(ingestArgs, ingestIo), + runTextIngest: async (textIngestArgs, ingestIo, ingestDeps) => { + const { runKtxTextIngest } = await import('./text-ingest.js'); + return await (ingestDeps.textIngest ?? runKtxTextIngest)(textIngestArgs, ingestIo); + }, }); registerScanCommands(program, context); registerWikiCommands(program, context); diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index 5e2430cf..2712558f 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -9,6 +9,7 @@ import type { KtxScanArgs } from './scan.js'; import type { KtxSetupArgs } from './setup.js'; import type { KtxSlArgs } from './sl.js'; import { profileMark, profileSpan } from './startup-profile.js'; +import type { KtxTextIngestArgs } from './text-ingest.js'; profileMark('module:cli-runtime'); @@ -30,6 +31,7 @@ export interface KtxCliDeps { connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise; doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise; ingest?: (args: KtxIngestArgs, io: KtxCliIo) => Promise; + textIngest?: (args: KtxTextIngestArgs, io: KtxCliIo) => Promise; runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise; scan?: (args: KtxScanArgs, io: KtxCliIo) => Promise; knowledge?: (args: KtxKnowledgeArgs, io: KtxCliIo) => Promise; diff --git a/packages/cli/src/commands/ingest-commands.ts b/packages/cli/src/commands/ingest-commands.ts index 5ad357e1..952b6aa0 100644 --- a/packages/cli/src/commands/ingest-commands.ts +++ b/packages/cli/src/commands/ingest-commands.ts @@ -1,10 +1,11 @@ import { resolve } from 'node:path'; import { type Command, Option } from '@commander-js/extra-typings'; -import { type KtxCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js'; +import { collectOption, type KtxCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js'; import type { KtxCliDeps, KtxCliIo } from '../index.js'; import type { KtxIngestArgs, KtxIngestOutputMode } from '../ingest.js'; import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; import { profileMark } from '../startup-profile.js'; +import type { KtxTextIngestArgs } from '../text-ingest.js'; profileMark('module:commands/ingest-commands'); @@ -15,6 +16,7 @@ interface IngestCommandOptions { deps: KtxCliDeps, defaultRunIngest: (args: KtxIngestArgs, io: KtxCliIo) => Promise, ) => Promise; + runTextIngest: (args: KtxTextIngestArgs, io: KtxCliIo, deps: KtxCliDeps) => Promise; } function outputMode(options: OutputModeOptions): KtxIngestOutputMode { @@ -101,6 +103,33 @@ export function registerIngestCommands( ); }); + ingest + .command('text') + .description('Ingest free-form text artifacts into KTX memory') + .argument('[files...]', 'Files to ingest; use - to read one item from stdin') + .option('--text ', 'Text content to ingest; repeat for a batch', collectOption, []) + .option('--connection-id ', 'Optional KTX connection id for semantic-layer capture') + .option('--user-id ', 'Memory user id for capture attribution', 'local-cli') + .option('--json', 'Print JSON output') + .option('--fail-fast', 'Stop after the first failed text item', false) + .action(async (files: string[], options, command) => { + context.setExitCode( + await commandOptions.runTextIngest( + { + projectDir: resolveCommandProjectDir(command), + texts: options.text, + files, + ...(options.connectionId ? { connectionId: options.connectionId } : {}), + userId: options.userId, + json: options.json === true, + failFast: options.failFast === true, + }, + context.io, + context.deps, + ), + ); + }); + ingest .command('status') .description('Print status for the latest or selected stored local ingest run or report file') diff --git a/packages/cli/src/context-build-view.test.ts b/packages/cli/src/context-build-view.test.ts index c8dc5130..db172484 100644 --- a/packages/cli/src/context-build-view.test.ts +++ b/packages/cli/src/context-build-view.test.ts @@ -158,6 +158,30 @@ describe('renderContextBuildView', () => { expect(output).toContain('dbt-main'); }); + it('supports text ingest labels while preserving the shared compact progress view', () => { + const state = initViewState([ + { connectionId: 'text-1', driver: 'text', operation: 'source-ingest', debugCommand: '', steps: ['memory-update'] }, + { connectionId: 'schema.md', driver: 'text', operation: 'source-ingest', debugCommand: '', steps: ['memory-update'] }, + ]); + state.contextSources[0].status = 'running'; + state.contextSources[0].detailLine = 'capturing...'; + + const output = renderContextBuildView(state, { + styled: false, + title: 'Ingesting text memory', + contextGroupLabel: 'Texts', + sourceIngestRunningText: 'capturing...', + completedItemName: { singular: 'text', plural: 'texts' }, + }); + + expect(output).toContain('Ingesting text memory'); + expect(output).toContain('Texts:'); + expect(output).toContain('text-1'); + expect(output).toContain('schema.md'); + expect(output).toContain('capturing...'); + expect(output).not.toContain('Context sources:'); + }); + it('renders header with total elapsed time when set', () => { const state = initViewState([ { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] }, diff --git a/packages/cli/src/context-build-view.ts b/packages/cli/src/context-build-view.ts index 38f3d674..e1f43ead 100644 --- a/packages/cli/src/context-build-view.ts +++ b/packages/cli/src/context-build-view.ts @@ -65,6 +65,24 @@ export interface ContextBuildSourceProgressUpdate { summaryText?: string; } +interface CompletedItemName { + singular: string; + plural: string; +} + +interface ContextBuildRenderOptions { + styled?: boolean; + showHint?: boolean; + hintText?: string; + projectDir?: string; + title?: string; + primaryGroupLabel?: string; + contextGroupLabel?: string; + scanRunningText?: string; + sourceIngestRunningText?: string; + completedItemName?: CompletedItemName; +} + export interface ContextBuildDeps { executeTarget?: typeof executePublicIngestTarget; now?: () => number; @@ -148,7 +166,7 @@ function staleProgressText(target: ContextBuildTargetState, styled: boolean): st return styled ? dim(text) : text; } -function targetDetail(target: ContextBuildTargetState, styled: boolean): string { +function targetDetail(target: ContextBuildTargetState, styled: boolean, options: ContextBuildRenderOptions): string { if (target.status === 'done') { const parts: string[] = []; if (target.summaryText) parts.push(target.summaryText); @@ -162,7 +180,9 @@ function targetDetail(target: ContextBuildTargetState, styled: boolean): string if (target.status === 'running') { const percent = extractPercent(target.detailLine); const progressText = target.detailLine?.replace(/^\[\d+%\]\s*/, '') - ?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...'); + ?? (target.target.operation === 'scan' + ? (options.scanRunningText ?? 'scanning...') + : (options.sourceIngestRunningText ?? 'ingesting...')); const elapsed = target.elapsedMs > 0 ? `(${formatDuration(target.elapsedMs)})` : null; const parts: string[] = []; if (percent !== null) { @@ -182,8 +202,14 @@ function columnWidth(state: ContextBuildViewState): number { return Math.max(12, ...all.map((t) => t.target.connectionId.length)) + 2; } -function renderTargetLine(target: ContextBuildTargetState, frame: number, styled: boolean, width: number): string { - return ` ${statusIcon(target.status, frame, styled)} ${target.target.connectionId.padEnd(width)} ${targetDetail(target, styled)}`; +function renderTargetLine( + target: ContextBuildTargetState, + frame: number, + styled: boolean, + width: number, + options: ContextBuildRenderOptions, +): string { + return ` ${statusIcon(target.status, frame, styled)} ${target.target.connectionId.padEnd(width)} ${targetDetail(target, styled, options)}`; } function renderTargetGroup( @@ -192,9 +218,10 @@ function renderTargetGroup( frame: number, styled: boolean, width: number, + options: ContextBuildRenderOptions, ): string[] { if (targets.length === 0) return []; - return ['', ` ${label}:`, ...targets.map((t) => renderTargetLine(t, frame, styled, width))]; + return ['', ` ${label}:`, ...targets.map((t) => renderTargetLine(t, frame, styled, width, options))]; } function resumeCommand(projectDir?: string): string { @@ -203,7 +230,7 @@ function resumeCommand(projectDir?: string): string { export function renderContextBuildView( state: ContextBuildViewState, - options: { styled?: boolean; showHint?: boolean; hintText?: string; projectDir?: string } = {}, + options: ContextBuildRenderOptions = {}, ): string { const styled = options.styled ?? true; const width = columnWidth(state); @@ -213,7 +240,7 @@ export function renderContextBuildView( const hasActive = allTargets.some((t) => t.status === 'running' || t.status === 'queued'); const allDone = totalCount > 0 && !hasActive; - const headerParts = ['Building KTX context']; + const headerParts = [options.title ?? 'Building KTX context']; if (totalCount > 0) { const progressParts: string[] = [`${doneCount}/${totalCount}`]; if (state.totalElapsedMs > 0) progressParts.push(formatDuration(state.totalElapsedMs)); @@ -229,13 +256,14 @@ export function renderContextBuildView( header, separator, ...(options.projectDir ? [` Project: ${options.projectDir}`] : []), - ...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width), - ...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width), + ...renderTargetGroup(options.primaryGroupLabel ?? 'Primary sources', state.primarySources, state.frame, styled, width, options), + ...renderTargetGroup(options.contextGroupLabel ?? 'Context sources', state.contextSources, state.frame, styled, width, options), '', ]; if (allDone && state.totalElapsedMs > 0) { - const sourcesLabel = totalCount === 1 ? '1 source' : `${totalCount} sources`; + const itemName = options.completedItemName ?? { singular: 'source', plural: 'sources' }; + const sourcesLabel = totalCount === 1 ? `1 ${itemName.singular}` : `${totalCount} ${itemName.plural}`; const summary = ` Done in ${formatDuration(state.totalElapsedMs)} · ${sourcesLabel} processed`; lines.push(styled ? green(summary) : summary); lines.push(''); diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 83f11639..cd635d78 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -734,14 +734,73 @@ describe('runKtxCli', () => { expect(testIo.stdout()).toContain('Usage: ktx ingest [options] [command]'); expect(testIo.stdout()).toContain('Run or inspect local ingest memory-flow output'); expect(testIo.stdout()).toContain('run'); + expect(testIo.stdout()).toContain('text'); expect(testIo.stdout()).toContain('status'); expect(testIo.stdout()).toContain('watch'); expect(testIo.stdout()).toContain('replay'); + expect(testIo.stdout()).not.toContain('--manifest'); expect(testIo.stdout()).not.toContain('--all'); expect(testIo.stderr()).toBe(''); expect(ingest).not.toHaveBeenCalled(); }); + it('routes text memory ingest through Commander without exposing chat ids', async () => { + const textIngest = vi.fn(async () => 0); + const testIo = makeIo(); + + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'ingest', + 'text', + '--text', + 'Revenue means gross receipts.', + '--text', + 'Orders are completed purchases.', + '--connection-id', + 'warehouse', + '--user-id', + 'agent', + '--json', + '--fail-fast', + ], + testIo.io, + { textIngest }, + ), + ).resolves.toBe(0); + + expect(textIngest).toHaveBeenCalledWith( + { + projectDir: tempDir, + texts: ['Revenue means gross receipts.', 'Orders are completed purchases.'], + files: [], + connectionId: 'warehouse', + userId: 'agent', + json: true, + failFast: true, + }, + testIo.io, + ); + expect(testIo.stderr()).toBe(''); + }); + + it('documents text ingest inputs without a manifest option', async () => { + const textIngest = vi.fn(async () => 0); + const testIo = makeIo(); + + await expect(runKtxCli(['ingest', 'text', '--help'], testIo.io, { textIngest })).resolves.toBe(0); + + expect(testIo.stdout()).toContain('Usage: ktx ingest text [options] [files...]'); + expect(testIo.stdout()).toContain('--text '); + expect(testIo.stdout()).toContain('--connection-id '); + expect(testIo.stdout()).toContain('--user-id '); + expect(testIo.stdout()).toContain('--fail-fast'); + expect(testIo.stdout()).not.toContain('--manifest'); + expect(textIngest).not.toHaveBeenCalled(); + }); + it('routes ingest run at the top level and rejects removed dev ingest', async () => { const runIo = makeIo(); const devRunIo = makeIo(); diff --git a/packages/cli/src/text-ingest.test.ts b/packages/cli/src/text-ingest.test.ts new file mode 100644 index 00000000..55dbe9e3 --- /dev/null +++ b/packages/cli/src/text-ingest.test.ts @@ -0,0 +1,339 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { MemoryCaptureStatus } from '@ktx/context/memory'; +import type { KtxLocalProject } from '@ktx/context/project'; +import { runKtxTextIngest, type TextMemoryCapturePort } from './text-ingest.js'; + +function makeIo(options: { isTTY?: boolean } = {}) { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + isTTY: options.isTTY, + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function fakeCapture( + options: { + failRunIds?: Set; + missingStatusRunIds?: Set; + events?: string[]; + } = {}, +): TextMemoryCapturePort { + let next = 1; + return { + capture: vi.fn(async () => { + const runId = `run-${next++}`; + options.events?.push(`capture:${runId}`); + return { runId }; + }), + waitForRun: vi.fn(async (runId: string) => { + options.events?.push(`wait:${runId}`); + }), + status: vi.fn(async (runId: string) => { + options.events?.push(`status:${runId}`); + if (options.missingStatusRunIds?.has(runId)) { + return null; + } + if (options.failRunIds?.has(runId)) { + return { + runId, + status: 'error', + stage: 'capturing', + done: true, + captured: { wiki: [], sl: [], xrefs: [] }, + error: `${runId} failed`, + commitHash: null, + skillsLoaded: [], + signalDetected: false, + } satisfies MemoryCaptureStatus; + } + return { + runId, + status: 'done', + stage: 'capturing', + done: true, + captured: { wiki: [`wiki-${runId}`], sl: [`sl-${runId}`], xrefs: [] }, + error: null, + commitHash: `commit-${runId}`, + skillsLoaded: ['wiki_capture', 'sl'], + signalDetected: true, + } satisfies MemoryCaptureStatus; + }), + }; +} + +function fakeProject(projectDir = '/tmp/project'): KtxLocalProject { + return { projectDir } as KtxLocalProject; +} + +describe('runKtxTextIngest', () => { + it('captures repeated inline text sequentially with generated internal chat ids', async () => { + const io = makeIo(); + const events: string[] = []; + const capture = fakeCapture({ events }); + const createMemoryCapture = vi.fn(() => capture); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['Revenue means gross receipts.', 'Orders are completed purchases.'], + files: [], + userId: 'local-cli', + json: true, + failFast: false, + }, + io.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture, + now: () => 1_700_000_000_000, + }, + ), + ).resolves.toBe(0); + + expect(createMemoryCapture).toHaveBeenCalledWith({ projectDir: '/tmp/project' }); + expect(capture.capture).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + userId: 'local-cli', + chatId: 'cli-text-ingest-1700000000000-1', + userMessage: 'Ingest external text artifact "Revenue means gross receipts." into KTX memory.', + assistantMessage: 'Revenue means gross receipts.', + sourceType: 'external_ingest', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + chatId: 'cli-text-ingest-1700000000000-2', + userMessage: 'Ingest external text artifact "Orders are completed purchases." into KTX memory.', + assistantMessage: 'Orders are completed purchases.', + }), + ); + expect(capture.capture).not.toHaveBeenCalledWith(expect.objectContaining({ connectionId: expect.anything() })); + expect(events).toEqual(['capture:run-1', 'wait:run-1', 'status:run-1', 'capture:run-2', 'wait:run-2', 'status:run-2']); + expect(JSON.parse(io.stdout())).toMatchObject({ + status: 'done', + results: [ + { + label: '"Revenue means gross receipts."', + runId: 'run-1', + status: 'done', + captured: { wiki: ['wiki-run-1'], sl: ['sl-run-1'] }, + }, + { + label: '"Orders are completed purchases."', + runId: 'run-2', + status: 'done', + captured: { wiki: ['wiki-run-2'], sl: ['sl-run-2'] }, + }, + ], + }); + }); + + it('loads files and stdin as batch items and passes a global connection id', async () => { + const io = makeIo(); + const capture = fakeCapture(); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: [], + files: ['/tmp/docs/revenue.md', '-'], + connectionId: 'warehouse', + userId: 'agent', + json: false, + failFast: false, + }, + io.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => capture), + readFile: vi.fn(async (path) => `file:${path}`), + readStdin: vi.fn(async () => 'stdin content'), + now: () => 10, + }, + ), + ).resolves.toBe(0); + + expect(capture.capture).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + connectionId: 'warehouse', + userId: 'agent', + userMessage: 'Ingest external text artifact "revenue.md" into KTX memory.', + assistantMessage: 'file:/tmp/docs/revenue.md', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + connectionId: 'warehouse', + userMessage: 'Ingest external text artifact "stdin" into KTX memory.', + assistantMessage: 'stdin content', + }), + ); + expect(io.stdout()).toContain('Ingesting text memory'); + expect(io.stdout()).toContain('Texts:'); + expect(io.stdout()).toContain('revenue.md'); + expect(io.stdout()).toContain('stdin'); + }); + + it('uses bounded inline text previews as labels in plain output and capture metadata', async () => { + const io = makeIo(); + const capture = fakeCapture(); + const longText = `This inline note is intentionally long ${'x'.repeat(120)}`; + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['remember to call me Andrey', ' first line\n\tsecond line ', longText], + files: [], + userId: 'local-cli', + json: false, + failFast: false, + }, + io.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => capture), + now: () => 10, + }, + ), + ).resolves.toBe(0); + + const output = io.stdout(); + expect(output).toContain('"remember to call me Andrey"'); + expect(output).toContain('"first line second line"'); + expect(output).toContain('"This inline note is intentionally long xxxxxxxx..."'); + expect(output).not.toContain('text-1'); + expect(output).not.toContain(longText); + + expect(capture.capture).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + userMessage: 'Ingest external text artifact "remember to call me Andrey" into KTX memory.', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + userMessage: 'Ingest external text artifact "first line second line" into KTX memory.', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + userMessage: 'Ingest external text artifact "This inline note is intentionally long xxxxxxxx..." into KTX memory.', + }), + ); + }); + + it('continues after an item failure by default and stops when failFast is set', async () => { + const continueIo = makeIo(); + const continueCapture = fakeCapture({ failRunIds: new Set(['run-1']) }); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['bad', 'good'], + files: [], + userId: 'local-cli', + json: true, + failFast: false, + }, + continueIo.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => continueCapture), + }, + ), + ).resolves.toBe(1); + + expect(continueCapture.capture).toHaveBeenCalledTimes(2); + expect(JSON.parse(continueIo.stdout())).toMatchObject({ + status: 'failed', + results: [ + { label: '"bad"', status: 'error', error: 'run-1 failed' }, + { label: '"good"', status: 'done' }, + ], + }); + + const failFastIo = makeIo(); + const failFastCapture = fakeCapture({ failRunIds: new Set(['run-1']) }); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['bad', 'skipped'], + files: [], + userId: 'local-cli', + json: true, + failFast: true, + }, + failFastIo.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => failFastCapture), + }, + ), + ).resolves.toBe(1); + + expect(failFastCapture.capture).toHaveBeenCalledTimes(1); + expect(JSON.parse(failFastIo.stdout()).results).toHaveLength(1); + }); + + it('rejects empty batches and empty text items', async () => { + const noInputIo = makeIo(); + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: [], + files: [], + userId: 'local-cli', + json: false, + failFast: false, + }, + noInputIo.io, + { loadProject: vi.fn(), createMemoryCapture: vi.fn() }, + ), + ).resolves.toBe(1); + expect(noInputIo.stderr()).toContain('Provide at least one text item'); + + const emptyIo = makeIo(); + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: [' '], + files: [], + userId: 'local-cli', + json: false, + failFast: false, + }, + emptyIo.io, + { loadProject: vi.fn(), createMemoryCapture: vi.fn() }, + ), + ).resolves.toBe(1); + expect(emptyIo.stderr()).toContain('Text item "text-1" is empty'); + }); +}); diff --git a/packages/cli/src/text-ingest.ts b/packages/cli/src/text-ingest.ts new file mode 100644 index 00000000..d48ee24b --- /dev/null +++ b/packages/cli/src/text-ingest.ts @@ -0,0 +1,354 @@ +import { readFile as fsReadFile } from 'node:fs/promises'; +import { basename, resolve } from 'node:path'; +import { createLocalProjectMemoryCapture, type MemoryAgentInput, type MemoryCaptureStatus } from '@ktx/context/memory'; +import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; +import type { KtxCliIo } from './cli-runtime.js'; +import { createRepainter, initViewState, renderContextBuildView, type ContextBuildTargetState } from './context-build-view.js'; +import { formatDuration } from './demo-metrics.js'; +import type { KtxPublicIngestPlanTarget } from './public-ingest.js'; + +export interface KtxTextIngestArgs { + projectDir: string; + texts: string[]; + files: string[]; + connectionId?: string; + userId: string; + json: boolean; + failFast: boolean; +} + +export interface TextMemoryCapturePort { + capture(input: MemoryAgentInput): Promise<{ runId: string }>; + waitForRun(runId: string): Promise; + status(runId: string): Promise; +} + +interface TextIngestItem { + label: string; + content: string; +} + +interface TextIngestResult { + label: string; + runId: string | null; + status: 'done' | 'error'; + captured: MemoryCaptureStatus['captured']; + commitHash: string | null; + error: string | null; +} + +export interface KtxTextIngestDeps { + loadProject?: (options: { projectDir: string }) => Promise; + createMemoryCapture?: (project: KtxLocalProject) => TextMemoryCapturePort; + readFile?: (path: string) => Promise; + readStdin?: () => Promise; + now?: () => number; +} + +const INLINE_TEXT_LABEL_MAX_LENGTH = 50; +const ANSI_ESCAPE_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/g; + +function defaultCreateMemoryCapture(project: KtxLocalProject): TextMemoryCapturePort { + return createLocalProjectMemoryCapture(project); +} + +async function defaultReadStdin(): Promise { + const chunks: string[] = []; + process.stdin.setEncoding('utf-8'); + for await (const chunk of process.stdin) { + chunks.push(String(chunk)); + } + return chunks.join(''); +} + +async function defaultReadFile(path: string): Promise { + return await fsReadFile(path, 'utf-8'); +} + +function emptyCaptured(): MemoryCaptureStatus['captured'] { + return { wiki: [], sl: [], xrefs: [] }; +} + +function normalizedTextPreview(content: string): string { + return content + .replace(ANSI_ESCAPE_PATTERN, '') + .replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function truncateLabel(label: string, maxLength = INLINE_TEXT_LABEL_MAX_LENGTH): string { + const chars = Array.from(label); + if (chars.length <= maxLength) { + return label; + } + return `${chars.slice(0, maxLength - 3).join('').trimEnd()}...`; +} + +function quoteInlineTextLabel(label: string): string { + return JSON.stringify(label); +} + +function makeUniqueLabel(label: string, usedLabels: Set): string { + if (!usedLabels.has(label)) { + return label; + } + + for (let index = 2; ; index++) { + const suffix = ` (${index})`; + const candidate = `${truncateLabel(label, INLINE_TEXT_LABEL_MAX_LENGTH - suffix.length)}${suffix}`; + if (!usedLabels.has(candidate)) { + return candidate; + } + } +} + +function textLabel(content: string, index: number, usedLabels: Set): string { + const preview = normalizedTextPreview(content); + const baseLabel = preview.length > 0 ? quoteInlineTextLabel(truncateLabel(preview)) : `text-${index + 1}`; + return makeUniqueLabel(baseLabel, usedLabels); +} + +function artifactReference(label: string): string { + return label.startsWith('"') ? label : `"${label}"`; +} + +function stdinLabel(items: TextIngestItem[]): string { + if (!items.some((item) => item.label === 'stdin')) { + return 'stdin'; + } + return `stdin-${items.filter((item) => item.label.startsWith('stdin')).length + 1}`; +} + +async function loadItems(args: KtxTextIngestArgs, deps: KtxTextIngestDeps): Promise { + const items: TextIngestItem[] = []; + const usedTextLabels = new Set(); + args.texts.forEach((content, index) => { + const label = textLabel(content, index, usedTextLabels); + usedTextLabels.add(label); + items.push({ label, content }); + }); + + const readFile = deps.readFile ?? defaultReadFile; + const readStdin = deps.readStdin ?? defaultReadStdin; + for (const file of args.files) { + if (file === '-') { + items.push({ label: stdinLabel(items), content: await readStdin() }); + } else { + const path = resolve(file); + items.push({ label: basename(path), content: await readFile(path) }); + } + } + + return items; +} + +function validateItems(items: TextIngestItem[], io: KtxCliIo): boolean { + if (items.length === 0) { + io.stderr.write('Provide at least one text item with --text, a file path, or - for stdin.\n'); + return false; + } + + for (const item of items) { + if (item.content.trim().length === 0) { + io.stderr.write(`Text item "${item.label}" is empty.\n`); + return false; + } + } + return true; +} + +function makeTarget(label: string): KtxPublicIngestPlanTarget { + return { + connectionId: label, + driver: 'text', + operation: 'source-ingest', + debugCommand: '', + steps: ['memory-update'], + }; +} + +function allTargets(state: ReturnType): ContextBuildTargetState[] { + return [...state.primarySources, ...state.contextSources]; +} + +function renderTextIngestView(state: ReturnType, styled: boolean): string { + return renderContextBuildView(state, { + styled, + title: 'Ingesting text memory', + contextGroupLabel: 'Texts', + sourceIngestRunningText: 'capturing...', + completedItemName: { singular: 'text', plural: 'texts' }, + }); +} + +function summarizeCaptured(captured: MemoryCaptureStatus['captured']): string { + const parts = [ + `wiki=${captured.wiki.length}`, + `sl=${captured.sl.length}`, + `xrefs=${captured.xrefs.length}`, + ]; + return parts.join(', '); +} + +function resultFromStatus(label: string, status: MemoryCaptureStatus): TextIngestResult { + return { + label, + runId: status.runId, + status: status.status === 'done' ? 'done' : 'error', + captured: status.captured, + commitHash: status.commitHash, + error: status.error, + }; +} + +function errorResult(label: string, runId: string | null, error: unknown): TextIngestResult { + return { + label, + runId, + status: 'error', + captured: emptyCaptured(), + commitHash: null, + error: error instanceof Error ? error.message : String(error), + }; +} + +function writeJsonResult(args: KtxTextIngestArgs, results: TextIngestResult[], io: KtxCliIo): void { + io.stdout.write( + `${JSON.stringify( + { + status: results.some((result) => result.status === 'error') ? 'failed' : 'done', + projectDir: args.projectDir, + connectionId: args.connectionId ?? null, + results, + }, + null, + 2, + )}\n`, + ); +} + +function writePlainFailures(results: TextIngestResult[], io: KtxCliIo): void { + const failures = results.filter((result) => result.status === 'error'); + if (failures.length === 0) { + return; + } + + io.stdout.write('\nFailed text items:\n'); + for (const result of failures) { + io.stdout.write(` ${result.label}: ${result.error ?? 'failed'}\n`); + } +} + +export async function runKtxTextIngest( + args: KtxTextIngestArgs, + io: KtxCliIo, + deps: KtxTextIngestDeps = {}, +): Promise { + const items = await loadItems(args, deps); + if (!validateItems(items, io)) { + return 1; + } + + const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir }); + const memoryCapture = (deps.createMemoryCapture ?? defaultCreateMemoryCapture)(project); + const now = deps.now ?? (() => Date.now()); + const batchId = now(); + const state = initViewState(items.map((item) => makeTarget(item.label))); + const targets = allTargets(state); + const isTTY = io.stdout.isTTY === true && args.json !== true; + const repainter = isTTY ? createRepainter(io) : null; + const results: TextIngestResult[] = []; + + state.startedAt = now(); + const paint = () => repainter?.paint(renderTextIngestView(state, true)); + paint(); + + let spinnerInterval: ReturnType | null = null; + if (repainter) { + spinnerInterval = setInterval(() => { + const current = now(); + state.frame++; + state.totalElapsedMs = state.startedAt === null ? 0 : current - state.startedAt; + for (const target of targets) { + if (target.status === 'running' && target.startedAt !== null) { + target.elapsedMs = current - target.startedAt; + } + } + paint(); + }, 140); + } + + try { + for (let index = 0; index < items.length; index++) { + const item = items[index]!; + const target = targets[index]!; + target.status = 'running'; + target.startedAt = now(); + target.detailLine = 'capturing...'; + target.progressUpdatedAtMs = target.startedAt; + paint(); + + let runId: string | null = null; + let result: TextIngestResult; + try { + const captureInput: MemoryAgentInput = { + userId: args.userId, + chatId: `cli-text-ingest-${batchId}-${index + 1}`, + userMessage: `Ingest external text artifact ${artifactReference(item.label)} into KTX memory.`, + assistantMessage: item.content.trim(), + ...(args.connectionId ? { connectionId: args.connectionId } : {}), + sourceType: 'external_ingest', + }; + const capture = await memoryCapture.capture(captureInput); + runId = capture.runId; + await memoryCapture.waitForRun(runId); + const status = await memoryCapture.status(runId); + if (!status) { + throw new Error(`Memory capture run "${runId}" was not found.`); + } + result = resultFromStatus(item.label, status); + } catch (error) { + result = errorResult(item.label, runId, error); + } + + results.push(result); + target.elapsedMs = now() - (target.startedAt ?? now()); + target.detailLine = null; + target.status = result.status === 'done' ? 'done' : 'failed'; + target.summaryText = result.status === 'done' ? summarizeCaptured(result.captured) : null; + target.failureText = result.status === 'error' ? result.error : null; + paint(); + + if (result.status === 'error' && args.failFast) { + break; + } + } + } finally { + if (spinnerInterval) { + clearInterval(spinnerInterval); + } + } + + if (state.startedAt !== null) { + state.totalElapsedMs = now() - state.startedAt; + } + + if (args.json) { + writeJsonResult(args, results, io); + } else if (repainter) { + repainter.paint(renderTextIngestView(state, true)); + writePlainFailures(results, io); + } else { + io.stdout.write(renderTextIngestView(state, false)); + writePlainFailures(results, io); + } + + if (!args.json && results.length > 0) { + const duration = state.totalElapsedMs > 0 ? ` in ${formatDuration(state.totalElapsedMs)}` : ''; + const outcome = results.some((result) => result.status === 'error') ? 'finished with failures' : 'finished'; + io.stdout.write(`Text memory ingest ${outcome}${duration}.\n`); + } + + return results.some((result) => result.status === 'error') ? 1 : 0; +} From 3fde4438b1ad524daf72bc19483411939e3e3c5c Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Wed, 13 May 2026 19:37:25 +0200 Subject: [PATCH 2/2] fix: stop requiring readonly connection config (#71) --- .../docs/cli-reference/ktx-connection.mdx | 3 +-- .../docs/integrations/context-sources.mdx | 9 ------- .../docs/integrations/primary-sources.mdx | 12 --------- examples/local-warehouse/ktx.yaml | 1 - .../orbit-relationship-verification/ktx.yaml | 1 - packages/cli/src/connection.test.ts | 6 ++--- packages/cli/src/demo-assets.ts | 1 - packages/cli/src/doctor.test.ts | 1 - packages/cli/src/historic-sql-doctor.test.ts | 8 +----- packages/cli/src/historic-sql-doctor.ts | 3 ++- packages/cli/src/local-adapters.test.ts | 3 --- packages/cli/src/local-adapters.ts | 20 ++++++-------- .../cli/src/local-scan-connectors.test.ts | 3 --- packages/cli/src/scan.test.ts | 7 ----- packages/cli/src/setup-databases.test.ts | 13 ---------- packages/cli/src/setup-databases.ts | 10 +------ packages/cli/src/setup-sources.test.ts | 3 +-- packages/cli/src/setup.test.ts | 3 --- packages/cli/src/sl.test.ts | 11 ++++---- packages/cli/src/standalone-smoke.test.ts | 1 - .../connector-bigquery/src/connector.test.ts | 7 ----- packages/connector-bigquery/src/connector.ts | 11 ++++---- .../src/connector.test.ts | 10 ------- .../connector-clickhouse/src/connector.ts | 11 ++++---- .../connector-mysql/src/connector.test.ts | 12 +-------- packages/connector-mysql/src/connector.ts | 11 ++++---- .../connector-postgres/src/connector.test.ts | 17 +++++------- packages/connector-postgres/src/connector.ts | 11 ++++---- .../src/historic-sql-query-client.test.ts | 1 - .../connector-snowflake/src/connector.test.ts | 11 -------- packages/connector-snowflake/src/connector.ts | 13 +++++----- .../connector-sqlite/src/connector.test.ts | 26 +++++++++---------- packages/connector-sqlite/src/connector.ts | 11 ++++---- .../connector-sqlserver/src/connector.test.ts | 11 -------- packages/connector-sqlserver/src/connector.ts | 11 ++++---- .../connections/local-query-executor.test.ts | 6 ++--- .../postgres-query-executor.test.ts | 16 +++--------- .../connections/postgres-query-executor.ts | 10 +++---- .../connections/sqlite-query-executor.test.ts | 25 ++++++------------ .../src/connections/sqlite-query-executor.ts | 3 --- .../daemon-introspection.test.ts | 9 ++----- .../live-database/daemon-introspection.ts | 3 --- .../src/ingest/local-stage-ingest.test.ts | 1 - .../src/mcp/local-project-ports.test.ts | 7 ----- .../context/src/memory/local-memory.test.ts | 2 +- packages/context/src/project/config.ts | 1 - packages/context/src/scan/local-scan.test.ts | 6 ----- .../src/scan/relationship-artifacts.test.ts | 1 - .../relationship-review-decisions.test.ts | 1 - packages/context/src/sl/local-query.test.ts | 6 ++--- scripts/build-adventureworks-oltp-fixture.mjs | 1 - scripts/examples-docs.test.mjs | 1 - scripts/installed-live-database-smoke.mjs | 1 - .../installed-live-database-smoke.test.mjs | 1 - scripts/package-artifacts.mjs | 1 - 55 files changed, 103 insertions(+), 292 deletions(-) diff --git a/docs-site/content/docs/cli-reference/ktx-connection.mdx b/docs-site/content/docs/cli-reference/ktx-connection.mdx index 0cec3eae..68b7f496 100644 --- a/docs-site/content/docs/cli-reference/ktx-connection.mdx +++ b/docs-site/content/docs/cli-reference/ktx-connection.mdx @@ -63,8 +63,7 @@ agents. "connections": [ { "id": "my-warehouse", - "driver": "postgres", - "readonly": false + "driver": "postgres" } ] } diff --git a/docs-site/content/docs/integrations/context-sources.mdx b/docs-site/content/docs/integrations/context-sources.mdx index 5b85bff2..be72c941 100644 --- a/docs-site/content/docs/integrations/context-sources.mdx +++ b/docs-site/content/docs/integrations/context-sources.mdx @@ -23,7 +23,6 @@ Agents should configure and ingest context sources in this order: | Field | Required | Description | |-------|----------|-------------| | `driver` | Yes | Source adapter: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, or `notion` | -| `readonly` | Strongly recommended | Marks the source as read-only for KTX | | `source_dir` | For local file sources | Absolute or project-relative source directory | | `repo_url` | For Git-hosted sources | Git repository URL | | `branch` | No | Git branch to read | @@ -49,7 +48,6 @@ connections: my-dbt: driver: dbt source_dir: /path/to/dbt/project - readonly: true ``` For a Git-hosted project: @@ -62,7 +60,6 @@ connections: branch: main path: analytics/dbt # For monorepos auth_token_ref: env:GITHUB_TOKEN - readonly: true ``` ### Authentication @@ -110,7 +107,6 @@ connections: branch: main path: dbt_metrics # Subdirectory for monorepos auth_token_ref: env:GITHUB_TOKEN - readonly: true ``` For a local path: @@ -157,7 +153,6 @@ connections: branch: main path: analytics # Subdirectory for monorepos auth_token_ref: env:GITHUB_TOKEN - readonly: true ``` For a local path: @@ -219,7 +214,6 @@ connections: syncEnabled: "3": true syncMode: ONLY # Only ingest mapped databases - readonly: true ``` ### Authentication @@ -276,7 +270,6 @@ connections: mappings: connectionMappings: postgres_connection: postgres-main # Looker conn → KTX conn - readonly: true ``` ### Authentication @@ -329,7 +322,6 @@ connections: crawl_mode: selected_roots root_page_ids: - "abc123def456..." - readonly: true ``` For crawling all accessible pages: @@ -340,7 +332,6 @@ connections: driver: notion auth_token_ref: env:NOTION_TOKEN crawl_mode: all_accessible - readonly: true ``` ### Authentication diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 94dc4e44..653c4e38 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -21,7 +21,6 @@ Agents should prefer environment or file references over literal secrets. | `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` | | `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, ClickHouse, SQL Server | Field-by-field connection values | | `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan | -| `readonly` | Strongly recommended | all primary sources | Marks the connection as read-only in KTX config | | `historicSql` | No | supported warehouses | Enables query-history ingestion when the warehouse supports it | | `path` | Yes for path-style SQLite | SQLite | Local SQLite database path or `env:NAME` reference | @@ -37,7 +36,6 @@ connections: driver: postgres url: postgresql://user:password@host:5432/database schema: public - readonly: true ``` Or with individual fields: @@ -55,7 +53,6 @@ connections: - public - analytics ssl: true - readonly: true ``` ### Authentication @@ -123,7 +120,6 @@ connections: username: KTX_SERVICE password: env:SNOWFLAKE_PASSWORD role: ANALYST - readonly: true ``` For multiple schemas: @@ -196,7 +192,6 @@ connections: credentials_json: file:~/.config/gcloud/bq-service-account.json dataset_id: analytics location: US - readonly: true ``` For multiple datasets: @@ -269,7 +264,6 @@ connections: my-clickhouse: driver: clickhouse url: http://localhost:8123/analytics - readonly: true ``` Or with individual fields: @@ -284,7 +278,6 @@ connections: username: default password: env:CH_PASSWORD ssl: false - readonly: true ``` ### Authentication @@ -328,7 +321,6 @@ connections: my-mysql: driver: mysql url: mysql://user:password@host:3306/database - readonly: true ``` Or with individual fields: @@ -343,7 +335,6 @@ connections: username: ktx_reader password: env:MYSQL_PASSWORD ssl: true - readonly: true ``` ### Authentication @@ -387,7 +378,6 @@ connections: my-sqlserver: driver: sqlserver url: mssql://user:password@host:1433/database?trustServerCertificate=true - readonly: true ``` Or with individual fields: @@ -403,7 +393,6 @@ connections: password: env:MSSQL_PASSWORD schema: dbo trustServerCertificate: true - readonly: true ``` For multiple schemas: @@ -455,7 +444,6 @@ connections: my-sqlite: driver: sqlite path: ./data/warehouse.sqlite - readonly: true ``` Path supports multiple formats: diff --git a/examples/local-warehouse/ktx.yaml b/examples/local-warehouse/ktx.yaml index 00ccffbd..7e814188 100644 --- a/examples/local-warehouse/ktx.yaml +++ b/examples/local-warehouse/ktx.yaml @@ -2,7 +2,6 @@ project: local-warehouse connections: warehouse: driver: postgres - readonly: true storage: state: sqlite search: sqlite-fts5 diff --git a/examples/orbit-relationship-verification/ktx.yaml b/examples/orbit-relationship-verification/ktx.yaml index 5f826daf..082e0835 100644 --- a/examples/orbit-relationship-verification/ktx.yaml +++ b/examples/orbit-relationship-verification/ktx.yaml @@ -3,7 +3,6 @@ connections: orbit: driver: sqlite path: ../../packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite - readonly: true storage: state: sqlite search: sqlite-fts5 diff --git a/packages/cli/src/connection.test.ts b/packages/cli/src/connection.test.ts index 57ed8742..6eb3a08c 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/src/connection.test.ts @@ -94,7 +94,7 @@ describe('runKtxConnection', () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); await writeConnections(projectDir, { - warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true }, + warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' }, docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' }, }); const io = makeIo(); @@ -123,7 +123,7 @@ describe('runKtxConnection', () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); await writeConnections(projectDir, { - warehouse: { driver: 'sqlite', readonly: true }, + warehouse: { driver: 'sqlite' }, }); const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']); const createScanConnector = vi.fn(async () => connector); @@ -202,7 +202,7 @@ describe('runKtxConnection', () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); await writeConnections(projectDir, { - warehouse: { driver: 'sqlite', readonly: true }, + warehouse: { driver: 'sqlite' }, }); const cleanup = vi.fn(async () => undefined); const connector: KtxScanConnector = { diff --git a/packages/cli/src/demo-assets.ts b/packages/cli/src/demo-assets.ts index 1e972ef7..aae9f1a2 100644 --- a/packages/cli/src/demo-assets.ts +++ b/packages/cli/src/demo-assets.ts @@ -57,7 +57,6 @@ function demoConfig(databasePath: string): string { ` ${DEMO_CONNECTION_ID}:`, ' driver: sqlite', ` path: ${JSON.stringify(databasePath)}`, - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index d0ebdb95..cd96e6d2 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -275,7 +275,6 @@ describe('runKtxDoctor', () => { ' warehouse:', ' driver: postgres', ' url: env:WAREHOUSE_DATABASE_URL', - ' readonly: true', ' historicSql:', ' enabled: true', ' dialect: postgres', diff --git a/packages/cli/src/historic-sql-doctor.test.ts b/packages/cli/src/historic-sql-doctor.test.ts index b6f5f0fa..f3bc347e 100644 --- a/packages/cli/src/historic-sql-doctor.test.ts +++ b/packages/cli/src/historic-sql-doctor.test.ts @@ -25,7 +25,7 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { it('passes when no Postgres historic-SQL connections are enabled', async () => { const checks = await runPostgresHistoricSqlDoctorChecks( projectWithConnections({ - warehouse: { driver: 'sqlite', path: './warehouse.db', readonly: true }, + warehouse: { driver: 'sqlite', path: './warehouse.db' }, }), { postgresHistoricSqlProbe: vi.fn(), @@ -53,7 +53,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, }), @@ -66,7 +65,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { connection: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, env: process.env, @@ -87,7 +85,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, }), @@ -119,7 +116,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, }), @@ -154,7 +150,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'mysql', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, }), @@ -180,7 +175,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, historicSql: { enabled: true, dialect: 'postgres' }, }, }), diff --git a/packages/cli/src/historic-sql-doctor.ts b/packages/cli/src/historic-sql-doctor.ts index c83e7fb4..bb9a681c 100644 --- a/packages/cli/src/historic-sql-doctor.ts +++ b/packages/cli/src/historic-sql-doctor.ts @@ -86,8 +86,9 @@ async function defaultPostgresHistoricSqlProbe( const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]); + const inputDriver = input.connection.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(input.connection)) { - throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection.driver ?? 'unknown'}"`); + throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`); } const client = new KtxPostgresHistoricSqlQueryClient({ diff --git a/packages/cli/src/local-adapters.test.ts b/packages/cli/src/local-adapters.test.ts index 517c0588..b7491920 100644 --- a/packages/cli/src/local-adapters.test.ts +++ b/packages/cli/src/local-adapters.test.ts @@ -45,7 +45,6 @@ describe('CLI local ingest adapters', () => { ' warehouse:', ' driver: postgres', ' url: env:WAREHOUSE_DATABASE_URL', - ' readonly: true', ' historicSql:', ' enabled: true', ' dialect: postgres', @@ -76,7 +75,6 @@ describe('CLI local ingest adapters', () => { 'connections:', ' bq:', ' driver: bigquery', - ' readonly: true', ' dataset_id: analytics', ' location: us', ' credentials_json: \'{"project_id":"demo-project"}\'', @@ -110,7 +108,6 @@ describe('CLI local ingest adapters', () => { 'connections:', ' sf:', ' driver: snowflake', - ' readonly: true', ' account: acct', ' warehouse: wh', ' database: ANALYTICS', diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index 9a6915c2..010a7188 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -190,10 +190,9 @@ function enabledHistoricSqlDialect(connection: unknown): 'postgres' | 'bigquery' function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, connectionId: string) { const connection = project.config.connections[connectionId] as KtxPostgresConnectionConfig | undefined; + const inputDriver = connection?.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a Postgres connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Historic SQL local ingest requires a Postgres connection, got ${String(inputDriver)}`); } return { async executeQuery(sql: string, params?: unknown[]) { @@ -212,10 +211,9 @@ function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, conn function createEphemeralBigQueryHistoricSqlClient(project: KtxLocalProject, connectionId: string) { const connection = project.config.connections[connectionId] as KtxBigQueryConnectionConfig | undefined; + const inputDriver = connection?.driver ?? 'unknown'; if (!isKtxBigQueryConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Historic SQL local ingest requires a BigQuery connection, got ${String(inputDriver)}`); } return { async executeQuery(query: string) { @@ -243,10 +241,9 @@ async function createEphemeralSnowflakeHistoricSqlClient( connectorModule: SnowflakeConnectorModule, ) { const connection = project.config.connections[connectionId]; + const inputDriver = connection?.driver ?? 'unknown'; if (!connectorModule.isKtxSnowflakeConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a Snowflake connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Historic SQL local ingest requires a Snowflake connection, got ${String(inputDriver)}`); } return { async executeQuery(query: string) { @@ -308,10 +305,9 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli } if (dialect === 'bigquery') { + const inputDriver = connection?.driver ?? 'unknown'; if (!isKtxBigQueryConnectionConfig(connection)) { - throw new Error( - `Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Historic SQL local ingest requires a BigQuery connection, got ${String(inputDriver)}`); } return { ...base, diff --git a/packages/cli/src/local-scan-connectors.test.ts b/packages/cli/src/local-scan-connectors.test.ts index 087e978d..e8a5c1e9 100644 --- a/packages/cli/src/local-scan-connectors.test.ts +++ b/packages/cli/src/local-scan-connectors.test.ts @@ -49,7 +49,6 @@ describe('createKtxCliScanConnector', () => { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -72,7 +71,6 @@ describe('createKtxCliScanConnector', () => { ' warehouse:', ' driver: bigquery', ' dataset_id: analytics', - ' readonly: true', ' max_bytes_billed: "987654321"', '', ].join('\n'), @@ -123,7 +121,6 @@ describe('createKtxCliScanConnector', () => { ' warehouse:', ' type: postgres', ' url: postgresql://example/db', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/scan.test.ts b/packages/cli/src/scan.test.ts index 28c60ea0..c4cbaf70 100644 --- a/packages/cli/src/scan.test.ts +++ b/packages/cli/src/scan.test.ts @@ -861,7 +861,6 @@ describe('runKtxScan', () => { ' warehouse:', ' driver: mysql', ' url: env:MYSQL_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -910,7 +909,6 @@ describe('runKtxScan', () => { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -968,7 +966,6 @@ describe('runKtxScan', () => { ' database: analytics', ' username: reader', ' password: env:POSTGRES_PASSWORD', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1035,7 +1032,6 @@ describe('runKtxScan', () => { ' database: analytics', ' username: reader', ' password: env:CLICKHOUSE_PASSWORD', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1087,7 +1083,6 @@ describe('runKtxScan', () => { ' database: analytics', ' username: reader', ' schema: dbo', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1153,7 +1148,6 @@ describe('runKtxScan', () => { ' dataset_id: analytics', ' credentials_json: env:BIGQUERY_CREDENTIALS_JSON', ' location: US', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1222,7 +1216,6 @@ describe('runKtxScan', () => { ' database: ANALYTICS', ' schema_name: PUBLIC', ' username: reader', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 95d1e3fb..d010a908 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -218,7 +218,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -281,7 +280,6 @@ describe('setup databases step', () => { expect(config.connections['postgres-warehouse']).toEqual({ driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }); }); @@ -542,7 +540,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -583,7 +580,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -698,7 +694,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -843,7 +838,6 @@ describe('setup databases step', () => { port: 5432, database: 'analytics', username: 'readonly', - readonly: true, }); expect(connection.password).toMatch(/^file:/); const secretPath = join(tempDir, '.ktx/secrets/postgres-warehouse-password'); @@ -998,7 +992,6 @@ describe('setup databases step', () => { expect(config.connections['postgres-warehouse']).toMatchObject({ driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }); }); @@ -1115,7 +1108,6 @@ describe('setup databases step', () => { driver: 'postgres', url: 'env:DATABASE_URL', schemas: ['public'], - readonly: true, }); expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], @@ -1153,7 +1145,6 @@ describe('setup databases step', () => { expect(config.connections.warehouse).toEqual({ driver: 'sqlite', path: './warehouse.sqlite', - readonly: true, }); expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], @@ -1170,7 +1161,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', ' analytics:', ' driver: snowflake', ' authMethod: password', @@ -1180,7 +1170,6 @@ describe('setup databases step', () => { ' schema_name: PUBLIC', ' username: reader', ' password: env:SNOWFLAKE_PASSWORD', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1443,7 +1432,6 @@ describe('setup databases step', () => { ' driver: bigquery', ' dataset_id: analytics', ' credentials_json: env:BIGQUERY_CREDENTIALS_JSON', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1492,7 +1480,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 85be2620..5b5b5f8a 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -593,7 +593,6 @@ async function buildFieldsConnectionConfig(input: { username, ...(passwordRef ? { password: passwordRef } : {}), ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -615,7 +614,6 @@ async function buildPastedUrlConnectionConfig(input: { driver: input.driver, url, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -629,7 +627,6 @@ async function buildPastedUrlConnectionConfig(input: { driver: input.driver, url: ref, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -637,7 +634,6 @@ async function buildPastedUrlConnectionConfig(input: { driver: input.driver, url, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -661,14 +657,12 @@ async function buildUrlConnectionConfig(input: { driver: input.driver, url: ref, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } return { driver: input.driver, url, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -706,7 +700,7 @@ async function buildConnectionConfig(input: { 'SQLite database file\nEnter a relative or absolute path, for example ./warehouse.sqlite.', )); if (path === undefined) return 'back'; - return path ? { driver: 'sqlite', path, readonly: true } : null; + return path ? { driver: 'sqlite', path } : null; } if (driver === 'postgres' || driver === 'mysql' || driver === 'clickhouse' || driver === 'sqlserver') { return await buildUrlConnectionConfig({ driver, connectionId: input.connectionId, args, prompts }); @@ -728,7 +722,6 @@ async function buildConnectionConfig(input: { dataset_id: datasetId, credentials_json: normalizeFileReference(credentialsPath), ...(location ? { location } : {}), - readonly: true, }; } if (driver === 'snowflake') { @@ -767,7 +760,6 @@ async function buildConnectionConfig(input: { username, password: passwordRef, ...(role ? { role } : {}), - readonly: true, }; } throw new Error(`Unsupported database driver: ${driver}`); diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index c0fb0227..7fe61f76 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -98,7 +98,7 @@ describe('setup sources step', () => { ...config, connections: { ...config.connections, - warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true }, + warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' }, }, setup: { ...config.setup, @@ -455,7 +455,6 @@ describe('setup sources step', () => { driver: 'snowflake', account: 'acme', database: 'analytics', - readonly: true, }); const cases: Array<{ diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index dd134fce..246484d3 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -170,7 +170,6 @@ describe('setup status', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -192,7 +191,6 @@ describe('setup status', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1373,7 +1371,6 @@ describe('setup status', () => { ' warehouse:', ' driver: postgres', ' url: env:DEMO_DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index ff4132b4..14f18337 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -190,7 +190,7 @@ joins: [] it('runs sl query and prints SQL output', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -247,7 +247,7 @@ joins: [] it('runs sl query from a JSON query file', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -314,7 +314,7 @@ joins: [] it('creates default sl query compute through the managed runtime helper', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -375,7 +375,7 @@ joins: [] it('executes sl query through the injected query executor', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -471,7 +471,7 @@ joins: [] `); db.close(); - project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db', readonly: true }; + project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db' }; await writeFile( join(projectDir, 'ktx.yaml'), [ @@ -480,7 +480,6 @@ joins: [] ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index 6f878617..0f53cca0 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -106,7 +106,6 @@ async function writeSqliteScanConfig(projectDir: string, dbPath: string, enrich ' warehouse:', ' driver: sqlite', ` path: ${JSON.stringify(dbPath)}`, - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/connector-bigquery/src/connector.test.ts b/packages/connector-bigquery/src/connector.test.ts index a1c23864..46dc3b53 100644 --- a/packages/connector-bigquery/src/connector.test.ts +++ b/packages/connector-bigquery/src/connector.test.ts @@ -100,7 +100,6 @@ const connection = { dataset_id: 'analytics', credentials_json: JSON.stringify({ project_id: 'project-1', client_email: 'reader@example.test' }), location: 'US', - readonly: true, }; describe('KtxBigQueryScanConnector', () => { @@ -112,12 +111,6 @@ describe('KtxBigQueryScanConnector', () => { datasetIds: ['analytics'], location: 'US', }); - expect(() => - bigQueryConnectionConfigFromConfig({ - connectionId: 'warehouse', - connection: { ...connection, readonly: false }, - }), - ).toThrow('Native BigQuery connector requires connections.warehouse.readonly: true'); }); it('introspects datasets, table metadata, primary keys, and normalized types', async () => { diff --git a/packages/connector-bigquery/src/connector.ts b/packages/connector-bigquery/src/connector.ts index a994912e..72cb8129 100644 --- a/packages/connector-bigquery/src/connector.ts +++ b/packages/connector-bigquery/src/connector.ts @@ -30,7 +30,6 @@ export interface KtxBigQueryConnectionConfig { dataset_ids?: string[]; credentials_json?: string; location?: string; - readonly?: boolean; [key: string]: unknown; } @@ -194,7 +193,9 @@ function normalizeValue(value: unknown): unknown { return value; } -export function isKtxBigQueryConnectionConfig(connection: KtxBigQueryConnectionConfig | undefined): boolean { +export function isKtxBigQueryConnectionConfig( + connection: KtxBigQueryConnectionConfig | undefined, +): connection is KtxBigQueryConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'bigquery'; } @@ -203,11 +204,9 @@ export function bigQueryConnectionConfigFromConfig(input: { connection: KtxBigQueryConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxBigQueryResolvedConnectionConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxBigQueryConnectionConfig(input.connection)) { - throw new Error(`Native BigQuery connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native BigQuery connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native BigQuery connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-clickhouse/src/connector.test.ts b/packages/connector-clickhouse/src/connector.test.ts index 7ed60efa..4f8b7f52 100644 --- a/packages/connector-clickhouse/src/connector.test.ts +++ b/packages/connector-clickhouse/src/connector.test.ts @@ -112,7 +112,6 @@ describe('KtxClickHouseScanConnector', () => { username: 'reader', password: 'test-pass', // pragma: allowlist secret ssl: true, - readonly: true, }, }), ).toMatchObject({ @@ -123,12 +122,6 @@ describe('KtxClickHouseScanConnector', () => { password: 'test-pass', // pragma: allowlist secret ssl: true, }); - expect(() => - clickHouseClientConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'clickhouse', host: 'ch.example.test', database: 'analytics', readonly: false }, - }), - ).toThrow('Native ClickHouse connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, and views', async () => { @@ -140,7 +133,6 @@ describe('KtxClickHouseScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-pass', // pragma: allowlist secret - readonly: true, }, clientFactory: fakeClientFactory(), now: () => new Date('2026-04-29T14:00:00.000Z'), @@ -189,7 +181,6 @@ describe('KtxClickHouseScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-pass', // pragma: allowlist secret - readonly: true, }, clientFactory, }); @@ -253,7 +244,6 @@ describe('KtxClickHouseScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-pass', // pragma: allowlist secret - readonly: true, }, }, clientFactory: fakeClientFactory(), diff --git a/packages/connector-clickhouse/src/connector.ts b/packages/connector-clickhouse/src/connector.ts index 0273a62b..4b39c943 100644 --- a/packages/connector-clickhouse/src/connector.ts +++ b/packages/connector-clickhouse/src/connector.ts @@ -35,7 +35,6 @@ export interface KtxClickHouseConnectionConfig { password?: string; url?: string; ssl?: boolean; - readonly?: boolean; [key: string]: unknown; } @@ -193,7 +192,9 @@ function isNullableClickHouseType(type: string): boolean { return type.startsWith('Nullable(') || type.startsWith('LowCardinality(Nullable('); } -export function isKtxClickHouseConnectionConfig(connection: KtxClickHouseConnectionConfig | undefined): boolean { +export function isKtxClickHouseConnectionConfig( + connection: KtxClickHouseConnectionConfig | undefined, +): connection is KtxClickHouseConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'clickhouse'; } @@ -202,11 +203,9 @@ export function clickHouseClientConfigFromConfig(input: { connection: KtxClickHouseConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxClickHouseResolvedClientConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxClickHouseConnectionConfig(input.connection)) { - throw new Error(`Native ClickHouse connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native ClickHouse connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native ClickHouse connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-mysql/src/connector.test.ts b/packages/connector-mysql/src/connector.test.ts index 3e7ac1e1..c5c5a3fa 100644 --- a/packages/connector-mysql/src/connector.test.ts +++ b/packages/connector-mysql/src/connector.test.ts @@ -92,7 +92,7 @@ function fakePoolFactory(): KtxMysqlPoolFactory { describe('KtxMysqlScanConnector', () => { it('resolves MySQL connection configuration safely', () => { - expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics', readonly: true })).toBe(true); + expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(true); expect(isKtxMysqlConnectionConfig({ driver: 'postgres', host: 'localhost', database: 'analytics' })).toBe(false); expect( mysqlConnectionPoolConfigFromConfig({ @@ -105,7 +105,6 @@ describe('KtxMysqlScanConnector', () => { username: 'reader', password: 'secret', // pragma: allowlist secret ssl: true, - readonly: true, }, }), ).toMatchObject({ @@ -116,12 +115,6 @@ describe('KtxMysqlScanConnector', () => { password: 'secret', // pragma: allowlist secret ssl: { rejectUnauthorized: false }, }); - expect(() => - mysqlConnectionPoolConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'mysql', host: 'db.example.test', database: 'analytics', readonly: false }, - }), - ).toThrow('Native MySQL connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => { @@ -133,7 +126,6 @@ describe('KtxMysqlScanConnector', () => { database: 'analytics', username: 'reader', password: 'secret', // pragma: allowlist secret - readonly: true, }, poolFactory: fakePoolFactory(), now: () => new Date('2026-04-29T12:00:00.000Z'), @@ -192,7 +184,6 @@ describe('KtxMysqlScanConnector', () => { database: 'analytics', username: 'reader', password: 'secret', // pragma: allowlist secret - readonly: true, }, poolFactory, }); @@ -249,7 +240,6 @@ describe('KtxMysqlScanConnector', () => { database: 'analytics', username: 'reader', password: 'secret', // pragma: allowlist secret - readonly: true, }, }, poolFactory: fakePoolFactory(), diff --git a/packages/connector-mysql/src/connector.ts b/packages/connector-mysql/src/connector.ts index 69a09272..62bb1880 100644 --- a/packages/connector-mysql/src/connector.ts +++ b/packages/connector-mysql/src/connector.ts @@ -35,7 +35,6 @@ export interface KtxMysqlConnectionConfig { password?: string; url?: string; ssl?: boolean | { rejectUnauthorized?: boolean }; - readonly?: boolean; [key: string]: unknown; } @@ -232,7 +231,9 @@ function queryParams(params: Record | unknown[] | undefined): u return Array.isArray(params) ? params : Object.values(params); } -export function isKtxMysqlConnectionConfig(connection: KtxMysqlConnectionConfig | undefined): boolean { +export function isKtxMysqlConnectionConfig( + connection: KtxMysqlConnectionConfig | undefined, +): connection is KtxMysqlConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'mysql'; } @@ -241,11 +242,9 @@ export function mysqlConnectionPoolConfigFromConfig(input: { connection: KtxMysqlConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxMysqlPoolConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxMysqlConnectionConfig(input.connection)) { - throw new Error(`Native MySQL connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native MySQL connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native MySQL connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-postgres/src/connector.test.ts b/packages/connector-postgres/src/connector.test.ts index 96443c90..8093acda 100644 --- a/packages/connector-postgres/src/connector.test.ts +++ b/packages/connector-postgres/src/connector.test.ts @@ -102,7 +102,7 @@ function metadataResults(): Map { describe('KtxPostgresScanConnector', () => { it('resolves configuration safely', () => { - expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL', readonly: true })).toBe(true); + expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(true); expect(isKtxPostgresConnectionConfig({ driver: 'postgresql', host: 'db', database: 'analytics' })).toBe(true); expect(isKtxPostgresConnectionConfig({ driver: 'mysql', host: 'db' })).toBe(false); expect( @@ -115,7 +115,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schemas: ['analytics', 'public'], - readonly: true, ssl: true, rejectUnauthorized: false, }, @@ -134,7 +133,6 @@ describe('KtxPostgresScanConnector', () => { connection: { driver: 'postgres', url: 'env:DEMO_DATABASE_URL', - readonly: true, }, env: { DEMO_DATABASE_URL: 'postgresql://reader@demo.example.test:5432/demo?sslmode=prefer', @@ -148,12 +146,16 @@ describe('KtxPostgresScanConnector', () => { }); expect(libpqPreferConfig).not.toHaveProperty('connectionString'); expect(libpqPreferConfig).not.toHaveProperty('ssl'); - expect(() => + expect( postgresPoolConfigFromConfig({ connectionId: 'warehouse', connection: { driver: 'postgres', host: 'db.example.test', database: 'analytics', username: 'reader' }, }), - ).toThrow('Native PostgreSQL connector requires connections.warehouse.readonly: true'); + ).toMatchObject({ + host: 'db.example.test', + database: 'analytics', + user: 'reader', + }); }); it('introspects schemas, tables, views, primary keys, comments, row counts, and foreign keys', async () => { @@ -166,7 +168,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, poolFactory: fakePoolFactory(metadataResults()), now: () => new Date('2026-04-29T10:00:00.000Z'), @@ -225,7 +226,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, poolFactory: fakePoolFactory(metadataResults()), }); @@ -274,7 +274,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, }, poolFactory: fakePoolFactory(metadataResults()), @@ -347,7 +346,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, }, poolFactory: endAwarePoolFactory, @@ -383,7 +381,6 @@ describe('KtxPostgresScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-password', // pragma: allowlist secret - readonly: true, }, poolFactory, }); diff --git a/packages/connector-postgres/src/connector.ts b/packages/connector-postgres/src/connector.ts index 65490040..7f5ed65c 100644 --- a/packages/connector-postgres/src/connector.ts +++ b/packages/connector-postgres/src/connector.ts @@ -61,7 +61,6 @@ export interface KtxPostgresConnectionConfig { sslmode?: string; sslMode?: string; rejectUnauthorized?: boolean; - readonly?: boolean; [key: string]: unknown; } @@ -291,7 +290,9 @@ function searchPathSchemasFromConnection(connection: KtxPostgresConnectionConfig return schemas.includes('public') ? schemas : [...schemas, 'public']; } -export function isKtxPostgresConnectionConfig(connection: KtxPostgresConnectionConfig | undefined): boolean { +export function isKtxPostgresConnectionConfig( + connection: KtxPostgresConnectionConfig | undefined, +): connection is KtxPostgresConnectionConfig { const driver = String(connection?.driver ?? '').toLowerCase(); return driver === 'postgres' || driver === 'postgresql'; } @@ -301,11 +302,9 @@ export function postgresPoolConfigFromConfig(input: { connection: KtxPostgresConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxPostgresPoolConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(input.connection)) { - throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native PostgreSQL connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-postgres/src/historic-sql-query-client.test.ts b/packages/connector-postgres/src/historic-sql-query-client.test.ts index f02cb9c3..b9c9fd40 100644 --- a/packages/connector-postgres/src/historic-sql-query-client.test.ts +++ b/packages/connector-postgres/src/historic-sql-query-client.test.ts @@ -30,7 +30,6 @@ describe('KtxPostgresHistoricSqlQueryClient', () => { connectionId: 'warehouse', connection: { driver: 'postgres', - readonly: true, url: 'postgresql://readonly:secret@pg.example.test/warehouse', // pragma: allowlist secret }, poolFactory, diff --git a/packages/connector-snowflake/src/connector.test.ts b/packages/connector-snowflake/src/connector.test.ts index 91bb33d4..a49be885 100644 --- a/packages/connector-snowflake/src/connector.test.ts +++ b/packages/connector-snowflake/src/connector.test.ts @@ -78,7 +78,6 @@ describe('KtxSnowflakeScanConnector', () => { warehouse: 'WH', database: 'ANALYTICS', username: 'reader', - readonly: true, }), ).toBe(true); expect(isKtxSnowflakeConnectionConfig({ driver: 'bigquery' })).toBe(false); @@ -94,7 +93,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, }), ).toMatchObject({ @@ -105,12 +103,6 @@ describe('KtxSnowflakeScanConnector', () => { username: 'reader', authMethod: 'password', }); - expect(() => - snowflakeConnectionConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'snowflake', account: 'acct', readonly: false }, - }), - ).toThrow('Native Snowflake connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, and dimensions', async () => { @@ -125,7 +117,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, driverFactory: fakeDriverFactory(), now: () => new Date('2026-04-29T18:00:00.000Z'), @@ -185,7 +176,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, driverFactory, }); @@ -243,7 +233,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, }, driverFactory: fakeDriverFactory(), diff --git a/packages/connector-snowflake/src/connector.ts b/packages/connector-snowflake/src/connector.ts index 063976f7..76fc34fd 100644 --- a/packages/connector-snowflake/src/connector.ts +++ b/packages/connector-snowflake/src/connector.ts @@ -38,7 +38,6 @@ export interface KtxSnowflakeConnectionConfig { privateKey?: string; passphrase?: string; role?: string; - readonly?: boolean; [key: string]: unknown; } @@ -191,7 +190,9 @@ function toSnowflakeBinds(params: unknown[] | undefined): snowflake.Binds | unde return params?.map((value) => toSnowflakeBind(value)); } -export function isKtxSnowflakeConnectionConfig(connection: KtxSnowflakeConnectionConfig | undefined): boolean { +export function isKtxSnowflakeConnectionConfig( + connection: KtxSnowflakeConnectionConfig | undefined, +): connection is KtxSnowflakeConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'snowflake'; } @@ -200,11 +201,9 @@ export function snowflakeConnectionConfigFromConfig(input: { connection: KtxSnowflakeConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxSnowflakeResolvedConnectionConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSnowflakeConnectionConfig(input.connection)) { - throw new Error(`Native Snowflake connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native Snowflake connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; const authMethod = input.connection?.authMethod ?? 'password'; @@ -395,7 +394,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver { private async createConnection(): Promise { const patch = await this.sdkOptionsProvider?.resolve({ account: this.resolved.account, - connection: { ...this.resolved, driver: 'snowflake', readonly: true }, + connection: { ...this.resolved, driver: 'snowflake' }, }); if (patch?.close) { this.closeSdkOptions.push(patch.close); diff --git a/packages/connector-sqlite/src/connector.test.ts b/packages/connector-sqlite/src/connector.test.ts index 4bc26ec9..9bee5567 100644 --- a/packages/connector-sqlite/src/connector.test.ts +++ b/packages/connector-sqlite/src/connector.test.ts @@ -53,45 +53,43 @@ describe('KtxSqliteScanConnector', () => { writeFileSync(pointerPath, dbPath, 'utf-8'); try { - expect(isKtxSqliteConnectionConfig({ driver: 'sqlite', path: 'warehouse.db', readonly: true })).toBe(true); - expect(isKtxSqliteConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL', readonly: true })).toBe( - false, - ); + expect(isKtxSqliteConnectionConfig({ driver: 'sqlite', path: 'warehouse.db' })).toBe(true); + expect(isKtxSqliteConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(false); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, }), ).toBe(dbPath); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true }, + connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' }, }), ).toBe(dbPath); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: `file://${dbPath}`, readonly: true }, + connection: { driver: 'sqlite', url: `file://${dbPath}` }, }), ).toBe(dbPath); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: `file:${pointerPath}`, readonly: true }, + connection: { driver: 'sqlite', path: `file:${pointerPath}` }, }), ).toBe(dbPath); - expect(() => + expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: false }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, }), - ).toThrow('Native SQLite connector requires connections.warehouse.readonly: true'); + ).toBe(dbPath); } finally { if (originalDatabaseUrl === undefined) { delete process.env.KTX_SQLITE_TEST_URL; @@ -104,7 +102,7 @@ describe('KtxSqliteScanConnector', () => { it('introspects schema, primary keys, row counts, views, and foreign keys', async () => { const connector = new KtxSqliteScanConnector({ connectionId: 'warehouse', - connection: { driver: 'sqlite', path: dbPath, readonly: true }, + connection: { driver: 'sqlite', path: dbPath }, now: () => new Date('2026-04-29T10:00:00.000Z'), }); @@ -151,7 +149,7 @@ describe('KtxSqliteScanConnector', () => { it('runs samples, distinct values, statistics, and read-only SQL', async () => { const connector = new KtxSqliteScanConnector({ connectionId: 'warehouse', - connection: { driver: 'sqlite', path: dbPath, readonly: true }, + connection: { driver: 'sqlite', path: dbPath }, }); await expect( @@ -199,7 +197,7 @@ describe('KtxSqliteScanConnector', () => { const introspection = createSqliteLiveDatabaseIntrospection({ projectDir: tempDir, connections: { - warehouse: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + warehouse: { driver: 'sqlite', path: 'warehouse.db' }, }, now: () => new Date('2026-04-29T10:00:00.000Z'), }); diff --git a/packages/connector-sqlite/src/connector.ts b/packages/connector-sqlite/src/connector.ts index c42db002..2a0d997e 100644 --- a/packages/connector-sqlite/src/connector.ts +++ b/packages/connector-sqlite/src/connector.ts @@ -29,7 +29,6 @@ export interface KtxSqliteConnectionConfig { path?: string; url?: string; file_path?: string; - readonly?: boolean; [key: string]: unknown; } @@ -135,17 +134,17 @@ function stripLeadingSqlComments(sql: string): string { return sql.slice(index); } -export function isKtxSqliteConnectionConfig(connection: KtxSqliteConnectionConfig | undefined): boolean { +export function isKtxSqliteConnectionConfig( + connection: KtxSqliteConnectionConfig | undefined, +): connection is KtxSqliteConnectionConfig { const driver = String(connection?.driver ?? '').toLowerCase(); return driver === 'sqlite' || driver === 'sqlite3'; } export function sqliteDatabasePathFromConfig(input: SqliteDatabasePathInput): string { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSqliteConnectionConfig(input.connection)) { - throw new Error(`Native SQLite connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native SQLite connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native SQLite connector cannot run driver "${inputDriver}"`); } const configuredPath = stringConfigValue(input.connection, 'path') ?? diff --git a/packages/connector-sqlserver/src/connector.test.ts b/packages/connector-sqlserver/src/connector.test.ts index eebab0ba..b7915fa8 100644 --- a/packages/connector-sqlserver/src/connector.test.ts +++ b/packages/connector-sqlserver/src/connector.test.ts @@ -145,7 +145,6 @@ describe('KtxSqlServerScanConnector', () => { driver: 'sqlserver', host: 'localhost', database: 'analytics', - readonly: true, }), ).toBe(true); expect(isKtxSqlServerConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(false); @@ -159,7 +158,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', trustServerCertificate: false, - readonly: true, }, }), ).toMatchObject({ @@ -169,12 +167,6 @@ describe('KtxSqlServerScanConnector', () => { user: 'reader', options: { encrypt: true, trustServerCertificate: false }, }); - expect(() => - sqlServerConnectionPoolConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'sqlserver', host: 'db.example.test', database: 'analytics', readonly: false }, - }), - ).toThrow('Native SQL Server connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => { @@ -186,7 +178,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', schema: 'dbo', - readonly: true, }, poolFactory: fakePoolFactory(), now: () => new Date('2026-04-29T16:00:00.000Z'), @@ -246,7 +237,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', schema: 'dbo', - readonly: true, }, poolFactory, }); @@ -315,7 +305,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', schema: 'dbo', - readonly: true, }, }, poolFactory: fakePoolFactory(), diff --git a/packages/connector-sqlserver/src/connector.ts b/packages/connector-sqlserver/src/connector.ts index 189ff98b..73a46aab 100644 --- a/packages/connector-sqlserver/src/connector.ts +++ b/packages/connector-sqlserver/src/connector.ts @@ -37,7 +37,6 @@ export interface KtxSqlServerConnectionConfig { schema?: string; schemas?: string[]; trustServerCertificate?: boolean; - readonly?: boolean; [key: string]: unknown; } @@ -234,7 +233,9 @@ function limitSqlForSqlServerExecution(sqlText: string, maxRows: number | undefi return `SELECT TOP ${maxRows} * FROM (${trimmed}) AS ktx_query_result`; } -export function isKtxSqlServerConnectionConfig(connection: KtxSqlServerConnectionConfig | undefined): boolean { +export function isKtxSqlServerConnectionConfig( + connection: KtxSqlServerConnectionConfig | undefined, +): connection is KtxSqlServerConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'sqlserver'; } @@ -243,11 +244,9 @@ export function sqlServerConnectionPoolConfigFromConfig(input: { connection: KtxSqlServerConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxSqlServerPoolConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSqlServerConnectionConfig(input.connection)) { - throw new Error(`Native SQL Server connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native SQL Server connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native SQL Server connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/context/src/connections/local-query-executor.test.ts b/packages/context/src/connections/local-query-executor.test.ts index fd94c6dc..d2f77975 100644 --- a/packages/context/src/connections/local-query-executor.test.ts +++ b/packages/context/src/connections/local-query-executor.test.ts @@ -26,14 +26,14 @@ describe('createDefaultLocalQueryExecutor', () => { await expect( executor.execute({ connectionId: 'pg', - connection: { driver: 'postgres', readonly: true }, + connection: { driver: 'postgres' }, sql: 'select 1', }), ).resolves.toMatchObject({ headers: ['pg'] }); await expect( executor.execute({ connectionId: 'local', - connection: { driver: 'sqlite', readonly: true }, + connection: { driver: 'sqlite' }, sql: 'select 1', }), ).resolves.toMatchObject({ headers: ['sqlite'] }); @@ -51,7 +51,7 @@ describe('createDefaultLocalQueryExecutor', () => { await expect( executor.execute({ connectionId: 'warehouse', - connection: { driver: 'snowflake', readonly: true }, + connection: { driver: 'snowflake' }, sql: 'select 1', }), ).rejects.toThrow('No local query executor is configured for driver "snowflake".'); diff --git a/packages/context/src/connections/postgres-query-executor.test.ts b/packages/context/src/connections/postgres-query-executor.test.ts index 2c52d97e..6bb522cf 100644 --- a/packages/context/src/connections/postgres-query-executor.test.ts +++ b/packages/context/src/connections/postgres-query-executor.test.ts @@ -37,7 +37,7 @@ describe('createPostgresQueryExecutor', () => { const result = await executor.execute({ connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db', readonly: true }, + connection: { driver: 'postgres', url: 'postgres://example/db' }, sql: 'select status, count(*) as order_count from public.orders group by status', maxRows: 50, }); @@ -80,7 +80,7 @@ describe('createPostgresQueryExecutor', () => { await expect( executor.execute({ connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db', readonly: true }, + connection: { driver: 'postgres', url: 'postgres://example/db' }, sql: 'select * from broken', maxRows: 10, }), @@ -89,23 +89,15 @@ describe('createPostgresQueryExecutor', () => { expect(client.end).toHaveBeenCalledTimes(1); }); - it('requires a Postgres url and read-only connection config', async () => { + it('requires a Postgres url', async () => { const executor = createPostgresQueryExecutor({ clientFactory: vi.fn() }); await expect( executor.execute({ connectionId: 'warehouse', - connection: { driver: 'postgres', readonly: true }, + connection: { driver: 'postgres' }, sql: 'select 1', }), ).rejects.toThrow('Local Postgres execution requires connections.warehouse.url'); - - await expect( - executor.execute({ - connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db', readonly: false }, - sql: 'select 1', - }), - ).rejects.toThrow('Local query execution requires connections.warehouse.readonly: true'); }); }); diff --git a/packages/context/src/connections/postgres-query-executor.ts b/packages/context/src/connections/postgres-query-executor.ts index 2ab142a5..b5f2d02e 100644 --- a/packages/context/src/connections/postgres-query-executor.ts +++ b/packages/context/src/connections/postgres-query-executor.ts @@ -37,18 +37,16 @@ export function createPostgresQueryExecutor(options: PostgresQueryExecutorOption return { async execute(input: KtxSqlQueryExecutionInput): Promise { const driver = connectionDriver(input); + const connection = input.connection; if (driver !== 'postgres' && driver !== 'postgresql') { - throw new Error(`Local Postgres execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`); + throw new Error(`Local Postgres execution cannot run driver "${connection?.driver ?? 'unknown'}".`); } - if (input.connection?.readonly !== true) { - throw new Error(`Local query execution requires connections.${input.connectionId}.readonly: true.`); - } - if (typeof input.connection.url !== 'string' || input.connection.url.trim().length === 0) { + if (typeof connection?.url !== 'string' || connection.url.trim().length === 0) { throw new Error(`Local Postgres execution requires connections.${input.connectionId}.url.`); } const client = clientFactory({ - connectionString: input.connection.url, + connectionString: connection.url, statement_timeout: options.statementTimeoutMs ?? 30_000, query_timeout: options.queryTimeoutMs ?? 35_000, connectionTimeoutMillis: options.connectionTimeoutMs ?? 5_000, diff --git a/packages/context/src/connections/sqlite-query-executor.test.ts b/packages/context/src/connections/sqlite-query-executor.test.ts index 3046f9bb..facb5139 100644 --- a/packages/context/src/connections/sqlite-query-executor.test.ts +++ b/packages/context/src/connections/sqlite-query-executor.test.ts @@ -38,7 +38,7 @@ describe('createSqliteQueryExecutor', () => { const result = await executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, sql: 'select status, count(*) as order_count from orders group by status order by status', maxRows: 10, }); @@ -60,7 +60,7 @@ describe('createSqliteQueryExecutor', () => { sqliteDatabasePathFromConnection({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: `file://${dbPath}`, readonly: true }, + connection: { driver: 'sqlite', url: `file://${dbPath}` }, sql: 'select 1', }), ).toBe(dbPath); @@ -74,7 +74,7 @@ describe('createSqliteQueryExecutor', () => { sqliteDatabasePathFromConnection({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: `file:${pointerPath}`, readonly: true }, + connection: { driver: 'sqlite', path: `file:${pointerPath}` }, sql: 'select 1', }), ).toBe(dbPath); @@ -89,7 +89,7 @@ describe('createSqliteQueryExecutor', () => { sqliteDatabasePathFromConnection({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true }, + connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' }, sql: 'select 1', }), ).toBe(dbPath); @@ -109,20 +109,20 @@ describe('createSqliteQueryExecutor', () => { executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, sql: 'delete from orders', }), ).rejects.toThrow('Only read-only SELECT/WITH queries can be executed locally'); }); - it('requires a SQLite driver, read-only config, and a database path', async () => { + it('requires a SQLite driver and a database path', async () => { const executor = createSqliteQueryExecutor(); await expect( executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'postgres', path: 'warehouse.db', readonly: true }, + connection: { driver: 'postgres', path: 'warehouse.db' }, sql: 'select 1', }), ).rejects.toThrow('Local SQLite execution cannot run driver "postgres"'); @@ -131,16 +131,7 @@ describe('createSqliteQueryExecutor', () => { executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: false }, - sql: 'select 1', - }), - ).rejects.toThrow('Local query execution requires connections.warehouse.readonly: true'); - - await expect( - executor.execute({ - connectionId: 'warehouse', - projectDir: tempDir, - connection: { driver: 'sqlite', readonly: true }, + connection: { driver: 'sqlite' }, sql: 'select 1', }), ).rejects.toThrow('Local SQLite execution requires connections.warehouse.path or connections.warehouse.url'); diff --git a/packages/context/src/connections/sqlite-query-executor.ts b/packages/context/src/connections/sqlite-query-executor.ts index d32a37ba..2a87ef7d 100644 --- a/packages/context/src/connections/sqlite-query-executor.ts +++ b/packages/context/src/connections/sqlite-query-executor.ts @@ -54,9 +54,6 @@ export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInpu if (driver !== 'sqlite' && driver !== 'sqlite3') { throw new Error(`Local SQLite execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`); } - if (input.connection?.readonly !== true) { - throw new Error(`Local query execution requires connections.${input.connectionId}.readonly: true.`); - } const pathValue = stringConfigValue(input.connection, 'path'); const urlValue = stringConfigValue(input.connection, 'url'); diff --git a/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts b/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts index fe65920e..93a9739d 100644 --- a/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts +++ b/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts @@ -45,7 +45,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => { warehouse: { driver: 'postgres', url: 'postgres://localhost:5432/warehouse', - readonly: true, }, }, schemas: ['public'], @@ -157,7 +156,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => { warehouse: { driver: 'postgresql', url: 'postgres://localhost:5432/warehouse', - readonly: true, }, }, baseUrl: `http://127.0.0.1:${address.port}`, @@ -186,20 +184,18 @@ describe('createDaemonLiveDatabaseIntrospection', () => { } }); - it('requires a configured read-only postgres connection with a url', async () => { + it('requires a configured postgres connection with a url', async () => { const introspection = createDaemonLiveDatabaseIntrospection({ connections: { warehouse: { driver: 'postgres', - url: 'postgres://localhost:5432/warehouse', - readonly: false, }, }, runJson: vi.fn(async () => daemonResponse), }); await expect(introspection.extractSchema('warehouse')).rejects.toThrow( - 'Local live-database ingest requires connections.warehouse.readonly: true.', + 'Local live-database ingest requires connections.warehouse.url.', ); }); @@ -210,7 +206,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => { warehouse: { driver: 'snowflake', url: 'snowflake://example', - readonly: true, }, }, runJson, diff --git a/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts b/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts index 531c1a66..6c333385 100644 --- a/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts +++ b/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts @@ -162,9 +162,6 @@ function requirePostgresConnection( if (driver !== 'postgres') { throw new Error(`Local live-database ingest cannot run driver "${connection?.driver ?? 'unknown'}".`); } - if (connection?.readonly !== true) { - throw new Error(`Local live-database ingest requires connections.${connectionId}.readonly: true.`); - } if (typeof connection.url !== 'string' || connection.url.trim().length === 0) { throw new Error(`Local live-database ingest requires connections.${connectionId}.url.`); } diff --git a/packages/context/src/ingest/local-stage-ingest.test.ts b/packages/context/src/ingest/local-stage-ingest.test.ts index 5f0ee501..3a0beaa5 100644 --- a/packages/context/src/ingest/local-stage-ingest.test.ts +++ b/packages/context/src/ingest/local-stage-ingest.test.ts @@ -39,7 +39,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise { ' warehouse:', ' driver: postgres', ' url: postgres://localhost:5432/warehouse', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/mcp/local-project-ports.test.ts b/packages/context/src/mcp/local-project-ports.test.ts index b95e4ad1..fab2f076 100644 --- a/packages/context/src/mcp/local-project-ports.test.ts +++ b/packages/context/src/mcp/local-project-ports.test.ts @@ -75,7 +75,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const ports = createLocalProjectMcpContextPorts(project); @@ -89,7 +88,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const connector = testConnector(); const createConnector = vi.fn(async () => connector); @@ -125,7 +123,6 @@ describe('createLocalProjectMcpContextPorts', () => { const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); project.config.connections.warehouse = { driver: 'postgres', - readonly: true, }; project.config.ingest.adapters = ['fake']; project.config.ingest.embeddings = { @@ -633,7 +630,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const shapeOnlyPorts = createLocalProjectMcpContextPorts(project); await shapeOnlyPorts.semanticLayer?.writeSource({ @@ -720,7 +716,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const shapeOnlyPorts = createLocalProjectMcpContextPorts(project); await shapeOnlyPorts.semanticLayer?.writeSource({ @@ -958,7 +953,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://localhost:5432/warehouse', - readonly: true, }; project.config.ingest.adapters = ['live-database']; project.config.llm = { @@ -1034,7 +1028,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; project.config.ingest.adapters = ['live-database']; const ports = createLocalProjectMcpContextPorts(project, { diff --git a/packages/context/src/memory/local-memory.test.ts b/packages/context/src/memory/local-memory.test.ts index e44a5bf1..83b22146 100644 --- a/packages/context/src/memory/local-memory.test.ts +++ b/packages/context/src/memory/local-memory.test.ts @@ -145,7 +145,7 @@ describe('createLocalProjectMemoryCapture', () => { it('captures a semantic-layer source for a named local connection id', async () => { const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; const agentRunner = { runLoop: async ({ toolSet, diff --git a/packages/context/src/project/config.ts b/packages/context/src/project/config.ts index ad5ecb8a..0c345473 100644 --- a/packages/context/src/project/config.ts +++ b/packages/context/src/project/config.ts @@ -69,7 +69,6 @@ export interface KtxProjectScanConfig { export interface KtxProjectConnectionConfig { driver: string; url?: string; - readonly?: boolean; [key: string]: unknown; } diff --git a/packages/context/src/scan/local-scan.test.ts b/packages/context/src/scan/local-scan.test.ts index 6c3e877f..581f02d7 100644 --- a/packages/context/src/scan/local-scan.test.ts +++ b/packages/context/src/scan/local-scan.test.ts @@ -110,7 +110,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1006,7 +1005,6 @@ describe('local scan', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1363,7 +1361,6 @@ describe('local scan', () => { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1396,7 +1393,6 @@ describe('local scan', () => { ' warehouse:', ' driver: mysql', ' url: env:MYSQL_URL', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1432,7 +1428,6 @@ describe('local scan', () => { ' database: analytics', ' username: reader', ' password: env:CLICKHOUSE_PASSWORD', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1468,7 +1463,6 @@ describe('local scan', () => { ' database: analytics', ' username: reader', ' schema: dbo', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/scan/relationship-artifacts.test.ts b/packages/context/src/scan/relationship-artifacts.test.ts index e5fd0190..c366f1b2 100644 --- a/packages/context/src/scan/relationship-artifacts.test.ts +++ b/packages/context/src/scan/relationship-artifacts.test.ts @@ -24,7 +24,6 @@ async function writeWarehouseConfig(projectDir: string): Promise { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/scan/relationship-review-decisions.test.ts b/packages/context/src/scan/relationship-review-decisions.test.ts index 3b92e345..30277c57 100644 --- a/packages/context/src/scan/relationship-review-decisions.test.ts +++ b/packages/context/src/scan/relationship-review-decisions.test.ts @@ -28,7 +28,6 @@ async function createProject(projectDir: string): Promise { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/sl/local-query.test.ts b/packages/context/src/sl/local-query.test.ts index 80209329..4105c9f6 100644 --- a/packages/context/src/sl/local-query.test.ts +++ b/packages/context/src/sl/local-query.test.ts @@ -14,7 +14,7 @@ describe('compileLocalSlQuery', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-query-')); project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -222,7 +222,7 @@ grain: [] expect(queryExecutor.execute).toHaveBeenCalledWith({ connectionId: 'warehouse', projectDir: project.projectDir, - connection: { driver: 'postgres', readonly: true }, + connection: { driver: 'postgres' }, sql: 'select status, count(*) as order_count from public.orders group by status', maxRows: 10, }); @@ -248,7 +248,7 @@ grain: [] }); it('requires connectionId when multiple connections are configured', async () => { - project.config.connections.analytics = { driver: 'bigquery', readonly: true }; + project.config.connections.analytics = { driver: 'bigquery' }; await expect( compileLocalSlQuery(project, { diff --git a/scripts/build-adventureworks-oltp-fixture.mjs b/scripts/build-adventureworks-oltp-fixture.mjs index 4ac2e5ba..2a57fe85 100644 --- a/scripts/build-adventureworks-oltp-fixture.mjs +++ b/scripts/build-adventureworks-oltp-fixture.mjs @@ -231,7 +231,6 @@ async function main() { driver: 'sqlserver', url, schemas: ['dbo', 'HumanResources', 'Person', 'Production', 'Purchasing', 'Sales'], - readonly: true, trustServerCertificate: true, }, now: () => new Date('2026-05-07T00:00:00.000Z'), diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 79e26f74..47fe3f10 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -50,7 +50,6 @@ describe('standalone example docs', () => { config, /path: \.\.\/\.\.\/packages\/context\/test\/fixtures\/relationship-benchmarks\/orbit_style_product_no_declared_constraints\/data\.sqlite/, ); - assert.match(config, /readonly: true/); assert.match(config, /llm_proposals: false/); assert.match(config, /validation_required_for_manifest: true/); }); diff --git a/scripts/installed-live-database-smoke.mjs b/scripts/installed-live-database-smoke.mjs index 653f53b9..88213366 100644 --- a/scripts/installed-live-database-smoke.mjs +++ b/scripts/installed-live-database-smoke.mjs @@ -92,7 +92,6 @@ export function buildKtxYaml(postgresUrl) { ' warehouse:', ' driver: postgres', ` url: "${postgresUrl}"`, - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/scripts/installed-live-database-smoke.test.mjs b/scripts/installed-live-database-smoke.test.mjs index c62e98b0..0a45cdf5 100644 --- a/scripts/installed-live-database-smoke.test.mjs +++ b/scripts/installed-live-database-smoke.test.mjs @@ -59,7 +59,6 @@ describe('installed live-database artifact smoke helpers', () => { ' warehouse:', ' driver: postgres', ' url: "postgresql://ktx:postgres@127.0.0.1:15432/warehouse"', // pragma: allowlist secret - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index e7998c58..ee785c27 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -646,7 +646,6 @@ try { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5',