mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-10 08:05:14 +02:00
merge origin/main into fix-precommit-all-files
This commit is contained in:
Andrey Avtomonov
commit
0f7b8666cf
63 changed files with 953 additions and 303 deletions
|
|
@ -63,8 +63,7 @@ agents.
|
|||
"connections": [
|
||||
{
|
||||
"id": "my-warehouse",
|
||||
"driver": "postgres",
|
||||
"readonly": false
|
||||
"driver": "postgres"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ Agents should configure and ingest context sources in this order:
|
|||
| Field | Required | Description |
|
||||
|-------|----------|-------------|
|
||||
| `driver` | Yes | Source adapter: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, or `notion` |
|
||||
| `readonly` | Strongly recommended | Marks the source as read-only for KTX |
|
||||
| `source_dir` | For local file sources | Absolute or project-relative source directory |
|
||||
| `repo_url` | For Git-hosted sources | Git repository URL |
|
||||
| `branch` | No | Git branch to read |
|
||||
|
|
@ -49,7 +48,6 @@ connections:
|
|||
my-dbt:
|
||||
driver: dbt
|
||||
source_dir: /path/to/dbt/project
|
||||
readonly: true
|
||||
```
|
||||
|
||||
For a Git-hosted project:
|
||||
|
|
@ -62,7 +60,6 @@ connections:
|
|||
branch: main
|
||||
path: analytics/dbt # For monorepos
|
||||
auth_token_ref: env:GITHUB_TOKEN
|
||||
readonly: true
|
||||
```
|
||||
|
||||
### Authentication
|
||||
|
|
@ -110,7 +107,6 @@ connections:
|
|||
branch: main
|
||||
path: dbt_metrics # Subdirectory for monorepos
|
||||
auth_token_ref: env:GITHUB_TOKEN
|
||||
readonly: true
|
||||
```
|
||||
|
||||
For a local path:
|
||||
|
|
@ -157,7 +153,6 @@ connections:
|
|||
branch: main
|
||||
path: analytics # Subdirectory for monorepos
|
||||
auth_token_ref: env:GITHUB_TOKEN
|
||||
readonly: true
|
||||
```
|
||||
|
||||
For a local path:
|
||||
|
|
@ -219,7 +214,6 @@ connections:
|
|||
syncEnabled:
|
||||
"3": true
|
||||
syncMode: ONLY # Only ingest mapped databases
|
||||
readonly: true
|
||||
```
|
||||
|
||||
### Authentication
|
||||
|
|
@ -276,7 +270,6 @@ connections:
|
|||
mappings:
|
||||
connectionMappings:
|
||||
postgres_connection: postgres-main # Looker conn → KTX conn
|
||||
readonly: true
|
||||
```
|
||||
|
||||
### Authentication
|
||||
|
|
@ -329,7 +322,6 @@ connections:
|
|||
crawl_mode: selected_roots
|
||||
root_page_ids:
|
||||
- "abc123def456..."
|
||||
readonly: true
|
||||
```
|
||||
|
||||
For crawling all accessible pages:
|
||||
|
|
@ -340,7 +332,6 @@ connections:
|
|||
driver: notion
|
||||
auth_token_ref: env:NOTION_TOKEN
|
||||
crawl_mode: all_accessible
|
||||
readonly: true
|
||||
```
|
||||
|
||||
### Authentication
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ Agents should prefer environment or file references over literal secrets.
|
|||
| `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` |
|
||||
| `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, ClickHouse, SQL Server | Field-by-field connection values |
|
||||
| `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan |
|
||||
| `readonly` | Strongly recommended | all primary sources | Marks the connection as read-only in KTX config |
|
||||
| `historicSql` | No | supported warehouses | Enables query-history ingestion when the warehouse supports it |
|
||||
| `path` | Yes for path-style SQLite | SQLite | Local SQLite database path or `env:NAME` reference |
|
||||
|
||||
|
|
@ -37,7 +36,6 @@ connections:
|
|||
driver: postgres
|
||||
url: env:DATABASE_URL
|
||||
schema: public
|
||||
readonly: true
|
||||
```
|
||||
|
||||
Or with individual fields:
|
||||
|
|
@ -55,7 +53,6 @@ connections:
|
|||
- public
|
||||
- analytics
|
||||
ssl: true
|
||||
readonly: true
|
||||
```
|
||||
|
||||
### Authentication
|
||||
|
|
@ -123,7 +120,6 @@ connections:
|
|||
username: KTX_SERVICE
|
||||
password: env:SNOWFLAKE_PASSWORD
|
||||
role: ANALYST
|
||||
readonly: true
|
||||
```
|
||||
|
||||
For multiple schemas:
|
||||
|
|
@ -196,7 +192,6 @@ connections:
|
|||
credentials_json: file:~/.config/gcloud/bq-service-account.json
|
||||
dataset_id: analytics
|
||||
location: US
|
||||
readonly: true
|
||||
```
|
||||
|
||||
For multiple datasets:
|
||||
|
|
@ -269,7 +264,6 @@ connections:
|
|||
my-clickhouse:
|
||||
driver: clickhouse
|
||||
url: http://localhost:8123/analytics
|
||||
readonly: true
|
||||
```
|
||||
|
||||
Or with individual fields:
|
||||
|
|
@ -284,7 +278,6 @@ connections:
|
|||
username: default
|
||||
password: env:CH_PASSWORD
|
||||
ssl: false
|
||||
readonly: true
|
||||
```
|
||||
|
||||
### Authentication
|
||||
|
|
@ -328,7 +321,6 @@ connections:
|
|||
my-mysql:
|
||||
driver: mysql
|
||||
url: env:MYSQL_DATABASE_URL
|
||||
readonly: true
|
||||
```
|
||||
|
||||
Or with individual fields:
|
||||
|
|
@ -343,7 +335,6 @@ connections:
|
|||
username: ktx_reader
|
||||
password: env:MYSQL_PASSWORD
|
||||
ssl: true
|
||||
readonly: true
|
||||
```
|
||||
|
||||
### Authentication
|
||||
|
|
@ -387,7 +378,6 @@ connections:
|
|||
my-sqlserver:
|
||||
driver: sqlserver
|
||||
url: env:SQLSERVER_DATABASE_URL
|
||||
readonly: true
|
||||
```
|
||||
|
||||
Or with individual fields:
|
||||
|
|
@ -403,7 +393,6 @@ connections:
|
|||
password: env:MSSQL_PASSWORD
|
||||
schema: dbo
|
||||
trustServerCertificate: true
|
||||
readonly: true
|
||||
```
|
||||
|
||||
For multiple schemas:
|
||||
|
|
@ -455,7 +444,6 @@ connections:
|
|||
my-sqlite:
|
||||
driver: sqlite
|
||||
path: ./data/warehouse.sqlite
|
||||
readonly: true
|
||||
```
|
||||
|
||||
Path supports multiple formats:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ project: local-warehouse
|
|||
connections:
|
||||
warehouse:
|
||||
driver: postgres
|
||||
readonly: true
|
||||
storage:
|
||||
state: sqlite
|
||||
search: sqlite-fts5
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ connections:
|
|||
orbit:
|
||||
driver: sqlite
|
||||
path: ../../packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite
|
||||
readonly: true
|
||||
storage:
|
||||
state: sqlite
|
||||
search: sqlite-fts5
|
||||
|
|
|
|||
|
|
@ -316,6 +316,10 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
|
|||
registerIngestCommands(program, context, {
|
||||
runIngestWithProgress: async (ingestArgs, ingestIo, ingestDeps, defaultRunIngest) =>
|
||||
await (ingestDeps.ingest ?? defaultRunIngest)(ingestArgs, ingestIo),
|
||||
runTextIngest: async (textIngestArgs, ingestIo, ingestDeps) => {
|
||||
const { runKtxTextIngest } = await import('./text-ingest.js');
|
||||
return await (ingestDeps.textIngest ?? runKtxTextIngest)(textIngestArgs, ingestIo);
|
||||
},
|
||||
});
|
||||
registerScanCommands(program, context);
|
||||
registerWikiCommands(program, context);
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import type { KtxScanArgs } from './scan.js';
|
|||
import type { KtxSetupArgs } from './setup.js';
|
||||
import type { KtxSlArgs } from './sl.js';
|
||||
import { profileMark, profileSpan } from './startup-profile.js';
|
||||
import type { KtxTextIngestArgs } from './text-ingest.js';
|
||||
|
||||
profileMark('module:cli-runtime');
|
||||
|
||||
|
|
@ -30,6 +31,7 @@ export interface KtxCliDeps {
|
|||
connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise<number>;
|
||||
doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise<number>;
|
||||
ingest?: (args: KtxIngestArgs, io: KtxCliIo) => Promise<number>;
|
||||
textIngest?: (args: KtxTextIngestArgs, io: KtxCliIo) => Promise<number>;
|
||||
runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise<number>;
|
||||
scan?: (args: KtxScanArgs, io: KtxCliIo) => Promise<number>;
|
||||
knowledge?: (args: KtxKnowledgeArgs, io: KtxCliIo) => Promise<number>;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import { resolve } from 'node:path';
|
||||
import { type Command, Option } from '@commander-js/extra-typings';
|
||||
import { type KtxCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import { collectOption, type KtxCliCommandContext, type OutputModeOptions, resolveCommandProjectDir } from '../cli-program.js';
|
||||
import type { KtxCliDeps, KtxCliIo } from '../index.js';
|
||||
import type { KtxIngestArgs, KtxIngestOutputMode } from '../ingest.js';
|
||||
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
import type { KtxTextIngestArgs } from '../text-ingest.js';
|
||||
|
||||
profileMark('module:commands/ingest-commands');
|
||||
|
||||
|
|
@ -15,6 +16,7 @@ interface IngestCommandOptions {
|
|||
deps: KtxCliDeps,
|
||||
defaultRunIngest: (args: KtxIngestArgs, io: KtxCliIo) => Promise<number>,
|
||||
) => Promise<number>;
|
||||
runTextIngest: (args: KtxTextIngestArgs, io: KtxCliIo, deps: KtxCliDeps) => Promise<number>;
|
||||
}
|
||||
|
||||
function outputMode(options: OutputModeOptions): KtxIngestOutputMode {
|
||||
|
|
@ -101,6 +103,33 @@ export function registerIngestCommands(
|
|||
);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('text')
|
||||
.description('Ingest free-form text artifacts into KTX memory')
|
||||
.argument('[files...]', 'Files to ingest; use - to read one item from stdin')
|
||||
.option('--text <content>', 'Text content to ingest; repeat for a batch', collectOption, [])
|
||||
.option('--connection-id <connectionId>', 'Optional KTX connection id for semantic-layer capture')
|
||||
.option('--user-id <id>', 'Memory user id for capture attribution', 'local-cli')
|
||||
.option('--json', 'Print JSON output')
|
||||
.option('--fail-fast', 'Stop after the first failed text item', false)
|
||||
.action(async (files: string[], options, command) => {
|
||||
context.setExitCode(
|
||||
await commandOptions.runTextIngest(
|
||||
{
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
texts: options.text,
|
||||
files,
|
||||
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
||||
userId: options.userId,
|
||||
json: options.json === true,
|
||||
failFast: options.failFast === true,
|
||||
},
|
||||
context.io,
|
||||
context.deps,
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
ingest
|
||||
.command('status')
|
||||
.description('Print status for the latest or selected stored local ingest run or report file')
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ describe('runKtxConnection', () => {
|
|||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeConnections(projectDir, {
|
||||
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true },
|
||||
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' },
|
||||
docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' },
|
||||
});
|
||||
const io = makeIo();
|
||||
|
|
@ -123,7 +123,7 @@ describe('runKtxConnection', () => {
|
|||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeConnections(projectDir, {
|
||||
warehouse: { driver: 'sqlite', readonly: true },
|
||||
warehouse: { driver: 'sqlite' },
|
||||
});
|
||||
const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']);
|
||||
const createScanConnector = vi.fn(async () => connector);
|
||||
|
|
@ -202,7 +202,7 @@ describe('runKtxConnection', () => {
|
|||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
await writeConnections(projectDir, {
|
||||
warehouse: { driver: 'sqlite', readonly: true },
|
||||
warehouse: { driver: 'sqlite' },
|
||||
});
|
||||
const cleanup = vi.fn(async () => undefined);
|
||||
const connector: KtxScanConnector = {
|
||||
|
|
|
|||
|
|
@ -158,6 +158,30 @@ describe('renderContextBuildView', () => {
|
|||
expect(output).toContain('dbt-main');
|
||||
});
|
||||
|
||||
it('supports text ingest labels while preserving the shared compact progress view', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'text-1', driver: 'text', operation: 'source-ingest', debugCommand: '', steps: ['memory-update'] },
|
||||
{ connectionId: 'schema.md', driver: 'text', operation: 'source-ingest', debugCommand: '', steps: ['memory-update'] },
|
||||
]);
|
||||
state.contextSources[0].status = 'running';
|
||||
state.contextSources[0].detailLine = 'capturing...';
|
||||
|
||||
const output = renderContextBuildView(state, {
|
||||
styled: false,
|
||||
title: 'Ingesting text memory',
|
||||
contextGroupLabel: 'Texts',
|
||||
sourceIngestRunningText: 'capturing...',
|
||||
completedItemName: { singular: 'text', plural: 'texts' },
|
||||
});
|
||||
|
||||
expect(output).toContain('Ingesting text memory');
|
||||
expect(output).toContain('Texts:');
|
||||
expect(output).toContain('text-1');
|
||||
expect(output).toContain('schema.md');
|
||||
expect(output).toContain('capturing...');
|
||||
expect(output).not.toContain('Context sources:');
|
||||
});
|
||||
|
||||
it('renders header with total elapsed time when set', () => {
|
||||
const state = initViewState([
|
||||
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
|
||||
|
|
|
|||
|
|
@ -65,6 +65,24 @@ export interface ContextBuildSourceProgressUpdate {
|
|||
summaryText?: string;
|
||||
}
|
||||
|
||||
interface CompletedItemName {
|
||||
singular: string;
|
||||
plural: string;
|
||||
}
|
||||
|
||||
interface ContextBuildRenderOptions {
|
||||
styled?: boolean;
|
||||
showHint?: boolean;
|
||||
hintText?: string;
|
||||
projectDir?: string;
|
||||
title?: string;
|
||||
primaryGroupLabel?: string;
|
||||
contextGroupLabel?: string;
|
||||
scanRunningText?: string;
|
||||
sourceIngestRunningText?: string;
|
||||
completedItemName?: CompletedItemName;
|
||||
}
|
||||
|
||||
export interface ContextBuildDeps {
|
||||
executeTarget?: typeof executePublicIngestTarget;
|
||||
now?: () => number;
|
||||
|
|
@ -148,7 +166,7 @@ function staleProgressText(target: ContextBuildTargetState, styled: boolean): st
|
|||
return styled ? dim(text) : text;
|
||||
}
|
||||
|
||||
function targetDetail(target: ContextBuildTargetState, styled: boolean): string {
|
||||
function targetDetail(target: ContextBuildTargetState, styled: boolean, options: ContextBuildRenderOptions): string {
|
||||
if (target.status === 'done') {
|
||||
const parts: string[] = [];
|
||||
if (target.summaryText) parts.push(target.summaryText);
|
||||
|
|
@ -162,7 +180,9 @@ function targetDetail(target: ContextBuildTargetState, styled: boolean): string
|
|||
if (target.status === 'running') {
|
||||
const percent = extractPercent(target.detailLine);
|
||||
const progressText = target.detailLine?.replace(/^\[\d+%\]\s*/, '')
|
||||
?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...');
|
||||
?? (target.target.operation === 'scan'
|
||||
? (options.scanRunningText ?? 'scanning...')
|
||||
: (options.sourceIngestRunningText ?? 'ingesting...'));
|
||||
const elapsed = target.elapsedMs > 0 ? `(${formatDuration(target.elapsedMs)})` : null;
|
||||
const parts: string[] = [];
|
||||
if (percent !== null) {
|
||||
|
|
@ -182,8 +202,14 @@ function columnWidth(state: ContextBuildViewState): number {
|
|||
return Math.max(12, ...all.map((t) => t.target.connectionId.length)) + 2;
|
||||
}
|
||||
|
||||
function renderTargetLine(target: ContextBuildTargetState, frame: number, styled: boolean, width: number): string {
|
||||
return ` ${statusIcon(target.status, frame, styled)} ${target.target.connectionId.padEnd(width)} ${targetDetail(target, styled)}`;
|
||||
function renderTargetLine(
|
||||
target: ContextBuildTargetState,
|
||||
frame: number,
|
||||
styled: boolean,
|
||||
width: number,
|
||||
options: ContextBuildRenderOptions,
|
||||
): string {
|
||||
return ` ${statusIcon(target.status, frame, styled)} ${target.target.connectionId.padEnd(width)} ${targetDetail(target, styled, options)}`;
|
||||
}
|
||||
|
||||
function renderTargetGroup(
|
||||
|
|
@ -192,9 +218,10 @@ function renderTargetGroup(
|
|||
frame: number,
|
||||
styled: boolean,
|
||||
width: number,
|
||||
options: ContextBuildRenderOptions,
|
||||
): string[] {
|
||||
if (targets.length === 0) return [];
|
||||
return ['', ` ${label}:`, ...targets.map((t) => renderTargetLine(t, frame, styled, width))];
|
||||
return ['', ` ${label}:`, ...targets.map((t) => renderTargetLine(t, frame, styled, width, options))];
|
||||
}
|
||||
|
||||
function resumeCommand(projectDir?: string): string {
|
||||
|
|
@ -203,7 +230,7 @@ function resumeCommand(projectDir?: string): string {
|
|||
|
||||
export function renderContextBuildView(
|
||||
state: ContextBuildViewState,
|
||||
options: { styled?: boolean; showHint?: boolean; hintText?: string; projectDir?: string } = {},
|
||||
options: ContextBuildRenderOptions = {},
|
||||
): string {
|
||||
const styled = options.styled ?? true;
|
||||
const width = columnWidth(state);
|
||||
|
|
@ -213,7 +240,7 @@ export function renderContextBuildView(
|
|||
const hasActive = allTargets.some((t) => t.status === 'running' || t.status === 'queued');
|
||||
const allDone = totalCount > 0 && !hasActive;
|
||||
|
||||
const headerParts = ['Building KTX context'];
|
||||
const headerParts = [options.title ?? 'Building KTX context'];
|
||||
if (totalCount > 0) {
|
||||
const progressParts: string[] = [`${doneCount}/${totalCount}`];
|
||||
if (state.totalElapsedMs > 0) progressParts.push(formatDuration(state.totalElapsedMs));
|
||||
|
|
@ -229,13 +256,14 @@ export function renderContextBuildView(
|
|||
header,
|
||||
separator,
|
||||
...(options.projectDir ? [` Project: ${options.projectDir}`] : []),
|
||||
...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width),
|
||||
...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width),
|
||||
...renderTargetGroup(options.primaryGroupLabel ?? 'Primary sources', state.primarySources, state.frame, styled, width, options),
|
||||
...renderTargetGroup(options.contextGroupLabel ?? 'Context sources', state.contextSources, state.frame, styled, width, options),
|
||||
'',
|
||||
];
|
||||
|
||||
if (allDone && state.totalElapsedMs > 0) {
|
||||
const sourcesLabel = totalCount === 1 ? '1 source' : `${totalCount} sources`;
|
||||
const itemName = options.completedItemName ?? { singular: 'source', plural: 'sources' };
|
||||
const sourcesLabel = totalCount === 1 ? `1 ${itemName.singular}` : `${totalCount} ${itemName.plural}`;
|
||||
const summary = ` Done in ${formatDuration(state.totalElapsedMs)} · ${sourcesLabel} processed`;
|
||||
lines.push(styled ? green(summary) : summary);
|
||||
lines.push('');
|
||||
|
|
|
|||
|
|
@ -57,7 +57,6 @@ function demoConfig(databasePath: string): string {
|
|||
` ${DEMO_CONNECTION_ID}:`,
|
||||
' driver: sqlite',
|
||||
` path: ${JSON.stringify(databasePath)}`,
|
||||
' readonly: true',
|
||||
'storage:',
|
||||
' state: sqlite',
|
||||
' search: sqlite-fts5',
|
||||
|
|
|
|||
|
|
@ -275,7 +275,6 @@ describe('runKtxDoctor', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:WAREHOUSE_DATABASE_URL',
|
||||
' readonly: true',
|
||||
' historicSql:',
|
||||
' enabled: true',
|
||||
' dialect: postgres',
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
|
|||
it('passes when no Postgres historic-SQL connections are enabled', async () => {
|
||||
const checks = await runPostgresHistoricSqlDoctorChecks(
|
||||
projectWithConnections({
|
||||
warehouse: { driver: 'sqlite', path: './warehouse.db', readonly: true },
|
||||
warehouse: { driver: 'sqlite', path: './warehouse.db' },
|
||||
}),
|
||||
{
|
||||
postgresHistoricSqlProbe: vi.fn<PostgresHistoricSqlDoctorProbe>(),
|
||||
|
|
@ -53,7 +53,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
|
|||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
}),
|
||||
|
|
@ -66,7 +65,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
|
|||
connection: {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
env: process.env,
|
||||
|
|
@ -87,7 +85,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
|
|||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
}),
|
||||
|
|
@ -119,7 +116,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
|
|||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
}),
|
||||
|
|
@ -154,7 +150,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
|
|||
warehouse: {
|
||||
driver: 'mysql',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
}),
|
||||
|
|
@ -180,7 +175,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
|
|||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'env:WAREHOUSE_DATABASE_URL',
|
||||
readonly: true,
|
||||
historicSql: { enabled: true, dialect: 'postgres' },
|
||||
},
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -86,8 +86,9 @@ async function defaultPostgresHistoricSqlProbe(
|
|||
const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] =
|
||||
await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]);
|
||||
|
||||
const inputDriver = input.connection.driver ?? 'unknown';
|
||||
if (!isKtxPostgresConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection.driver ?? 'unknown'}"`);
|
||||
throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
|
||||
const client = new KtxPostgresHistoricSqlQueryClient({
|
||||
|
|
|
|||
|
|
@ -734,14 +734,73 @@ describe('runKtxCli', () => {
|
|||
expect(testIo.stdout()).toContain('Usage: ktx ingest [options] [command]');
|
||||
expect(testIo.stdout()).toContain('Run or inspect local ingest memory-flow output');
|
||||
expect(testIo.stdout()).toContain('run');
|
||||
expect(testIo.stdout()).toContain('text');
|
||||
expect(testIo.stdout()).toContain('status');
|
||||
expect(testIo.stdout()).toContain('watch');
|
||||
expect(testIo.stdout()).toContain('replay');
|
||||
expect(testIo.stdout()).not.toContain('--manifest');
|
||||
expect(testIo.stdout()).not.toContain('--all');
|
||||
expect(testIo.stderr()).toBe('');
|
||||
expect(ingest).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('routes text memory ingest through Commander without exposing chat ids', async () => {
|
||||
const textIngest = vi.fn(async () => 0);
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxCli(
|
||||
[
|
||||
'--project-dir',
|
||||
tempDir,
|
||||
'ingest',
|
||||
'text',
|
||||
'--text',
|
||||
'Revenue means gross receipts.',
|
||||
'--text',
|
||||
'Orders are completed purchases.',
|
||||
'--connection-id',
|
||||
'warehouse',
|
||||
'--user-id',
|
||||
'agent',
|
||||
'--json',
|
||||
'--fail-fast',
|
||||
],
|
||||
testIo.io,
|
||||
{ textIngest },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(textIngest).toHaveBeenCalledWith(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
texts: ['Revenue means gross receipts.', 'Orders are completed purchases.'],
|
||||
files: [],
|
||||
connectionId: 'warehouse',
|
||||
userId: 'agent',
|
||||
json: true,
|
||||
failFast: true,
|
||||
},
|
||||
testIo.io,
|
||||
);
|
||||
expect(testIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('documents text ingest inputs without a manifest option', async () => {
|
||||
const textIngest = vi.fn(async () => 0);
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(runKtxCli(['ingest', 'text', '--help'], testIo.io, { textIngest })).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Usage: ktx ingest text [options] [files...]');
|
||||
expect(testIo.stdout()).toContain('--text <content>');
|
||||
expect(testIo.stdout()).toContain('--connection-id <connectionId>');
|
||||
expect(testIo.stdout()).toContain('--user-id <id>');
|
||||
expect(testIo.stdout()).toContain('--fail-fast');
|
||||
expect(testIo.stdout()).not.toContain('--manifest');
|
||||
expect(textIngest).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('routes ingest run at the top level and rejects removed dev ingest', async () => {
|
||||
const runIo = makeIo();
|
||||
const devRunIo = makeIo();
|
||||
|
|
|
|||
|
|
@ -45,7 +45,6 @@ describe('CLI local ingest adapters', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:WAREHOUSE_DATABASE_URL',
|
||||
' readonly: true',
|
||||
' historicSql:',
|
||||
' enabled: true',
|
||||
' dialect: postgres',
|
||||
|
|
@ -76,7 +75,6 @@ describe('CLI local ingest adapters', () => {
|
|||
'connections:',
|
||||
' bq:',
|
||||
' driver: bigquery',
|
||||
' readonly: true',
|
||||
' dataset_id: analytics',
|
||||
' location: us',
|
||||
' credentials_json: \'{"project_id":"demo-project"}\'',
|
||||
|
|
@ -110,7 +108,6 @@ describe('CLI local ingest adapters', () => {
|
|||
'connections:',
|
||||
' sf:',
|
||||
' driver: snowflake',
|
||||
' readonly: true',
|
||||
' account: acct',
|
||||
' warehouse: wh',
|
||||
' database: ANALYTICS',
|
||||
|
|
|
|||
|
|
@ -190,10 +190,9 @@ function enabledHistoricSqlDialect(connection: unknown): 'postgres' | 'bigquery'
|
|||
|
||||
function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, connectionId: string) {
|
||||
const connection = project.config.connections[connectionId] as KtxPostgresConnectionConfig | undefined;
|
||||
const inputDriver = connection?.driver ?? 'unknown';
|
||||
if (!isKtxPostgresConnectionConfig(connection)) {
|
||||
throw new Error(
|
||||
`Historic SQL local ingest requires a Postgres connection, got ${String(connection?.driver ?? 'unknown')}`,
|
||||
);
|
||||
throw new Error(`Historic SQL local ingest requires a Postgres connection, got ${String(inputDriver)}`);
|
||||
}
|
||||
return {
|
||||
async executeQuery(sql: string, params?: unknown[]) {
|
||||
|
|
@ -212,10 +211,9 @@ function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, conn
|
|||
|
||||
function createEphemeralBigQueryHistoricSqlClient(project: KtxLocalProject, connectionId: string) {
|
||||
const connection = project.config.connections[connectionId] as KtxBigQueryConnectionConfig | undefined;
|
||||
const inputDriver = connection?.driver ?? 'unknown';
|
||||
if (!isKtxBigQueryConnectionConfig(connection)) {
|
||||
throw new Error(
|
||||
`Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`,
|
||||
);
|
||||
throw new Error(`Historic SQL local ingest requires a BigQuery connection, got ${String(inputDriver)}`);
|
||||
}
|
||||
return {
|
||||
async executeQuery(query: string) {
|
||||
|
|
@ -243,10 +241,9 @@ async function createEphemeralSnowflakeHistoricSqlClient(
|
|||
connectorModule: SnowflakeConnectorModule,
|
||||
) {
|
||||
const connection = project.config.connections[connectionId];
|
||||
const inputDriver = connection?.driver ?? 'unknown';
|
||||
if (!connectorModule.isKtxSnowflakeConnectionConfig(connection)) {
|
||||
throw new Error(
|
||||
`Historic SQL local ingest requires a Snowflake connection, got ${String(connection?.driver ?? 'unknown')}`,
|
||||
);
|
||||
throw new Error(`Historic SQL local ingest requires a Snowflake connection, got ${String(inputDriver)}`);
|
||||
}
|
||||
return {
|
||||
async executeQuery(query: string) {
|
||||
|
|
@ -308,10 +305,9 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli
|
|||
}
|
||||
|
||||
if (dialect === 'bigquery') {
|
||||
const inputDriver = connection?.driver ?? 'unknown';
|
||||
if (!isKtxBigQueryConnectionConfig(connection)) {
|
||||
throw new Error(
|
||||
`Historic SQL local ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`,
|
||||
);
|
||||
throw new Error(`Historic SQL local ingest requires a BigQuery connection, got ${String(inputDriver)}`);
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
|
|
|
|||
|
|
@ -49,7 +49,6 @@ describe('createKtxCliScanConnector', () => {
|
|||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: warehouse.db',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -72,7 +71,6 @@ describe('createKtxCliScanConnector', () => {
|
|||
' warehouse:',
|
||||
' driver: bigquery',
|
||||
' dataset_id: analytics',
|
||||
' readonly: true',
|
||||
' max_bytes_billed: "987654321"',
|
||||
'',
|
||||
].join('\n'),
|
||||
|
|
@ -123,7 +121,6 @@ describe('createKtxCliScanConnector', () => {
|
|||
' warehouse:',
|
||||
' type: postgres',
|
||||
' url: postgresql://example/db',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
|
|||
|
|
@ -861,7 +861,6 @@ describe('runKtxScan', () => {
|
|||
' warehouse:',
|
||||
' driver: mysql',
|
||||
' url: env:MYSQL_URL',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -910,7 +909,6 @@ describe('runKtxScan', () => {
|
|||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: warehouse.db',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -968,7 +966,6 @@ describe('runKtxScan', () => {
|
|||
' database: analytics',
|
||||
' username: reader',
|
||||
' password: env:POSTGRES_PASSWORD',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -1035,7 +1032,6 @@ describe('runKtxScan', () => {
|
|||
' database: analytics',
|
||||
' username: reader',
|
||||
' password: env:CLICKHOUSE_PASSWORD',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -1087,7 +1083,6 @@ describe('runKtxScan', () => {
|
|||
' database: analytics',
|
||||
' username: reader',
|
||||
' schema: dbo',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -1153,7 +1148,6 @@ describe('runKtxScan', () => {
|
|||
' dataset_id: analytics',
|
||||
' credentials_json: env:BIGQUERY_CREDENTIALS_JSON',
|
||||
' location: US',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -1222,7 +1216,6 @@ describe('runKtxScan', () => {
|
|||
' database: ANALYTICS',
|
||||
' schema_name: PUBLIC',
|
||||
' username: reader',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
|
|||
|
|
@ -218,7 +218,6 @@ describe('setup databases step', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -281,7 +280,6 @@ describe('setup databases step', () => {
|
|||
expect(config.connections['postgres-warehouse']).toEqual({
|
||||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
readonly: true,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -542,7 +540,6 @@ describe('setup databases step', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
' readonly: true',
|
||||
'setup:',
|
||||
' database_connection_ids:',
|
||||
' - warehouse',
|
||||
|
|
@ -583,7 +580,6 @@ describe('setup databases step', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
' readonly: true',
|
||||
'setup:',
|
||||
' database_connection_ids:',
|
||||
' - warehouse',
|
||||
|
|
@ -698,7 +694,6 @@ describe('setup databases step', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
' readonly: true',
|
||||
'setup:',
|
||||
' database_connection_ids:',
|
||||
' - warehouse',
|
||||
|
|
@ -843,7 +838,6 @@ describe('setup databases step', () => {
|
|||
port: 5432,
|
||||
database: 'analytics',
|
||||
username: 'readonly',
|
||||
readonly: true,
|
||||
});
|
||||
expect(connection.password).toMatch(/^file:/);
|
||||
const secretPath = join(tempDir, '.ktx/secrets/postgres-warehouse-password');
|
||||
|
|
@ -998,7 +992,6 @@ describe('setup databases step', () => {
|
|||
expect(config.connections['postgres-warehouse']).toMatchObject({
|
||||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
readonly: true,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1115,7 +1108,6 @@ describe('setup databases step', () => {
|
|||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
schemas: ['public'],
|
||||
readonly: true,
|
||||
});
|
||||
expect(config.setup).toEqual({
|
||||
database_connection_ids: ['warehouse'],
|
||||
|
|
@ -1153,7 +1145,6 @@ describe('setup databases step', () => {
|
|||
expect(config.connections.warehouse).toEqual({
|
||||
driver: 'sqlite',
|
||||
path: './warehouse.sqlite',
|
||||
readonly: true,
|
||||
});
|
||||
expect(config.setup).toEqual({
|
||||
database_connection_ids: ['warehouse'],
|
||||
|
|
@ -1170,7 +1161,6 @@ describe('setup databases step', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
' readonly: true',
|
||||
' analytics:',
|
||||
' driver: snowflake',
|
||||
' authMethod: password',
|
||||
|
|
@ -1180,7 +1170,6 @@ describe('setup databases step', () => {
|
|||
' schema_name: PUBLIC',
|
||||
' username: reader',
|
||||
' password: env:SNOWFLAKE_PASSWORD',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -1443,7 +1432,6 @@ describe('setup databases step', () => {
|
|||
' driver: bigquery',
|
||||
' dataset_id: analytics',
|
||||
' credentials_json: env:BIGQUERY_CREDENTIALS_JSON',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -1492,7 +1480,6 @@ describe('setup databases step', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
|
|||
|
|
@ -593,7 +593,6 @@ async function buildFieldsConnectionConfig(input: {
|
|||
username,
|
||||
...(passwordRef ? { password: passwordRef } : {}),
|
||||
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
|
||||
readonly: true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -615,7 +614,6 @@ async function buildPastedUrlConnectionConfig(input: {
|
|||
driver: input.driver,
|
||||
url,
|
||||
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
|
||||
readonly: true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -629,7 +627,6 @@ async function buildPastedUrlConnectionConfig(input: {
|
|||
driver: input.driver,
|
||||
url: ref,
|
||||
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
|
||||
readonly: true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -637,7 +634,6 @@ async function buildPastedUrlConnectionConfig(input: {
|
|||
driver: input.driver,
|
||||
url,
|
||||
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
|
||||
readonly: true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -661,14 +657,12 @@ async function buildUrlConnectionConfig(input: {
|
|||
driver: input.driver,
|
||||
url: ref,
|
||||
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
|
||||
readonly: true,
|
||||
};
|
||||
}
|
||||
return {
|
||||
driver: input.driver,
|
||||
url,
|
||||
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
|
||||
readonly: true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -706,7 +700,7 @@ async function buildConnectionConfig(input: {
|
|||
'SQLite database file\nEnter a relative or absolute path, for example ./warehouse.sqlite.',
|
||||
));
|
||||
if (path === undefined) return 'back';
|
||||
return path ? { driver: 'sqlite', path, readonly: true } : null;
|
||||
return path ? { driver: 'sqlite', path } : null;
|
||||
}
|
||||
if (driver === 'postgres' || driver === 'mysql' || driver === 'clickhouse' || driver === 'sqlserver') {
|
||||
return await buildUrlConnectionConfig({ driver, connectionId: input.connectionId, args, prompts });
|
||||
|
|
@ -728,7 +722,6 @@ async function buildConnectionConfig(input: {
|
|||
dataset_id: datasetId,
|
||||
credentials_json: normalizeFileReference(credentialsPath),
|
||||
...(location ? { location } : {}),
|
||||
readonly: true,
|
||||
};
|
||||
}
|
||||
if (driver === 'snowflake') {
|
||||
|
|
@ -767,7 +760,6 @@ async function buildConnectionConfig(input: {
|
|||
username,
|
||||
password: passwordRef,
|
||||
...(role ? { role } : {}),
|
||||
readonly: true,
|
||||
};
|
||||
}
|
||||
throw new Error(`Unsupported database driver: ${driver}`);
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ describe('setup sources step', () => {
|
|||
...config,
|
||||
connections: {
|
||||
...config.connections,
|
||||
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true },
|
||||
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' },
|
||||
},
|
||||
setup: {
|
||||
...config.setup,
|
||||
|
|
@ -455,7 +455,6 @@ describe('setup sources step', () => {
|
|||
driver: 'snowflake',
|
||||
account: 'acme',
|
||||
database: 'analytics',
|
||||
readonly: true,
|
||||
});
|
||||
|
||||
const cases: Array<{
|
||||
|
|
|
|||
|
|
@ -170,7 +170,6 @@ describe('setup status', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -192,7 +191,6 @@ describe('setup status', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
@ -1373,7 +1371,6 @@ describe('setup status', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DEMO_DATABASE_URL',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ joins: []
|
|||
it('runs sl query and prints SQL output', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
|
||||
project.config.connections.warehouse = { driver: 'postgres' };
|
||||
await project.fileStore.writeFile(
|
||||
'semantic-layer/warehouse/orders.yaml',
|
||||
`name: orders
|
||||
|
|
@ -247,7 +247,7 @@ joins: []
|
|||
it('runs sl query from a JSON query file', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
|
||||
project.config.connections.warehouse = { driver: 'postgres' };
|
||||
await project.fileStore.writeFile(
|
||||
'semantic-layer/warehouse/orders.yaml',
|
||||
`name: orders
|
||||
|
|
@ -314,7 +314,7 @@ joins: []
|
|||
it('creates default sl query compute through the managed runtime helper', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
|
||||
project.config.connections.warehouse = { driver: 'postgres' };
|
||||
await project.fileStore.writeFile(
|
||||
'semantic-layer/warehouse/orders.yaml',
|
||||
`name: orders
|
||||
|
|
@ -375,7 +375,7 @@ joins: []
|
|||
it('executes sl query through the injected query executor', async () => {
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
|
||||
project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db', readonly: true };
|
||||
project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db' };
|
||||
await project.fileStore.writeFile(
|
||||
'semantic-layer/warehouse/orders.yaml',
|
||||
`name: orders
|
||||
|
|
@ -471,7 +471,7 @@ joins: []
|
|||
`);
|
||||
db.close();
|
||||
|
||||
project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db', readonly: true };
|
||||
project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db' };
|
||||
await writeFile(
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
[
|
||||
|
|
@ -480,7 +480,6 @@ joins: []
|
|||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: warehouse.db',
|
||||
' readonly: true',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf-8',
|
||||
|
|
|
|||
|
|
@ -106,7 +106,6 @@ async function writeSqliteScanConfig(projectDir: string, dbPath: string, enrich
|
|||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
` path: ${JSON.stringify(dbPath)}`,
|
||||
' readonly: true',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
|
|
|
|||
339
packages/cli/src/text-ingest.test.ts
Normal file
339
packages/cli/src/text-ingest.test.ts
Normal file
|
|
@ -0,0 +1,339 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { MemoryCaptureStatus } from '@ktx/context/memory';
|
||||
import type { KtxLocalProject } from '@ktx/context/project';
|
||||
import { runKtxTextIngest, type TextMemoryCapturePort } from './text-ingest.js';
|
||||
|
||||
function makeIo(options: { isTTY?: boolean } = {}) {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
isTTY: options.isTTY,
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeCapture(
|
||||
options: {
|
||||
failRunIds?: Set<string>;
|
||||
missingStatusRunIds?: Set<string>;
|
||||
events?: string[];
|
||||
} = {},
|
||||
): TextMemoryCapturePort {
|
||||
let next = 1;
|
||||
return {
|
||||
capture: vi.fn(async () => {
|
||||
const runId = `run-${next++}`;
|
||||
options.events?.push(`capture:${runId}`);
|
||||
return { runId };
|
||||
}),
|
||||
waitForRun: vi.fn(async (runId: string) => {
|
||||
options.events?.push(`wait:${runId}`);
|
||||
}),
|
||||
status: vi.fn(async (runId: string) => {
|
||||
options.events?.push(`status:${runId}`);
|
||||
if (options.missingStatusRunIds?.has(runId)) {
|
||||
return null;
|
||||
}
|
||||
if (options.failRunIds?.has(runId)) {
|
||||
return {
|
||||
runId,
|
||||
status: 'error',
|
||||
stage: 'capturing',
|
||||
done: true,
|
||||
captured: { wiki: [], sl: [], xrefs: [] },
|
||||
error: `${runId} failed`,
|
||||
commitHash: null,
|
||||
skillsLoaded: [],
|
||||
signalDetected: false,
|
||||
} satisfies MemoryCaptureStatus;
|
||||
}
|
||||
return {
|
||||
runId,
|
||||
status: 'done',
|
||||
stage: 'capturing',
|
||||
done: true,
|
||||
captured: { wiki: [`wiki-${runId}`], sl: [`sl-${runId}`], xrefs: [] },
|
||||
error: null,
|
||||
commitHash: `commit-${runId}`,
|
||||
skillsLoaded: ['wiki_capture', 'sl'],
|
||||
signalDetected: true,
|
||||
} satisfies MemoryCaptureStatus;
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeProject(projectDir = '/tmp/project'): KtxLocalProject {
|
||||
return { projectDir } as KtxLocalProject;
|
||||
}
|
||||
|
||||
describe('runKtxTextIngest', () => {
|
||||
it('captures repeated inline text sequentially with generated internal chat ids', async () => {
|
||||
const io = makeIo();
|
||||
const events: string[] = [];
|
||||
const capture = fakeCapture({ events });
|
||||
const createMemoryCapture = vi.fn(() => capture);
|
||||
|
||||
await expect(
|
||||
runKtxTextIngest(
|
||||
{
|
||||
projectDir: '/tmp/project',
|
||||
texts: ['Revenue means gross receipts.', 'Orders are completed purchases.'],
|
||||
files: [],
|
||||
userId: 'local-cli',
|
||||
json: true,
|
||||
failFast: false,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => fakeProject()),
|
||||
createMemoryCapture,
|
||||
now: () => 1_700_000_000_000,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(createMemoryCapture).toHaveBeenCalledWith({ projectDir: '/tmp/project' });
|
||||
expect(capture.capture).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
userId: 'local-cli',
|
||||
chatId: 'cli-text-ingest-1700000000000-1',
|
||||
userMessage: 'Ingest external text artifact "Revenue means gross receipts." into KTX memory.',
|
||||
assistantMessage: 'Revenue means gross receipts.',
|
||||
sourceType: 'external_ingest',
|
||||
}),
|
||||
);
|
||||
expect(capture.capture).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
chatId: 'cli-text-ingest-1700000000000-2',
|
||||
userMessage: 'Ingest external text artifact "Orders are completed purchases." into KTX memory.',
|
||||
assistantMessage: 'Orders are completed purchases.',
|
||||
}),
|
||||
);
|
||||
expect(capture.capture).not.toHaveBeenCalledWith(expect.objectContaining({ connectionId: expect.anything() }));
|
||||
expect(events).toEqual(['capture:run-1', 'wait:run-1', 'status:run-1', 'capture:run-2', 'wait:run-2', 'status:run-2']);
|
||||
expect(JSON.parse(io.stdout())).toMatchObject({
|
||||
status: 'done',
|
||||
results: [
|
||||
{
|
||||
label: '"Revenue means gross receipts."',
|
||||
runId: 'run-1',
|
||||
status: 'done',
|
||||
captured: { wiki: ['wiki-run-1'], sl: ['sl-run-1'] },
|
||||
},
|
||||
{
|
||||
label: '"Orders are completed purchases."',
|
||||
runId: 'run-2',
|
||||
status: 'done',
|
||||
captured: { wiki: ['wiki-run-2'], sl: ['sl-run-2'] },
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('loads files and stdin as batch items and passes a global connection id', async () => {
|
||||
const io = makeIo();
|
||||
const capture = fakeCapture();
|
||||
|
||||
await expect(
|
||||
runKtxTextIngest(
|
||||
{
|
||||
projectDir: '/tmp/project',
|
||||
texts: [],
|
||||
files: ['/tmp/docs/revenue.md', '-'],
|
||||
connectionId: 'warehouse',
|
||||
userId: 'agent',
|
||||
json: false,
|
||||
failFast: false,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => fakeProject()),
|
||||
createMemoryCapture: vi.fn(() => capture),
|
||||
readFile: vi.fn(async (path) => `file:${path}`),
|
||||
readStdin: vi.fn(async () => 'stdin content'),
|
||||
now: () => 10,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(capture.capture).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
connectionId: 'warehouse',
|
||||
userId: 'agent',
|
||||
userMessage: 'Ingest external text artifact "revenue.md" into KTX memory.',
|
||||
assistantMessage: 'file:/tmp/docs/revenue.md',
|
||||
}),
|
||||
);
|
||||
expect(capture.capture).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
connectionId: 'warehouse',
|
||||
userMessage: 'Ingest external text artifact "stdin" into KTX memory.',
|
||||
assistantMessage: 'stdin content',
|
||||
}),
|
||||
);
|
||||
expect(io.stdout()).toContain('Ingesting text memory');
|
||||
expect(io.stdout()).toContain('Texts:');
|
||||
expect(io.stdout()).toContain('revenue.md');
|
||||
expect(io.stdout()).toContain('stdin');
|
||||
});
|
||||
|
||||
it('uses bounded inline text previews as labels in plain output and capture metadata', async () => {
|
||||
const io = makeIo();
|
||||
const capture = fakeCapture();
|
||||
const longText = `This inline note is intentionally long ${'x'.repeat(120)}`;
|
||||
|
||||
await expect(
|
||||
runKtxTextIngest(
|
||||
{
|
||||
projectDir: '/tmp/project',
|
||||
texts: ['remember to call me Andrey', ' first line\n\tsecond line ', longText],
|
||||
files: [],
|
||||
userId: 'local-cli',
|
||||
json: false,
|
||||
failFast: false,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => fakeProject()),
|
||||
createMemoryCapture: vi.fn(() => capture),
|
||||
now: () => 10,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
|
||||
const output = io.stdout();
|
||||
expect(output).toContain('"remember to call me Andrey"');
|
||||
expect(output).toContain('"first line second line"');
|
||||
expect(output).toContain('"This inline note is intentionally long xxxxxxxx..."');
|
||||
expect(output).not.toContain('text-1');
|
||||
expect(output).not.toContain(longText);
|
||||
|
||||
expect(capture.capture).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
userMessage: 'Ingest external text artifact "remember to call me Andrey" into KTX memory.',
|
||||
}),
|
||||
);
|
||||
expect(capture.capture).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
userMessage: 'Ingest external text artifact "first line second line" into KTX memory.',
|
||||
}),
|
||||
);
|
||||
expect(capture.capture).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.objectContaining({
|
||||
userMessage: 'Ingest external text artifact "This inline note is intentionally long xxxxxxxx..." into KTX memory.',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('continues after an item failure by default and stops when failFast is set', async () => {
|
||||
const continueIo = makeIo();
|
||||
const continueCapture = fakeCapture({ failRunIds: new Set(['run-1']) });
|
||||
|
||||
await expect(
|
||||
runKtxTextIngest(
|
||||
{
|
||||
projectDir: '/tmp/project',
|
||||
texts: ['bad', 'good'],
|
||||
files: [],
|
||||
userId: 'local-cli',
|
||||
json: true,
|
||||
failFast: false,
|
||||
},
|
||||
continueIo.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => fakeProject()),
|
||||
createMemoryCapture: vi.fn(() => continueCapture),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(continueCapture.capture).toHaveBeenCalledTimes(2);
|
||||
expect(JSON.parse(continueIo.stdout())).toMatchObject({
|
||||
status: 'failed',
|
||||
results: [
|
||||
{ label: '"bad"', status: 'error', error: 'run-1 failed' },
|
||||
{ label: '"good"', status: 'done' },
|
||||
],
|
||||
});
|
||||
|
||||
const failFastIo = makeIo();
|
||||
const failFastCapture = fakeCapture({ failRunIds: new Set(['run-1']) });
|
||||
|
||||
await expect(
|
||||
runKtxTextIngest(
|
||||
{
|
||||
projectDir: '/tmp/project',
|
||||
texts: ['bad', 'skipped'],
|
||||
files: [],
|
||||
userId: 'local-cli',
|
||||
json: true,
|
||||
failFast: true,
|
||||
},
|
||||
failFastIo.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => fakeProject()),
|
||||
createMemoryCapture: vi.fn(() => failFastCapture),
|
||||
},
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(failFastCapture.capture).toHaveBeenCalledTimes(1);
|
||||
expect(JSON.parse(failFastIo.stdout()).results).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('rejects empty batches and empty text items', async () => {
|
||||
const noInputIo = makeIo();
|
||||
await expect(
|
||||
runKtxTextIngest(
|
||||
{
|
||||
projectDir: '/tmp/project',
|
||||
texts: [],
|
||||
files: [],
|
||||
userId: 'local-cli',
|
||||
json: false,
|
||||
failFast: false,
|
||||
},
|
||||
noInputIo.io,
|
||||
{ loadProject: vi.fn(), createMemoryCapture: vi.fn() },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
expect(noInputIo.stderr()).toContain('Provide at least one text item');
|
||||
|
||||
const emptyIo = makeIo();
|
||||
await expect(
|
||||
runKtxTextIngest(
|
||||
{
|
||||
projectDir: '/tmp/project',
|
||||
texts: [' '],
|
||||
files: [],
|
||||
userId: 'local-cli',
|
||||
json: false,
|
||||
failFast: false,
|
||||
},
|
||||
emptyIo.io,
|
||||
{ loadProject: vi.fn(), createMemoryCapture: vi.fn() },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
expect(emptyIo.stderr()).toContain('Text item "text-1" is empty');
|
||||
});
|
||||
});
|
||||
354
packages/cli/src/text-ingest.ts
Normal file
354
packages/cli/src/text-ingest.ts
Normal file
|
|
@ -0,0 +1,354 @@
|
|||
import { readFile as fsReadFile } from 'node:fs/promises';
|
||||
import { basename, resolve } from 'node:path';
|
||||
import { createLocalProjectMemoryCapture, type MemoryAgentInput, type MemoryCaptureStatus } from '@ktx/context/memory';
|
||||
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
|
||||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import { createRepainter, initViewState, renderContextBuildView, type ContextBuildTargetState } from './context-build-view.js';
|
||||
import { formatDuration } from './demo-metrics.js';
|
||||
import type { KtxPublicIngestPlanTarget } from './public-ingest.js';
|
||||
|
||||
export interface KtxTextIngestArgs {
|
||||
projectDir: string;
|
||||
texts: string[];
|
||||
files: string[];
|
||||
connectionId?: string;
|
||||
userId: string;
|
||||
json: boolean;
|
||||
failFast: boolean;
|
||||
}
|
||||
|
||||
export interface TextMemoryCapturePort {
|
||||
capture(input: MemoryAgentInput): Promise<{ runId: string }>;
|
||||
waitForRun(runId: string): Promise<void>;
|
||||
status(runId: string): Promise<MemoryCaptureStatus | null>;
|
||||
}
|
||||
|
||||
interface TextIngestItem {
|
||||
label: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
interface TextIngestResult {
|
||||
label: string;
|
||||
runId: string | null;
|
||||
status: 'done' | 'error';
|
||||
captured: MemoryCaptureStatus['captured'];
|
||||
commitHash: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface KtxTextIngestDeps {
|
||||
loadProject?: (options: { projectDir: string }) => Promise<KtxLocalProject>;
|
||||
createMemoryCapture?: (project: KtxLocalProject) => TextMemoryCapturePort;
|
||||
readFile?: (path: string) => Promise<string>;
|
||||
readStdin?: () => Promise<string>;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
const INLINE_TEXT_LABEL_MAX_LENGTH = 50;
|
||||
const ANSI_ESCAPE_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/g;
|
||||
|
||||
function defaultCreateMemoryCapture(project: KtxLocalProject): TextMemoryCapturePort {
|
||||
return createLocalProjectMemoryCapture(project);
|
||||
}
|
||||
|
||||
async function defaultReadStdin(): Promise<string> {
|
||||
const chunks: string[] = [];
|
||||
process.stdin.setEncoding('utf-8');
|
||||
for await (const chunk of process.stdin) {
|
||||
chunks.push(String(chunk));
|
||||
}
|
||||
return chunks.join('');
|
||||
}
|
||||
|
||||
async function defaultReadFile(path: string): Promise<string> {
|
||||
return await fsReadFile(path, 'utf-8');
|
||||
}
|
||||
|
||||
function emptyCaptured(): MemoryCaptureStatus['captured'] {
|
||||
return { wiki: [], sl: [], xrefs: [] };
|
||||
}
|
||||
|
||||
function normalizedTextPreview(content: string): string {
|
||||
return content
|
||||
.replace(ANSI_ESCAPE_PATTERN, '')
|
||||
.replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function truncateLabel(label: string, maxLength = INLINE_TEXT_LABEL_MAX_LENGTH): string {
|
||||
const chars = Array.from(label);
|
||||
if (chars.length <= maxLength) {
|
||||
return label;
|
||||
}
|
||||
return `${chars.slice(0, maxLength - 3).join('').trimEnd()}...`;
|
||||
}
|
||||
|
||||
function quoteInlineTextLabel(label: string): string {
|
||||
return JSON.stringify(label);
|
||||
}
|
||||
|
||||
function makeUniqueLabel(label: string, usedLabels: Set<string>): string {
|
||||
if (!usedLabels.has(label)) {
|
||||
return label;
|
||||
}
|
||||
|
||||
for (let index = 2; ; index++) {
|
||||
const suffix = ` (${index})`;
|
||||
const candidate = `${truncateLabel(label, INLINE_TEXT_LABEL_MAX_LENGTH - suffix.length)}${suffix}`;
|
||||
if (!usedLabels.has(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function textLabel(content: string, index: number, usedLabels: Set<string>): string {
|
||||
const preview = normalizedTextPreview(content);
|
||||
const baseLabel = preview.length > 0 ? quoteInlineTextLabel(truncateLabel(preview)) : `text-${index + 1}`;
|
||||
return makeUniqueLabel(baseLabel, usedLabels);
|
||||
}
|
||||
|
||||
function artifactReference(label: string): string {
|
||||
return label.startsWith('"') ? label : `"${label}"`;
|
||||
}
|
||||
|
||||
function stdinLabel(items: TextIngestItem[]): string {
|
||||
if (!items.some((item) => item.label === 'stdin')) {
|
||||
return 'stdin';
|
||||
}
|
||||
return `stdin-${items.filter((item) => item.label.startsWith('stdin')).length + 1}`;
|
||||
}
|
||||
|
||||
async function loadItems(args: KtxTextIngestArgs, deps: KtxTextIngestDeps): Promise<TextIngestItem[]> {
|
||||
const items: TextIngestItem[] = [];
|
||||
const usedTextLabels = new Set<string>();
|
||||
args.texts.forEach((content, index) => {
|
||||
const label = textLabel(content, index, usedTextLabels);
|
||||
usedTextLabels.add(label);
|
||||
items.push({ label, content });
|
||||
});
|
||||
|
||||
const readFile = deps.readFile ?? defaultReadFile;
|
||||
const readStdin = deps.readStdin ?? defaultReadStdin;
|
||||
for (const file of args.files) {
|
||||
if (file === '-') {
|
||||
items.push({ label: stdinLabel(items), content: await readStdin() });
|
||||
} else {
|
||||
const path = resolve(file);
|
||||
items.push({ label: basename(path), content: await readFile(path) });
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
function validateItems(items: TextIngestItem[], io: KtxCliIo): boolean {
|
||||
if (items.length === 0) {
|
||||
io.stderr.write('Provide at least one text item with --text, a file path, or - for stdin.\n');
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const item of items) {
|
||||
if (item.content.trim().length === 0) {
|
||||
io.stderr.write(`Text item "${item.label}" is empty.\n`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function makeTarget(label: string): KtxPublicIngestPlanTarget {
|
||||
return {
|
||||
connectionId: label,
|
||||
driver: 'text',
|
||||
operation: 'source-ingest',
|
||||
debugCommand: '',
|
||||
steps: ['memory-update'],
|
||||
};
|
||||
}
|
||||
|
||||
function allTargets(state: ReturnType<typeof initViewState>): ContextBuildTargetState[] {
|
||||
return [...state.primarySources, ...state.contextSources];
|
||||
}
|
||||
|
||||
function renderTextIngestView(state: ReturnType<typeof initViewState>, styled: boolean): string {
|
||||
return renderContextBuildView(state, {
|
||||
styled,
|
||||
title: 'Ingesting text memory',
|
||||
contextGroupLabel: 'Texts',
|
||||
sourceIngestRunningText: 'capturing...',
|
||||
completedItemName: { singular: 'text', plural: 'texts' },
|
||||
});
|
||||
}
|
||||
|
||||
function summarizeCaptured(captured: MemoryCaptureStatus['captured']): string {
|
||||
const parts = [
|
||||
`wiki=${captured.wiki.length}`,
|
||||
`sl=${captured.sl.length}`,
|
||||
`xrefs=${captured.xrefs.length}`,
|
||||
];
|
||||
return parts.join(', ');
|
||||
}
|
||||
|
||||
function resultFromStatus(label: string, status: MemoryCaptureStatus): TextIngestResult {
|
||||
return {
|
||||
label,
|
||||
runId: status.runId,
|
||||
status: status.status === 'done' ? 'done' : 'error',
|
||||
captured: status.captured,
|
||||
commitHash: status.commitHash,
|
||||
error: status.error,
|
||||
};
|
||||
}
|
||||
|
||||
function errorResult(label: string, runId: string | null, error: unknown): TextIngestResult {
|
||||
return {
|
||||
label,
|
||||
runId,
|
||||
status: 'error',
|
||||
captured: emptyCaptured(),
|
||||
commitHash: null,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
|
||||
function writeJsonResult(args: KtxTextIngestArgs, results: TextIngestResult[], io: KtxCliIo): void {
|
||||
io.stdout.write(
|
||||
`${JSON.stringify(
|
||||
{
|
||||
status: results.some((result) => result.status === 'error') ? 'failed' : 'done',
|
||||
projectDir: args.projectDir,
|
||||
connectionId: args.connectionId ?? null,
|
||||
results,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
function writePlainFailures(results: TextIngestResult[], io: KtxCliIo): void {
|
||||
const failures = results.filter((result) => result.status === 'error');
|
||||
if (failures.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
io.stdout.write('\nFailed text items:\n');
|
||||
for (const result of failures) {
|
||||
io.stdout.write(` ${result.label}: ${result.error ?? 'failed'}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function runKtxTextIngest(
|
||||
args: KtxTextIngestArgs,
|
||||
io: KtxCliIo,
|
||||
deps: KtxTextIngestDeps = {},
|
||||
): Promise<number> {
|
||||
const items = await loadItems(args, deps);
|
||||
if (!validateItems(items, io)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir });
|
||||
const memoryCapture = (deps.createMemoryCapture ?? defaultCreateMemoryCapture)(project);
|
||||
const now = deps.now ?? (() => Date.now());
|
||||
const batchId = now();
|
||||
const state = initViewState(items.map((item) => makeTarget(item.label)));
|
||||
const targets = allTargets(state);
|
||||
const isTTY = io.stdout.isTTY === true && args.json !== true;
|
||||
const repainter = isTTY ? createRepainter(io) : null;
|
||||
const results: TextIngestResult[] = [];
|
||||
|
||||
state.startedAt = now();
|
||||
const paint = () => repainter?.paint(renderTextIngestView(state, true));
|
||||
paint();
|
||||
|
||||
let spinnerInterval: ReturnType<typeof setInterval> | null = null;
|
||||
if (repainter) {
|
||||
spinnerInterval = setInterval(() => {
|
||||
const current = now();
|
||||
state.frame++;
|
||||
state.totalElapsedMs = state.startedAt === null ? 0 : current - state.startedAt;
|
||||
for (const target of targets) {
|
||||
if (target.status === 'running' && target.startedAt !== null) {
|
||||
target.elapsedMs = current - target.startedAt;
|
||||
}
|
||||
}
|
||||
paint();
|
||||
}, 140);
|
||||
}
|
||||
|
||||
try {
|
||||
for (let index = 0; index < items.length; index++) {
|
||||
const item = items[index]!;
|
||||
const target = targets[index]!;
|
||||
target.status = 'running';
|
||||
target.startedAt = now();
|
||||
target.detailLine = 'capturing...';
|
||||
target.progressUpdatedAtMs = target.startedAt;
|
||||
paint();
|
||||
|
||||
let runId: string | null = null;
|
||||
let result: TextIngestResult;
|
||||
try {
|
||||
const captureInput: MemoryAgentInput = {
|
||||
userId: args.userId,
|
||||
chatId: `cli-text-ingest-${batchId}-${index + 1}`,
|
||||
userMessage: `Ingest external text artifact ${artifactReference(item.label)} into KTX memory.`,
|
||||
assistantMessage: item.content.trim(),
|
||||
...(args.connectionId ? { connectionId: args.connectionId } : {}),
|
||||
sourceType: 'external_ingest',
|
||||
};
|
||||
const capture = await memoryCapture.capture(captureInput);
|
||||
runId = capture.runId;
|
||||
await memoryCapture.waitForRun(runId);
|
||||
const status = await memoryCapture.status(runId);
|
||||
if (!status) {
|
||||
throw new Error(`Memory capture run "${runId}" was not found.`);
|
||||
}
|
||||
result = resultFromStatus(item.label, status);
|
||||
} catch (error) {
|
||||
result = errorResult(item.label, runId, error);
|
||||
}
|
||||
|
||||
results.push(result);
|
||||
target.elapsedMs = now() - (target.startedAt ?? now());
|
||||
target.detailLine = null;
|
||||
target.status = result.status === 'done' ? 'done' : 'failed';
|
||||
target.summaryText = result.status === 'done' ? summarizeCaptured(result.captured) : null;
|
||||
target.failureText = result.status === 'error' ? result.error : null;
|
||||
paint();
|
||||
|
||||
if (result.status === 'error' && args.failFast) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (spinnerInterval) {
|
||||
clearInterval(spinnerInterval);
|
||||
}
|
||||
}
|
||||
|
||||
if (state.startedAt !== null) {
|
||||
state.totalElapsedMs = now() - state.startedAt;
|
||||
}
|
||||
|
||||
if (args.json) {
|
||||
writeJsonResult(args, results, io);
|
||||
} else if (repainter) {
|
||||
repainter.paint(renderTextIngestView(state, true));
|
||||
writePlainFailures(results, io);
|
||||
} else {
|
||||
io.stdout.write(renderTextIngestView(state, false));
|
||||
writePlainFailures(results, io);
|
||||
}
|
||||
|
||||
if (!args.json && results.length > 0) {
|
||||
const duration = state.totalElapsedMs > 0 ? ` in ${formatDuration(state.totalElapsedMs)}` : '';
|
||||
const outcome = results.some((result) => result.status === 'error') ? 'finished with failures' : 'finished';
|
||||
io.stdout.write(`Text memory ingest ${outcome}${duration}.\n`);
|
||||
}
|
||||
|
||||
return results.some((result) => result.status === 'error') ? 1 : 0;
|
||||
}
|
||||
|
|
@ -100,7 +100,6 @@ const connection = {
|
|||
dataset_id: 'analytics',
|
||||
credentials_json: JSON.stringify({ project_id: 'project-1', client_email: 'reader@example.test' }),
|
||||
location: 'US',
|
||||
readonly: true,
|
||||
};
|
||||
|
||||
describe('KtxBigQueryScanConnector', () => {
|
||||
|
|
@ -112,12 +111,6 @@ describe('KtxBigQueryScanConnector', () => {
|
|||
datasetIds: ['analytics'],
|
||||
location: 'US',
|
||||
});
|
||||
expect(() =>
|
||||
bigQueryConnectionConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { ...connection, readonly: false },
|
||||
}),
|
||||
).toThrow('Native BigQuery connector requires connections.warehouse.readonly: true');
|
||||
});
|
||||
|
||||
it('introspects datasets, table metadata, primary keys, and normalized types', async () => {
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ export interface KtxBigQueryConnectionConfig {
|
|||
dataset_ids?: string[];
|
||||
credentials_json?: string;
|
||||
location?: string;
|
||||
readonly?: boolean;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
@ -194,7 +193,9 @@ function normalizeValue(value: unknown): unknown {
|
|||
return value;
|
||||
}
|
||||
|
||||
export function isKtxBigQueryConnectionConfig(connection: KtxBigQueryConnectionConfig | undefined): boolean {
|
||||
export function isKtxBigQueryConnectionConfig(
|
||||
connection: KtxBigQueryConnectionConfig | undefined,
|
||||
): connection is KtxBigQueryConnectionConfig {
|
||||
return String(connection?.driver ?? '').toLowerCase() === 'bigquery';
|
||||
}
|
||||
|
||||
|
|
@ -203,11 +204,9 @@ export function bigQueryConnectionConfigFromConfig(input: {
|
|||
connection: KtxBigQueryConnectionConfig | undefined;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): KtxBigQueryResolvedConnectionConfig {
|
||||
const inputDriver = input.connection?.driver ?? 'unknown';
|
||||
if (!isKtxBigQueryConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native BigQuery connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
|
||||
}
|
||||
if (input.connection?.readonly !== true) {
|
||||
throw new Error(`Native BigQuery connector requires connections.${input.connectionId}.readonly: true`);
|
||||
throw new Error(`Native BigQuery connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
|
||||
const env = input.env ?? process.env;
|
||||
|
|
|
|||
|
|
@ -112,7 +112,6 @@ describe('KtxClickHouseScanConnector', () => {
|
|||
username: 'reader',
|
||||
password: 'test-pass', // pragma: allowlist secret
|
||||
ssl: true,
|
||||
readonly: true,
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
|
|
@ -123,12 +122,6 @@ describe('KtxClickHouseScanConnector', () => {
|
|||
password: 'test-pass', // pragma: allowlist secret
|
||||
ssl: true,
|
||||
});
|
||||
expect(() =>
|
||||
clickHouseClientConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'clickhouse', host: 'ch.example.test', database: 'analytics', readonly: false },
|
||||
}),
|
||||
).toThrow('Native ClickHouse connector requires connections.warehouse.readonly: true');
|
||||
});
|
||||
|
||||
it('introspects schema, primary keys, comments, row counts, and views', async () => {
|
||||
|
|
@ -140,7 +133,6 @@ describe('KtxClickHouseScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'test-pass', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
clientFactory: fakeClientFactory(),
|
||||
now: () => new Date('2026-04-29T14:00:00.000Z'),
|
||||
|
|
@ -189,7 +181,6 @@ describe('KtxClickHouseScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'test-pass', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
clientFactory,
|
||||
});
|
||||
|
|
@ -253,7 +244,6 @@ describe('KtxClickHouseScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'test-pass', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
clientFactory: fakeClientFactory(),
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ export interface KtxClickHouseConnectionConfig {
|
|||
password?: string;
|
||||
url?: string;
|
||||
ssl?: boolean;
|
||||
readonly?: boolean;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
@ -193,7 +192,9 @@ function isNullableClickHouseType(type: string): boolean {
|
|||
return type.startsWith('Nullable(') || type.startsWith('LowCardinality(Nullable(');
|
||||
}
|
||||
|
||||
export function isKtxClickHouseConnectionConfig(connection: KtxClickHouseConnectionConfig | undefined): boolean {
|
||||
export function isKtxClickHouseConnectionConfig(
|
||||
connection: KtxClickHouseConnectionConfig | undefined,
|
||||
): connection is KtxClickHouseConnectionConfig {
|
||||
return String(connection?.driver ?? '').toLowerCase() === 'clickhouse';
|
||||
}
|
||||
|
||||
|
|
@ -202,11 +203,9 @@ export function clickHouseClientConfigFromConfig(input: {
|
|||
connection: KtxClickHouseConnectionConfig | undefined;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): KtxClickHouseResolvedClientConfig {
|
||||
const inputDriver = input.connection?.driver ?? 'unknown';
|
||||
if (!isKtxClickHouseConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native ClickHouse connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
|
||||
}
|
||||
if (input.connection?.readonly !== true) {
|
||||
throw new Error(`Native ClickHouse connector requires connections.${input.connectionId}.readonly: true`);
|
||||
throw new Error(`Native ClickHouse connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
|
||||
const env = input.env ?? process.env;
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ function fakePoolFactory(): KtxMysqlPoolFactory {
|
|||
|
||||
describe('KtxMysqlScanConnector', () => {
|
||||
it('resolves MySQL connection configuration safely', () => {
|
||||
expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics', readonly: true })).toBe(true);
|
||||
expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(true);
|
||||
expect(isKtxMysqlConnectionConfig({ driver: 'postgres', host: 'localhost', database: 'analytics' })).toBe(false);
|
||||
expect(
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
|
|
@ -105,7 +105,6 @@ describe('KtxMysqlScanConnector', () => {
|
|||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
ssl: true,
|
||||
readonly: true,
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
|
|
@ -116,12 +115,6 @@ describe('KtxMysqlScanConnector', () => {
|
|||
password: 'secret', // pragma: allowlist secret
|
||||
ssl: { rejectUnauthorized: false },
|
||||
});
|
||||
expect(() =>
|
||||
mysqlConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'mysql', host: 'db.example.test', database: 'analytics', readonly: false },
|
||||
}),
|
||||
).toThrow('Native MySQL connector requires connections.warehouse.readonly: true');
|
||||
});
|
||||
|
||||
it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => {
|
||||
|
|
@ -133,7 +126,6 @@ describe('KtxMysqlScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
poolFactory: fakePoolFactory(),
|
||||
now: () => new Date('2026-04-29T12:00:00.000Z'),
|
||||
|
|
@ -192,7 +184,6 @@ describe('KtxMysqlScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
poolFactory,
|
||||
});
|
||||
|
|
@ -249,7 +240,6 @@ describe('KtxMysqlScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'secret', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
poolFactory: fakePoolFactory(),
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ export interface KtxMysqlConnectionConfig {
|
|||
password?: string;
|
||||
url?: string;
|
||||
ssl?: boolean | { rejectUnauthorized?: boolean };
|
||||
readonly?: boolean;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
@ -232,7 +231,9 @@ function queryParams(params: Record<string, unknown> | unknown[] | undefined): u
|
|||
return Array.isArray(params) ? params : Object.values(params);
|
||||
}
|
||||
|
||||
export function isKtxMysqlConnectionConfig(connection: KtxMysqlConnectionConfig | undefined): boolean {
|
||||
export function isKtxMysqlConnectionConfig(
|
||||
connection: KtxMysqlConnectionConfig | undefined,
|
||||
): connection is KtxMysqlConnectionConfig {
|
||||
return String(connection?.driver ?? '').toLowerCase() === 'mysql';
|
||||
}
|
||||
|
||||
|
|
@ -241,11 +242,9 @@ export function mysqlConnectionPoolConfigFromConfig(input: {
|
|||
connection: KtxMysqlConnectionConfig | undefined;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): KtxMysqlPoolConfig {
|
||||
const inputDriver = input.connection?.driver ?? 'unknown';
|
||||
if (!isKtxMysqlConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native MySQL connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
|
||||
}
|
||||
if (input.connection?.readonly !== true) {
|
||||
throw new Error(`Native MySQL connector requires connections.${input.connectionId}.readonly: true`);
|
||||
throw new Error(`Native MySQL connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
|
||||
const env = input.env ?? process.env;
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ function metadataResults(): Map<string, FakeQueryResult> {
|
|||
|
||||
describe('KtxPostgresScanConnector', () => {
|
||||
it('resolves configuration safely', () => {
|
||||
expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL', readonly: true })).toBe(true);
|
||||
expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(true);
|
||||
expect(isKtxPostgresConnectionConfig({ driver: 'postgresql', host: 'db', database: 'analytics' })).toBe(true);
|
||||
expect(isKtxPostgresConnectionConfig({ driver: 'mysql', host: 'db' })).toBe(false);
|
||||
expect(
|
||||
|
|
@ -115,7 +115,6 @@ describe('KtxPostgresScanConnector', () => {
|
|||
username: 'reader',
|
||||
password: 'test-password', // pragma: allowlist secret
|
||||
schemas: ['analytics', 'public'],
|
||||
readonly: true,
|
||||
ssl: true,
|
||||
rejectUnauthorized: false,
|
||||
},
|
||||
|
|
@ -134,7 +133,6 @@ describe('KtxPostgresScanConnector', () => {
|
|||
connection: {
|
||||
driver: 'postgres',
|
||||
url: 'env:DEMO_DATABASE_URL',
|
||||
readonly: true,
|
||||
},
|
||||
env: {
|
||||
DEMO_DATABASE_URL: 'postgresql://reader@demo.example.test:5432/demo?sslmode=prefer',
|
||||
|
|
@ -148,12 +146,16 @@ describe('KtxPostgresScanConnector', () => {
|
|||
});
|
||||
expect(libpqPreferConfig).not.toHaveProperty('connectionString');
|
||||
expect(libpqPreferConfig).not.toHaveProperty('ssl');
|
||||
expect(() =>
|
||||
expect(
|
||||
postgresPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'postgres', host: 'db.example.test', database: 'analytics', username: 'reader' },
|
||||
}),
|
||||
).toThrow('Native PostgreSQL connector requires connections.warehouse.readonly: true');
|
||||
).toMatchObject({
|
||||
host: 'db.example.test',
|
||||
database: 'analytics',
|
||||
user: 'reader',
|
||||
});
|
||||
});
|
||||
|
||||
it('introspects schemas, tables, views, primary keys, comments, row counts, and foreign keys', async () => {
|
||||
|
|
@ -166,7 +168,6 @@ describe('KtxPostgresScanConnector', () => {
|
|||
username: 'reader',
|
||||
password: 'test-password', // pragma: allowlist secret
|
||||
schema: 'public',
|
||||
readonly: true,
|
||||
},
|
||||
poolFactory: fakePoolFactory(metadataResults()),
|
||||
now: () => new Date('2026-04-29T10:00:00.000Z'),
|
||||
|
|
@ -225,7 +226,6 @@ describe('KtxPostgresScanConnector', () => {
|
|||
username: 'reader',
|
||||
password: 'test-password', // pragma: allowlist secret
|
||||
schema: 'public',
|
||||
readonly: true,
|
||||
},
|
||||
poolFactory: fakePoolFactory(metadataResults()),
|
||||
});
|
||||
|
|
@ -274,7 +274,6 @@ describe('KtxPostgresScanConnector', () => {
|
|||
username: 'reader',
|
||||
password: 'test-password', // pragma: allowlist secret
|
||||
schema: 'public',
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
poolFactory: fakePoolFactory(metadataResults()),
|
||||
|
|
@ -347,7 +346,6 @@ describe('KtxPostgresScanConnector', () => {
|
|||
username: 'reader',
|
||||
password: 'test-password', // pragma: allowlist secret
|
||||
schema: 'public',
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
poolFactory: endAwarePoolFactory,
|
||||
|
|
@ -383,7 +381,6 @@ describe('KtxPostgresScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
password: 'test-password', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
poolFactory,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -61,7 +61,6 @@ export interface KtxPostgresConnectionConfig {
|
|||
sslmode?: string;
|
||||
sslMode?: string;
|
||||
rejectUnauthorized?: boolean;
|
||||
readonly?: boolean;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
@ -291,7 +290,9 @@ function searchPathSchemasFromConnection(connection: KtxPostgresConnectionConfig
|
|||
return schemas.includes('public') ? schemas : [...schemas, 'public'];
|
||||
}
|
||||
|
||||
export function isKtxPostgresConnectionConfig(connection: KtxPostgresConnectionConfig | undefined): boolean {
|
||||
export function isKtxPostgresConnectionConfig(
|
||||
connection: KtxPostgresConnectionConfig | undefined,
|
||||
): connection is KtxPostgresConnectionConfig {
|
||||
const driver = String(connection?.driver ?? '').toLowerCase();
|
||||
return driver === 'postgres' || driver === 'postgresql';
|
||||
}
|
||||
|
|
@ -301,11 +302,9 @@ export function postgresPoolConfigFromConfig(input: {
|
|||
connection: KtxPostgresConnectionConfig | undefined;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): KtxPostgresPoolConfig {
|
||||
const inputDriver = input.connection?.driver ?? 'unknown';
|
||||
if (!isKtxPostgresConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
|
||||
}
|
||||
if (input.connection?.readonly !== true) {
|
||||
throw new Error(`Native PostgreSQL connector requires connections.${input.connectionId}.readonly: true`);
|
||||
throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
|
||||
const env = input.env ?? process.env;
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ describe('KtxPostgresHistoricSqlQueryClient', () => {
|
|||
connectionId: 'warehouse',
|
||||
connection: {
|
||||
driver: 'postgres',
|
||||
readonly: true,
|
||||
url: 'postgresql://readonly:secret@pg.example.test/warehouse', // pragma: allowlist secret
|
||||
},
|
||||
poolFactory,
|
||||
|
|
|
|||
|
|
@ -78,7 +78,6 @@ describe('KtxSnowflakeScanConnector', () => {
|
|||
warehouse: 'WH',
|
||||
database: 'ANALYTICS',
|
||||
username: 'reader',
|
||||
readonly: true,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(isKtxSnowflakeConnectionConfig({ driver: 'bigquery' })).toBe(false);
|
||||
|
|
@ -94,7 +93,6 @@ describe('KtxSnowflakeScanConnector', () => {
|
|||
schema_name: 'PUBLIC',
|
||||
username: 'reader',
|
||||
password: 'fixture-pass', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
|
|
@ -105,12 +103,6 @@ describe('KtxSnowflakeScanConnector', () => {
|
|||
username: 'reader',
|
||||
authMethod: 'password',
|
||||
});
|
||||
expect(() =>
|
||||
snowflakeConnectionConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'snowflake', account: 'acct', readonly: false },
|
||||
}),
|
||||
).toThrow('Native Snowflake connector requires connections.warehouse.readonly: true');
|
||||
});
|
||||
|
||||
it('introspects schema, primary keys, comments, row counts, and dimensions', async () => {
|
||||
|
|
@ -125,7 +117,6 @@ describe('KtxSnowflakeScanConnector', () => {
|
|||
schema_name: 'PUBLIC',
|
||||
username: 'reader',
|
||||
password: 'fixture-pass', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
driverFactory: fakeDriverFactory(),
|
||||
now: () => new Date('2026-04-29T18:00:00.000Z'),
|
||||
|
|
@ -185,7 +176,6 @@ describe('KtxSnowflakeScanConnector', () => {
|
|||
schema_name: 'PUBLIC',
|
||||
username: 'reader',
|
||||
password: 'fixture-pass', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
driverFactory,
|
||||
});
|
||||
|
|
@ -243,7 +233,6 @@ describe('KtxSnowflakeScanConnector', () => {
|
|||
schema_name: 'PUBLIC',
|
||||
username: 'reader',
|
||||
password: 'fixture-pass', // pragma: allowlist secret
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
driverFactory: fakeDriverFactory(),
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ export interface KtxSnowflakeConnectionConfig {
|
|||
privateKey?: string;
|
||||
passphrase?: string;
|
||||
role?: string;
|
||||
readonly?: boolean;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
@ -191,7 +190,9 @@ function toSnowflakeBinds(params: unknown[] | undefined): snowflake.Binds | unde
|
|||
return params?.map((value) => toSnowflakeBind(value));
|
||||
}
|
||||
|
||||
export function isKtxSnowflakeConnectionConfig(connection: KtxSnowflakeConnectionConfig | undefined): boolean {
|
||||
export function isKtxSnowflakeConnectionConfig(
|
||||
connection: KtxSnowflakeConnectionConfig | undefined,
|
||||
): connection is KtxSnowflakeConnectionConfig {
|
||||
return String(connection?.driver ?? '').toLowerCase() === 'snowflake';
|
||||
}
|
||||
|
||||
|
|
@ -200,11 +201,9 @@ export function snowflakeConnectionConfigFromConfig(input: {
|
|||
connection: KtxSnowflakeConnectionConfig | undefined;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): KtxSnowflakeResolvedConnectionConfig {
|
||||
const inputDriver = input.connection?.driver ?? 'unknown';
|
||||
if (!isKtxSnowflakeConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native Snowflake connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
|
||||
}
|
||||
if (input.connection?.readonly !== true) {
|
||||
throw new Error(`Native Snowflake connector requires connections.${input.connectionId}.readonly: true`);
|
||||
throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
const env = input.env ?? process.env;
|
||||
const authMethod = input.connection?.authMethod ?? 'password';
|
||||
|
|
@ -395,7 +394,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver {
|
|||
private async createConnection(): Promise<snowflake.Connection> {
|
||||
const patch = await this.sdkOptionsProvider?.resolve({
|
||||
account: this.resolved.account,
|
||||
connection: { ...this.resolved, driver: 'snowflake', readonly: true },
|
||||
connection: { ...this.resolved, driver: 'snowflake' },
|
||||
});
|
||||
if (patch?.close) {
|
||||
this.closeSdkOptions.push(patch.close);
|
||||
|
|
|
|||
|
|
@ -53,45 +53,43 @@ describe('KtxSqliteScanConnector', () => {
|
|||
writeFileSync(pointerPath, dbPath, 'utf-8');
|
||||
|
||||
try {
|
||||
expect(isKtxSqliteConnectionConfig({ driver: 'sqlite', path: 'warehouse.db', readonly: true })).toBe(true);
|
||||
expect(isKtxSqliteConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL', readonly: true })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(isKtxSqliteConnectionConfig({ driver: 'sqlite', path: 'warehouse.db' })).toBe(true);
|
||||
expect(isKtxSqliteConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(false);
|
||||
expect(
|
||||
sqliteDatabasePathFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true },
|
||||
connection: { driver: 'sqlite', path: 'warehouse.db' },
|
||||
}),
|
||||
).toBe(dbPath);
|
||||
expect(
|
||||
sqliteDatabasePathFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true },
|
||||
connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' },
|
||||
}),
|
||||
).toBe(dbPath);
|
||||
expect(
|
||||
sqliteDatabasePathFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', url: `file://${dbPath}`, readonly: true },
|
||||
connection: { driver: 'sqlite', url: `file://${dbPath}` },
|
||||
}),
|
||||
).toBe(dbPath);
|
||||
expect(
|
||||
sqliteDatabasePathFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', path: `file:${pointerPath}`, readonly: true },
|
||||
connection: { driver: 'sqlite', path: `file:${pointerPath}` },
|
||||
}),
|
||||
).toBe(dbPath);
|
||||
expect(() =>
|
||||
expect(
|
||||
sqliteDatabasePathFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', path: 'warehouse.db', readonly: false },
|
||||
connection: { driver: 'sqlite', path: 'warehouse.db' },
|
||||
}),
|
||||
).toThrow('Native SQLite connector requires connections.warehouse.readonly: true');
|
||||
).toBe(dbPath);
|
||||
} finally {
|
||||
if (originalDatabaseUrl === undefined) {
|
||||
delete process.env.KTX_SQLITE_TEST_URL;
|
||||
|
|
@ -104,7 +102,7 @@ describe('KtxSqliteScanConnector', () => {
|
|||
it('introspects schema, primary keys, row counts, views, and foreign keys', async () => {
|
||||
const connector = new KtxSqliteScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'sqlite', path: dbPath, readonly: true },
|
||||
connection: { driver: 'sqlite', path: dbPath },
|
||||
now: () => new Date('2026-04-29T10:00:00.000Z'),
|
||||
});
|
||||
|
||||
|
|
@ -151,7 +149,7 @@ describe('KtxSqliteScanConnector', () => {
|
|||
it('runs samples, distinct values, statistics, and read-only SQL', async () => {
|
||||
const connector = new KtxSqliteScanConnector({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'sqlite', path: dbPath, readonly: true },
|
||||
connection: { driver: 'sqlite', path: dbPath },
|
||||
});
|
||||
|
||||
await expect(
|
||||
|
|
@ -199,7 +197,7 @@ describe('KtxSqliteScanConnector', () => {
|
|||
const introspection = createSqliteLiveDatabaseIntrospection({
|
||||
projectDir: tempDir,
|
||||
connections: {
|
||||
warehouse: { driver: 'sqlite', path: 'warehouse.db', readonly: true },
|
||||
warehouse: { driver: 'sqlite', path: 'warehouse.db' },
|
||||
},
|
||||
now: () => new Date('2026-04-29T10:00:00.000Z'),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ export interface KtxSqliteConnectionConfig {
|
|||
path?: string;
|
||||
url?: string;
|
||||
file_path?: string;
|
||||
readonly?: boolean;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
@ -135,17 +134,17 @@ function stripLeadingSqlComments(sql: string): string {
|
|||
return sql.slice(index);
|
||||
}
|
||||
|
||||
export function isKtxSqliteConnectionConfig(connection: KtxSqliteConnectionConfig | undefined): boolean {
|
||||
export function isKtxSqliteConnectionConfig(
|
||||
connection: KtxSqliteConnectionConfig | undefined,
|
||||
): connection is KtxSqliteConnectionConfig {
|
||||
const driver = String(connection?.driver ?? '').toLowerCase();
|
||||
return driver === 'sqlite' || driver === 'sqlite3';
|
||||
}
|
||||
|
||||
export function sqliteDatabasePathFromConfig(input: SqliteDatabasePathInput): string {
|
||||
const inputDriver = input.connection?.driver ?? 'unknown';
|
||||
if (!isKtxSqliteConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native SQLite connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
|
||||
}
|
||||
if (input.connection?.readonly !== true) {
|
||||
throw new Error(`Native SQLite connector requires connections.${input.connectionId}.readonly: true`);
|
||||
throw new Error(`Native SQLite connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
const configuredPath =
|
||||
stringConfigValue(input.connection, 'path') ??
|
||||
|
|
|
|||
|
|
@ -145,7 +145,6 @@ describe('KtxSqlServerScanConnector', () => {
|
|||
driver: 'sqlserver',
|
||||
host: 'localhost',
|
||||
database: 'analytics',
|
||||
readonly: true,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(isKtxSqlServerConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(false);
|
||||
|
|
@ -159,7 +158,6 @@ describe('KtxSqlServerScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
trustServerCertificate: false,
|
||||
readonly: true,
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
|
|
@ -169,12 +167,6 @@ describe('KtxSqlServerScanConnector', () => {
|
|||
user: 'reader',
|
||||
options: { encrypt: true, trustServerCertificate: false },
|
||||
});
|
||||
expect(() =>
|
||||
sqlServerConnectionPoolConfigFromConfig({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'sqlserver', host: 'db.example.test', database: 'analytics', readonly: false },
|
||||
}),
|
||||
).toThrow('Native SQL Server connector requires connections.warehouse.readonly: true');
|
||||
});
|
||||
|
||||
it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => {
|
||||
|
|
@ -186,7 +178,6 @@ describe('KtxSqlServerScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
schema: 'dbo',
|
||||
readonly: true,
|
||||
},
|
||||
poolFactory: fakePoolFactory(),
|
||||
now: () => new Date('2026-04-29T16:00:00.000Z'),
|
||||
|
|
@ -246,7 +237,6 @@ describe('KtxSqlServerScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
schema: 'dbo',
|
||||
readonly: true,
|
||||
},
|
||||
poolFactory,
|
||||
});
|
||||
|
|
@ -315,7 +305,6 @@ describe('KtxSqlServerScanConnector', () => {
|
|||
database: 'analytics',
|
||||
username: 'reader',
|
||||
schema: 'dbo',
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
poolFactory: fakePoolFactory(),
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ export interface KtxSqlServerConnectionConfig {
|
|||
schema?: string;
|
||||
schemas?: string[];
|
||||
trustServerCertificate?: boolean;
|
||||
readonly?: boolean;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
@ -234,7 +233,9 @@ function limitSqlForSqlServerExecution(sqlText: string, maxRows: number | undefi
|
|||
return `SELECT TOP ${maxRows} * FROM (${trimmed}) AS ktx_query_result`;
|
||||
}
|
||||
|
||||
export function isKtxSqlServerConnectionConfig(connection: KtxSqlServerConnectionConfig | undefined): boolean {
|
||||
export function isKtxSqlServerConnectionConfig(
|
||||
connection: KtxSqlServerConnectionConfig | undefined,
|
||||
): connection is KtxSqlServerConnectionConfig {
|
||||
return String(connection?.driver ?? '').toLowerCase() === 'sqlserver';
|
||||
}
|
||||
|
||||
|
|
@ -243,11 +244,9 @@ export function sqlServerConnectionPoolConfigFromConfig(input: {
|
|||
connection: KtxSqlServerConnectionConfig | undefined;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): KtxSqlServerPoolConfig {
|
||||
const inputDriver = input.connection?.driver ?? 'unknown';
|
||||
if (!isKtxSqlServerConnectionConfig(input.connection)) {
|
||||
throw new Error(`Native SQL Server connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
|
||||
}
|
||||
if (input.connection?.readonly !== true) {
|
||||
throw new Error(`Native SQL Server connector requires connections.${input.connectionId}.readonly: true`);
|
||||
throw new Error(`Native SQL Server connector cannot run driver "${inputDriver}"`);
|
||||
}
|
||||
|
||||
const env = input.env ?? process.env;
|
||||
|
|
|
|||
|
|
@ -26,14 +26,14 @@ describe('createDefaultLocalQueryExecutor', () => {
|
|||
await expect(
|
||||
executor.execute({
|
||||
connectionId: 'pg',
|
||||
connection: { driver: 'postgres', readonly: true },
|
||||
connection: { driver: 'postgres' },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).resolves.toMatchObject({ headers: ['pg'] });
|
||||
await expect(
|
||||
executor.execute({
|
||||
connectionId: 'local',
|
||||
connection: { driver: 'sqlite', readonly: true },
|
||||
connection: { driver: 'sqlite' },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).resolves.toMatchObject({ headers: ['sqlite'] });
|
||||
|
|
@ -51,7 +51,7 @@ describe('createDefaultLocalQueryExecutor', () => {
|
|||
await expect(
|
||||
executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'snowflake', readonly: true },
|
||||
connection: { driver: 'snowflake' },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).rejects.toThrow('No local query executor is configured for driver "snowflake".');
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ describe('createPostgresQueryExecutor', () => {
|
|||
|
||||
const result = await executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'postgres', url: 'postgres://example/db', readonly: true },
|
||||
connection: { driver: 'postgres', url: 'postgres://example/db' },
|
||||
sql: 'select status, count(*) as order_count from public.orders group by status',
|
||||
maxRows: 50,
|
||||
});
|
||||
|
|
@ -80,7 +80,7 @@ describe('createPostgresQueryExecutor', () => {
|
|||
await expect(
|
||||
executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'postgres', url: 'postgres://example/db', readonly: true },
|
||||
connection: { driver: 'postgres', url: 'postgres://example/db' },
|
||||
sql: 'select * from broken',
|
||||
maxRows: 10,
|
||||
}),
|
||||
|
|
@ -89,23 +89,15 @@ describe('createPostgresQueryExecutor', () => {
|
|||
expect(client.end).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('requires a Postgres url and read-only connection config', async () => {
|
||||
it('requires a Postgres url', async () => {
|
||||
const executor = createPostgresQueryExecutor({ clientFactory: vi.fn() });
|
||||
|
||||
await expect(
|
||||
executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'postgres', readonly: true },
|
||||
connection: { driver: 'postgres' },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).rejects.toThrow('Local Postgres execution requires connections.warehouse.url');
|
||||
|
||||
await expect(
|
||||
executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
connection: { driver: 'postgres', url: 'postgres://example/db', readonly: false },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).rejects.toThrow('Local query execution requires connections.warehouse.readonly: true');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -37,18 +37,16 @@ export function createPostgresQueryExecutor(options: PostgresQueryExecutorOption
|
|||
return {
|
||||
async execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult> {
|
||||
const driver = connectionDriver(input);
|
||||
const connection = input.connection;
|
||||
if (driver !== 'postgres' && driver !== 'postgresql') {
|
||||
throw new Error(`Local Postgres execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`);
|
||||
throw new Error(`Local Postgres execution cannot run driver "${connection?.driver ?? 'unknown'}".`);
|
||||
}
|
||||
if (input.connection?.readonly !== true) {
|
||||
throw new Error(`Local query execution requires connections.${input.connectionId}.readonly: true.`);
|
||||
}
|
||||
if (typeof input.connection.url !== 'string' || input.connection.url.trim().length === 0) {
|
||||
if (typeof connection?.url !== 'string' || connection.url.trim().length === 0) {
|
||||
throw new Error(`Local Postgres execution requires connections.${input.connectionId}.url.`);
|
||||
}
|
||||
|
||||
const client = clientFactory({
|
||||
connectionString: input.connection.url,
|
||||
connectionString: connection.url,
|
||||
statement_timeout: options.statementTimeoutMs ?? 30_000,
|
||||
query_timeout: options.queryTimeoutMs ?? 35_000,
|
||||
connectionTimeoutMillis: options.connectionTimeoutMs ?? 5_000,
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ describe('createSqliteQueryExecutor', () => {
|
|||
const result = await executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true },
|
||||
connection: { driver: 'sqlite', path: 'warehouse.db' },
|
||||
sql: 'select status, count(*) as order_count from orders group by status order by status',
|
||||
maxRows: 10,
|
||||
});
|
||||
|
|
@ -60,7 +60,7 @@ describe('createSqliteQueryExecutor', () => {
|
|||
sqliteDatabasePathFromConnection({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', url: `file://${dbPath}`, readonly: true },
|
||||
connection: { driver: 'sqlite', url: `file://${dbPath}` },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).toBe(dbPath);
|
||||
|
|
@ -74,7 +74,7 @@ describe('createSqliteQueryExecutor', () => {
|
|||
sqliteDatabasePathFromConnection({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', path: `file:${pointerPath}`, readonly: true },
|
||||
connection: { driver: 'sqlite', path: `file:${pointerPath}` },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).toBe(dbPath);
|
||||
|
|
@ -89,7 +89,7 @@ describe('createSqliteQueryExecutor', () => {
|
|||
sqliteDatabasePathFromConnection({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true },
|
||||
connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).toBe(dbPath);
|
||||
|
|
@ -109,20 +109,20 @@ describe('createSqliteQueryExecutor', () => {
|
|||
executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true },
|
||||
connection: { driver: 'sqlite', path: 'warehouse.db' },
|
||||
sql: 'delete from orders',
|
||||
}),
|
||||
).rejects.toThrow('Only read-only SELECT/WITH queries can be executed locally');
|
||||
});
|
||||
|
||||
it('requires a SQLite driver, read-only config, and a database path', async () => {
|
||||
it('requires a SQLite driver and a database path', async () => {
|
||||
const executor = createSqliteQueryExecutor();
|
||||
|
||||
await expect(
|
||||
executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'postgres', path: 'warehouse.db', readonly: true },
|
||||
connection: { driver: 'postgres', path: 'warehouse.db' },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).rejects.toThrow('Local SQLite execution cannot run driver "postgres"');
|
||||
|
|
@ -131,16 +131,7 @@ describe('createSqliteQueryExecutor', () => {
|
|||
executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', path: 'warehouse.db', readonly: false },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).rejects.toThrow('Local query execution requires connections.warehouse.readonly: true');
|
||||
|
||||
await expect(
|
||||
executor.execute({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: tempDir,
|
||||
connection: { driver: 'sqlite', readonly: true },
|
||||
connection: { driver: 'sqlite' },
|
||||
sql: 'select 1',
|
||||
}),
|
||||
).rejects.toThrow('Local SQLite execution requires connections.warehouse.path or connections.warehouse.url');
|
||||
|
|
|
|||
|
|
@ -54,9 +54,6 @@ export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInpu
|
|||
if (driver !== 'sqlite' && driver !== 'sqlite3') {
|
||||
throw new Error(`Local SQLite execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`);
|
||||
}
|
||||
if (input.connection?.readonly !== true) {
|
||||
throw new Error(`Local query execution requires connections.${input.connectionId}.readonly: true.`);
|
||||
}
|
||||
|
||||
const pathValue = stringConfigValue(input.connection, 'path');
|
||||
const urlValue = stringConfigValue(input.connection, 'url');
|
||||
|
|
|
|||
|
|
@ -45,7 +45,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => {
|
|||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'postgres://localhost:5432/warehouse',
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
schemas: ['public'],
|
||||
|
|
@ -157,7 +156,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => {
|
|||
warehouse: {
|
||||
driver: 'postgresql',
|
||||
url: 'postgres://localhost:5432/warehouse',
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
baseUrl: `http://127.0.0.1:${address.port}`,
|
||||
|
|
@ -186,20 +184,18 @@ describe('createDaemonLiveDatabaseIntrospection', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('requires a configured read-only postgres connection with a url', async () => {
|
||||
it('requires a configured postgres connection with a url', async () => {
|
||||
const introspection = createDaemonLiveDatabaseIntrospection({
|
||||
connections: {
|
||||
warehouse: {
|
||||
driver: 'postgres',
|
||||
url: 'postgres://localhost:5432/warehouse',
|
||||
readonly: false,
|
||||
},
|
||||
},
|
||||
runJson: vi.fn(async () => daemonResponse),
|
||||
});
|
||||
|
||||
await expect(introspection.extractSchema('warehouse')).rejects.toThrow(
|
||||
'Local live-database ingest requires connections.warehouse.readonly: true.',
|
||||
'Local live-database ingest requires connections.warehouse.url.',
|
||||
);
|
||||
});
|
||||
|
||||
|
|
@ -210,7 +206,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => {
|
|||
warehouse: {
|
||||
driver: 'snowflake',
|
||||
url: 'snowflake://example',
|
||||
readonly: true,
|
||||
},
|
||||
},
|
||||
runJson,
|
||||
|
|
|
|||
|
|
@ -162,9 +162,6 @@ function requirePostgresConnection(
|
|||
if (driver !== 'postgres') {
|
||||
throw new Error(`Local live-database ingest cannot run driver "${connection?.driver ?? 'unknown'}".`);
|
||||
}
|
||||
if (connection?.readonly !== true) {
|
||||
throw new Error(`Local live-database ingest requires connections.${connectionId}.readonly: true.`);
|
||||
}
|
||||
if (typeof connection.url !== 'string' || connection.url.trim().length === 0) {
|
||||
throw new Error(`Local live-database ingest requires connections.${connectionId}.url.`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise<void> {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: postgres://localhost:5432/warehouse',
|
||||
' readonly: true',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
|
|
|
|||
|
|
@ -75,7 +75,6 @@ describe('createLocalProjectMcpContextPorts', () => {
|
|||
project.config.connections.warehouse = {
|
||||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
readonly: true,
|
||||
};
|
||||
const ports = createLocalProjectMcpContextPorts(project);
|
||||
|
||||
|
|
@ -89,7 +88,6 @@ describe('createLocalProjectMcpContextPorts', () => {
|
|||
project.config.connections.warehouse = {
|
||||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
readonly: true,
|
||||
};
|
||||
const connector = testConnector();
|
||||
const createConnector = vi.fn(async () => connector);
|
||||
|
|
@ -125,7 +123,6 @@ describe('createLocalProjectMcpContextPorts', () => {
|
|||
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
|
||||
project.config.connections.warehouse = {
|
||||
driver: 'postgres',
|
||||
readonly: true,
|
||||
};
|
||||
project.config.ingest.adapters = ['fake'];
|
||||
project.config.ingest.embeddings = {
|
||||
|
|
@ -633,7 +630,6 @@ describe('createLocalProjectMcpContextPorts', () => {
|
|||
project.config.connections.warehouse = {
|
||||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
readonly: true,
|
||||
};
|
||||
const shapeOnlyPorts = createLocalProjectMcpContextPorts(project);
|
||||
await shapeOnlyPorts.semanticLayer?.writeSource({
|
||||
|
|
@ -720,7 +716,6 @@ describe('createLocalProjectMcpContextPorts', () => {
|
|||
project.config.connections.warehouse = {
|
||||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
readonly: true,
|
||||
};
|
||||
const shapeOnlyPorts = createLocalProjectMcpContextPorts(project);
|
||||
await shapeOnlyPorts.semanticLayer?.writeSource({
|
||||
|
|
@ -958,7 +953,6 @@ describe('createLocalProjectMcpContextPorts', () => {
|
|||
project.config.connections.warehouse = {
|
||||
driver: 'postgres',
|
||||
url: 'postgres://localhost:5432/warehouse',
|
||||
readonly: true,
|
||||
};
|
||||
project.config.ingest.adapters = ['live-database'];
|
||||
project.config.llm = {
|
||||
|
|
@ -1034,7 +1028,6 @@ describe('createLocalProjectMcpContextPorts', () => {
|
|||
project.config.connections.warehouse = {
|
||||
driver: 'postgres',
|
||||
url: 'env:DATABASE_URL',
|
||||
readonly: true,
|
||||
};
|
||||
project.config.ingest.adapters = ['live-database'];
|
||||
const ports = createLocalProjectMcpContextPorts(project, {
|
||||
|
|
|
|||
|
|
@ -145,7 +145,7 @@ describe('createLocalProjectMemoryCapture', () => {
|
|||
|
||||
it('captures a semantic-layer source for a named local connection id', async () => {
|
||||
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
|
||||
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
|
||||
project.config.connections.warehouse = { driver: 'postgres' };
|
||||
const agentRunner = {
|
||||
runLoop: async ({
|
||||
toolSet,
|
||||
|
|
|
|||
|
|
@ -69,7 +69,6 @@ export interface KtxProjectScanConfig {
|
|||
export interface KtxProjectConnectionConfig {
|
||||
driver: string;
|
||||
url?: string;
|
||||
readonly?: boolean;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -110,7 +110,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise<void> {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
' readonly: true',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
|
|
@ -1006,7 +1005,6 @@ describe('local scan', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: env:DATABASE_URL',
|
||||
' readonly: true',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
|
|
@ -1363,7 +1361,6 @@ describe('local scan', () => {
|
|||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: warehouse.db',
|
||||
' readonly: true',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
|
|
@ -1396,7 +1393,6 @@ describe('local scan', () => {
|
|||
' warehouse:',
|
||||
' driver: mysql',
|
||||
' url: env:MYSQL_URL',
|
||||
' readonly: true',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
|
|
@ -1432,7 +1428,6 @@ describe('local scan', () => {
|
|||
' database: analytics',
|
||||
' username: reader',
|
||||
' password: env:CLICKHOUSE_PASSWORD',
|
||||
' readonly: true',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
|
|
@ -1468,7 +1463,6 @@ describe('local scan', () => {
|
|||
' database: analytics',
|
||||
' username: reader',
|
||||
' schema: dbo',
|
||||
' readonly: true',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ async function writeWarehouseConfig(projectDir: string): Promise<void> {
|
|||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: warehouse.db',
|
||||
' readonly: true',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ async function createProject(projectDir: string): Promise<void> {
|
|||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: warehouse.db',
|
||||
' readonly: true',
|
||||
'ingest:',
|
||||
' adapters:',
|
||||
' - live-database',
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ describe('compileLocalSlQuery', () => {
|
|||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-query-'));
|
||||
project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' });
|
||||
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
|
||||
project.config.connections.warehouse = { driver: 'postgres' };
|
||||
await project.fileStore.writeFile(
|
||||
'semantic-layer/warehouse/orders.yaml',
|
||||
`name: orders
|
||||
|
|
@ -222,7 +222,7 @@ grain: []
|
|||
expect(queryExecutor.execute).toHaveBeenCalledWith({
|
||||
connectionId: 'warehouse',
|
||||
projectDir: project.projectDir,
|
||||
connection: { driver: 'postgres', readonly: true },
|
||||
connection: { driver: 'postgres' },
|
||||
sql: 'select status, count(*) as order_count from public.orders group by status',
|
||||
maxRows: 10,
|
||||
});
|
||||
|
|
@ -248,7 +248,7 @@ grain: []
|
|||
});
|
||||
|
||||
it('requires connectionId when multiple connections are configured', async () => {
|
||||
project.config.connections.analytics = { driver: 'bigquery', readonly: true };
|
||||
project.config.connections.analytics = { driver: 'bigquery' };
|
||||
|
||||
await expect(
|
||||
compileLocalSlQuery(project, {
|
||||
|
|
|
|||
|
|
@ -231,7 +231,6 @@ async function main() {
|
|||
driver: 'sqlserver',
|
||||
url,
|
||||
schemas: ['dbo', 'HumanResources', 'Person', 'Production', 'Purchasing', 'Sales'],
|
||||
readonly: true,
|
||||
trustServerCertificate: true,
|
||||
},
|
||||
now: () => new Date('2026-05-07T00:00:00.000Z'),
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@ describe('standalone example docs', () => {
|
|||
config,
|
||||
/path: \.\.\/\.\.\/packages\/context\/test\/fixtures\/relationship-benchmarks\/orbit_style_product_no_declared_constraints\/data\.sqlite/,
|
||||
);
|
||||
assert.match(config, /readonly: true/);
|
||||
assert.match(config, /llm_proposals: false/);
|
||||
assert.match(config, /validation_required_for_manifest: true/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -92,7 +92,6 @@ export function buildKtxYaml(postgresUrl) {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
` url: "${postgresUrl}"`,
|
||||
' readonly: true',
|
||||
'storage:',
|
||||
' state: sqlite',
|
||||
' search: sqlite-fts5',
|
||||
|
|
|
|||
|
|
@ -59,7 +59,6 @@ describe('installed live-database artifact smoke helpers', () => {
|
|||
' warehouse:',
|
||||
' driver: postgres',
|
||||
' url: "postgresql://ktx:postgres@127.0.0.1:15432/warehouse"', // pragma: allowlist secret
|
||||
' readonly: true',
|
||||
'storage:',
|
||||
' state: sqlite',
|
||||
' search: sqlite-fts5',
|
||||
|
|
|
|||
|
|
@ -646,7 +646,6 @@ try {
|
|||
' warehouse:',
|
||||
' driver: sqlite',
|
||||
' path: warehouse.db',
|
||||
' readonly: true',
|
||||
'storage:',
|
||||
' state: sqlite',
|
||||
' search: sqlite-fts5',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue