mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-22 08:38:08 +02:00
test: split cli tests from source tree (#216)
* feat(cli): define full warehouse dialect contract
* test(cli): keep dialect edge tests focused
* fix(cli): stabilize dialect contract foundation
* refactor(connectors): own read-only query preparation
* refactor(connectors): resolve dialects through registry
* refactor(connectors): keep concrete dialect classes internal
* chore(workspace): enforce dialect import boundary
* refactor(cli): resolve relationship dialect at scan boundary
* refactor(cli): use dialect display parsing for entity details
* refactor(cli): use dialect display parsing for warehouse catalog
* refactor(cli): use dialect SQL in relationship workflows
* test(cli): verify solid dialect scan workflow closure
* test: split cli tests from source tree
* refactor(cli): standardize BigQuery scope listing
* feat(sqlite): implement connector scope listing
* test(connectors): cover required table listing
* feat(cli): add warehouse driver registry
* refactor(setup): route scope discovery through driver registry
* refactor(cli): route local query execution through driver registry
* refactor(historic-sql): route dialect support through driver registry
* refactor(cli): test warehouse connections through driver registry
* fix(cli): close driver registry type export gaps
* Improve setup daemon diagnostics
* refactor(setup): centralize rail-prefixed diagnostics + query-history fallback
Extract errorMessage, writePrefixedLines, and flushPrefixedBufferedCommandOutput
into clack.ts so the setup wizard, managed daemons, and embedding/agent steps
share one rail-formatted writer. setup-databases.ts also adds a
"disable query history and retry" option when the schema-context build fails
and query history is the likely culprit, surfaced via a new
failed-query-history-unavailable status.
* fix(cli): carry catalog through the picker so BigQuery/Snowflake/SQL Server scope filters match
The setup picker's KtxTableListEntry was a 2-level { schema, name }, so
qualifiedTableId always wrote db.name into enabled_tables. When BigQuery,
Snowflake, or SQL Server later ran fast ingest, their introspect step filtered
the scope set with scopedTableNames(scope, { catalog: projectId|database, db })
— catalog was non-null on the introspect side but null in the scope refs, so
every entry was rejected, the live-database adapter staged zero table files,
and detect() failed with 'Adapter "live-database" did not recognize fetched
source output'.
Align the picker boundary with the canonical 3-level KtxTableRef:
- Add catalog: string | null to KtxTableListEntry.
- BigQuery/Snowflake/SQL Server listTables populate catalog from the
resolved projectId / database; Postgres/MySQL/ClickHouse/SQLite set null.
- qualifiedTableId emits catalog.schema.name when catalog is non-null
(resolveEnabledTables already accepts the 3-part shape) and
schemasFromEnabledTables now goes through parseDottedTableEntry so it
recovers the schema correctly from both 2-part and 3-part entries.
- Export parseDottedTableEntry from enabled-tables.ts (@internal) for picker
reuse.
Update listTables expectations in all seven connector tests and the setup /
picker test fixtures. Add a picker regression test that covers the
catalog-bearing round-trip (save + refine).
* fix(cli): allow debug telemetry under opt-out env
This commit is contained in:
Andrey Avtomonov
GitHub
parent
924868841d
commit
56985b7e09
548 changed files with 5048 additions and 2228 deletions
|
|
@ -1,223 +0,0 @@
|
|||
import { access, mkdtemp, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { initKtxProject } from '../../context/project/project.js';
|
||||
import { createLocalProjectMemoryIngest } from './local-memory.js';
|
||||
import { LocalMemoryRunStore } from './local-memory-runs.js';
|
||||
|
||||
vi.mock('ai', () => ({
|
||||
generateText: vi.fn().mockResolvedValue({ text: '', toolCalls: [] }),
|
||||
stepCountIs: (stepBudget: number) => stepBudget,
|
||||
tool: (definition: unknown) => definition,
|
||||
}));
|
||||
|
||||
async function expectPathMissing(path: string): Promise<void> {
|
||||
await expect(access(path)).rejects.toThrow();
|
||||
}
|
||||
|
||||
describe('LocalMemoryRunStore', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-memory-runs-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('persists running, done, and reloadable memory run status in SQLite', async () => {
|
||||
const store = new LocalMemoryRunStore({
|
||||
projectDir: tempDir,
|
||||
idFactory: () => 'memory-run-1',
|
||||
});
|
||||
|
||||
const created = await store.createRunning({ inputHash: 'hash-1', chatId: 'chat-1' });
|
||||
expect(created).toEqual({ id: 'memory-run-1' });
|
||||
|
||||
await store.markRunning('memory-run-1', 'capturing');
|
||||
await store.markDone('memory-run-1', {
|
||||
signalDetected: true,
|
||||
actions: [{ target: 'wiki', type: 'created', key: 'revenue', detail: 'Revenue definition' }],
|
||||
skillsLoaded: ['wiki_capture'],
|
||||
commitHash: 'abc123',
|
||||
});
|
||||
|
||||
await expect(access(join(tempDir, '.ktx/db.sqlite'))).resolves.toBeUndefined();
|
||||
await expectPathMissing(join(tempDir, '.ktx/memory-runs/memory-run-1.json'));
|
||||
|
||||
await expect(store.findById('memory-run-1')).resolves.toMatchObject({
|
||||
id: 'memory-run-1',
|
||||
status: 'done',
|
||||
stage: 'done',
|
||||
inputHash: 'hash-1',
|
||||
chatId: 'chat-1',
|
||||
error: null,
|
||||
outputSummary: {
|
||||
signalDetected: true,
|
||||
commitHash: 'abc123',
|
||||
},
|
||||
});
|
||||
|
||||
const reloaded = new LocalMemoryRunStore({ projectDir: tempDir });
|
||||
await expect(reloaded.findById('memory-run-1')).resolves.toMatchObject({
|
||||
id: 'memory-run-1',
|
||||
status: 'done',
|
||||
stage: 'done',
|
||||
inputHash: 'hash-1',
|
||||
chatId: 'chat-1',
|
||||
outputSummary: {
|
||||
actions: [{ target: 'wiki', type: 'created', key: 'revenue', detail: 'Revenue definition' }],
|
||||
skillsLoaded: ['wiki_capture'],
|
||||
signalDetected: true,
|
||||
commitHash: 'abc123',
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('createLocalProjectMemoryIngest', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-memory-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('warns when embeddings are configured but memory ingest is created without an embedding provider', async () => {
|
||||
const project = await initKtxProject({ projectDir: tempDir });
|
||||
project.config.ingest.embeddings = {
|
||||
backend: 'openai',
|
||||
model: 'text-embedding-3-small',
|
||||
dimensions: 1536,
|
||||
};
|
||||
const logger = { log: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
||||
|
||||
createLocalProjectMemoryIngest(project, {
|
||||
agentRunner: { runLoop: vi.fn() } as never,
|
||||
logger: logger as never,
|
||||
});
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'[memory-ingest] embeddings backend "openai" is configured but no embedding provider was passed; semantic search will fall back to a no-op embedding port.',
|
||||
);
|
||||
});
|
||||
|
||||
it('captures a wiki page through the local memory agent and persists pollable status', async () => {
|
||||
const project = await initKtxProject({ projectDir: tempDir });
|
||||
const agentRunner = {
|
||||
runLoop: async ({
|
||||
toolSet,
|
||||
}: {
|
||||
toolSet: Record<string, { execute: (input: unknown, options?: { toolCallId?: string }) => Promise<unknown> }>;
|
||||
}) => {
|
||||
await toolSet.load_skill.execute({ name: 'wiki_capture' });
|
||||
await toolSet.wiki_write.execute(
|
||||
{
|
||||
key: 'revenue',
|
||||
summary: 'Revenue definition',
|
||||
content: 'Revenue means paid order value net of refunds.',
|
||||
tags: ['finance'],
|
||||
},
|
||||
{ toolCallId: 'wiki-write' },
|
||||
);
|
||||
return { stopReason: 'natural' as const };
|
||||
},
|
||||
};
|
||||
|
||||
const ingest = createLocalProjectMemoryIngest(project, {
|
||||
agentRunner: agentRunner as never,
|
||||
runIdFactory: () => 'memory-run-1',
|
||||
});
|
||||
|
||||
await expect(
|
||||
ingest.ingest({
|
||||
userId: 'local-user',
|
||||
chatId: 'chat-1',
|
||||
userMessage: 'define revenue as paid order value net of refunds',
|
||||
assistantMessage: 'Captured.',
|
||||
sourceType: 'external_ingest',
|
||||
}),
|
||||
).resolves.toEqual({ runId: 'memory-run-1' });
|
||||
await ingest.waitForRun('memory-run-1');
|
||||
|
||||
await expect(access(join(project.projectDir, '.ktx/db.sqlite'))).resolves.toBeUndefined();
|
||||
await expectPathMissing(join(project.projectDir, '.ktx/memory-runs/memory-run-1.json'));
|
||||
|
||||
await expect(ingest.status('memory-run-1')).resolves.toMatchObject({
|
||||
runId: 'memory-run-1',
|
||||
status: 'done',
|
||||
done: true,
|
||||
captured: { wiki: ['revenue'], sl: [], xrefs: [] },
|
||||
skillsLoaded: ['wiki_capture'],
|
||||
signalDetected: true,
|
||||
});
|
||||
|
||||
await expect(readFile(join(project.projectDir, 'wiki/global/revenue.md'), 'utf-8')).resolves.toContain(
|
||||
'Revenue means paid order value net of refunds.',
|
||||
);
|
||||
});
|
||||
|
||||
it('captures a semantic-layer source for a named local connection id', async () => {
|
||||
const project = await initKtxProject({ projectDir: tempDir });
|
||||
project.config.connections.warehouse = { driver: 'postgres' };
|
||||
const agentRunner = {
|
||||
runLoop: async ({
|
||||
toolSet,
|
||||
}: {
|
||||
toolSet: Record<string, { execute: (input: unknown, options?: { toolCallId?: string }) => Promise<unknown> }>;
|
||||
}) => {
|
||||
await toolSet.load_skill.execute({ name: 'sl' });
|
||||
await toolSet.sl_write_source.execute(
|
||||
{
|
||||
connectionId: 'warehouse',
|
||||
sourceName: 'orders',
|
||||
source: {
|
||||
name: 'orders',
|
||||
table: 'public.orders',
|
||||
grain: ['id'],
|
||||
columns: [{ name: 'id', type: 'number' }],
|
||||
joins: [],
|
||||
measures: [{ name: 'order_count', expr: 'count(*)' }],
|
||||
},
|
||||
},
|
||||
{ toolCallId: 'sl-write' },
|
||||
);
|
||||
return { stopReason: 'natural' as const };
|
||||
},
|
||||
};
|
||||
|
||||
const ingest = createLocalProjectMemoryIngest(project, {
|
||||
agentRunner: agentRunner as never,
|
||||
runIdFactory: () => 'memory-run-2',
|
||||
});
|
||||
|
||||
await ingest.ingest({
|
||||
userId: 'local-user',
|
||||
chatId: 'chat-2',
|
||||
userMessage: 'going forward define orders count as count of public orders',
|
||||
assistantMessage: 'Captured.',
|
||||
connectionId: 'warehouse',
|
||||
sourceType: 'external_ingest',
|
||||
});
|
||||
await ingest.waitForRun('memory-run-2');
|
||||
|
||||
await expect(access(join(project.projectDir, '.ktx/db.sqlite'))).resolves.toBeUndefined();
|
||||
await expectPathMissing(join(project.projectDir, '.ktx/memory-runs/memory-run-2.json'));
|
||||
|
||||
await expect(ingest.status('memory-run-2')).resolves.toMatchObject({
|
||||
runId: 'memory-run-2',
|
||||
status: 'done',
|
||||
captured: { wiki: [], sl: ['orders'], xrefs: [] },
|
||||
skillsLoaded: ['sl'],
|
||||
signalDetected: true,
|
||||
});
|
||||
await expect(readFile(join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), 'utf-8')).resolves.toContain(
|
||||
'order_count',
|
||||
);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue